From 15f455387d9545bbb9c463db6fd43f0d102827ad Mon Sep 17 00:00:00 2001 From: jason Date: Mon, 9 Mar 2026 00:58:19 -0500 Subject: [PATCH 1/3] Add automatic migration system with schema validation --- server/db/migrations.js | 321 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 server/db/migrations.js diff --git a/server/db/migrations.js b/server/db/migrations.js new file mode 100644 index 0000000..5ff9c37 --- /dev/null +++ b/server/db/migrations.js @@ -0,0 +1,321 @@ +const Database = require('better-sqlite3'); +const path = require('path'); +const fs = require('fs'); + +/** + * Migration System for BREEDR + * Automatically runs on startup to ensure schema is correct + */ + +class MigrationRunner { + constructor(dbPath) { + this.dbPath = dbPath; + this.db = null; + } + + connect() { + this.db = new Database(this.dbPath); + this.db.pragma('foreign_keys = ON'); + } + + close() { + if (this.db) { + this.db.close(); + } + } + + // Get current schema version from database + getSchemaVersion() { + try { + const result = this.db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); + return result ? result.version : 0; + } catch (error) { + // schema_version table doesn't exist, create it + this.db.exec(` + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER PRIMARY KEY, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP, + description TEXT + ) + `); + return 0; + } + } + + // Record migration completion + recordMigration(version, description) { + this.db.prepare('INSERT OR IGNORE INTO schema_version (version, description) VALUES (?, ?)').run(version, description); + } + + // Check if dogs table has old sire/dam columns + hasOldSchema() { + const columns = this.db.prepare("PRAGMA table_info(dogs)").all(); + return columns.some(col => col.name === 'sire' || col.name === 'dam'); + } + + // Check if litter_id column exists + hasLitterIdColumn() { + const columns = this.db.prepare("PRAGMA table_info(dogs)").all(); + return columns.some(col => col.name === 'litter_id'); + } + + // Migration 1: Remove sire/dam columns, use parents table + migration001_removeOldParentColumns() { + console.log('[Migration 001] Checking for old sire/dam columns...'); + + if (!this.hasOldSchema()) { + console.log('[Migration 001] Schema is already correct, skipping'); + return; + } + + console.log('[Migration 001] Found old schema with sire/dam columns'); + console.log('[Migration 001] Migrating to parents table...'); + + this.db.exec('BEGIN TRANSACTION'); + + try { + // Ensure parents table exists + this.db.exec(` + CREATE TABLE IF NOT EXISTS parents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dog_id INTEGER NOT NULL, + parent_id INTEGER NOT NULL, + parent_type TEXT NOT NULL CHECK(parent_type IN ('sire', 'dam')), + FOREIGN KEY (dog_id) REFERENCES dogs(id) ON DELETE CASCADE, + FOREIGN KEY (parent_id) REFERENCES dogs(id) ON DELETE CASCADE, + UNIQUE(dog_id, parent_type) + ); + + CREATE INDEX IF NOT EXISTS idx_parents_dog ON parents(dog_id); + CREATE INDEX IF NOT EXISTS idx_parents_parent ON parents(parent_id); + `); + + // Backup current dogs table + this.db.exec('DROP TABLE IF EXISTS dogs_migration_backup'); + this.db.exec('CREATE TABLE dogs_migration_backup AS SELECT * FROM dogs'); + + const backupCount = this.db.prepare('SELECT COUNT(*) as count FROM dogs_migration_backup').get(); + console.log(`[Migration 001] Backed up ${backupCount.count} dogs`); + + // Migrate parent relationships to parents table + const columns = this.db.prepare("PRAGMA table_info(dogs_migration_backup)").all(); + const hasSire = columns.some(col => col.name === 'sire'); + const hasDam = columns.some(col => col.name === 'dam'); + const hasLitterId = columns.some(col => col.name === 'litter_id'); + + if (hasSire) { + const sireResult = this.db.prepare(` + INSERT OR IGNORE INTO parents (dog_id, parent_id, parent_type) + SELECT id, sire, 'sire' FROM dogs_migration_backup WHERE sire IS NOT NULL + `).run(); + console.log(`[Migration 001] Migrated ${sireResult.changes} sire relationships`); + } + + if (hasDam) { + const damResult = this.db.prepare(` + INSERT OR IGNORE INTO parents (dog_id, parent_id, parent_type) + SELECT id, dam, 'dam' FROM dogs_migration_backup WHERE dam IS NOT NULL + `).run(); + console.log(`[Migration 001] Migrated ${damResult.changes} dam relationships`); + } + + // Drop old dogs table + this.db.exec('DROP TABLE dogs'); + console.log('[Migration 001] Dropped old dogs table'); + + // Create new dogs table with correct schema + this.db.exec(` + CREATE TABLE dogs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + registration_number TEXT, + microchip TEXT, + sex TEXT CHECK(sex IN ('male', 'female')), + birth_date DATE, + breed TEXT, + color TEXT, + weight REAL, + height REAL, + notes TEXT, + litter_id INTEGER, + photo_urls TEXT, + is_active INTEGER DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (litter_id) REFERENCES litters(id) ON DELETE SET NULL + ) + `); + console.log('[Migration 001] Created new dogs table'); + + // Restore data (excluding sire/dam columns) + const columnList = ['id', 'name', 'registration_number', 'microchip', 'sex', 'birth_date', 'breed', 'color', 'weight', 'height', 'notes', 'photo_urls', 'is_active', 'created_at', 'updated_at']; + if (hasLitterId) { + columnList.splice(11, 0, 'litter_id'); // Insert after notes + } + + const columnsStr = columnList.join(', '); + this.db.exec(`INSERT INTO dogs (${columnsStr}) SELECT ${columnsStr} FROM dogs_migration_backup`); + + const restoredCount = this.db.prepare('SELECT COUNT(*) as count FROM dogs').get(); + console.log(`[Migration 001] Restored ${restoredCount.count} dogs`); + + // Create indexes + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_dogs_microchip + ON dogs(microchip) WHERE microchip IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_dogs_name ON dogs(name); + CREATE INDEX IF NOT EXISTS idx_dogs_registration ON dogs(registration_number); + `); + + // Clean up backup + this.db.exec('DROP TABLE dogs_migration_backup'); + + this.db.exec('COMMIT'); + console.log('[Migration 001] ✓ Migration complete!'); + + } catch (error) { + this.db.exec('ROLLBACK'); + console.error('[Migration 001] ✗ Migration failed:', error.message); + throw error; + } + } + + // Migration 2: Add litter_id column if missing + migration002_addLitterIdColumn() { + console.log('[Migration 002] Checking for litter_id column...'); + + if (this.hasLitterIdColumn()) { + console.log('[Migration 002] litter_id column already exists, skipping'); + return; + } + + console.log('[Migration 002] Adding litter_id column...'); + + try { + this.db.exec(` + ALTER TABLE dogs ADD COLUMN litter_id INTEGER + REFERENCES litters(id) ON DELETE SET NULL + `); + console.log('[Migration 002] ✓ litter_id column added'); + } catch (error) { + console.error('[Migration 002] ✗ Failed to add litter_id:', error.message); + throw error; + } + } + + // Validate final schema + validateSchema() { + console.log('[Validation] Checking database schema...'); + + const checks = [ + { + name: 'Dogs table exists', + test: () => { + const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='dogs'").all(); + return tables.length > 0; + } + }, + { + name: 'Dogs table has no sire/dam columns', + test: () => { + const columns = this.db.prepare("PRAGMA table_info(dogs)").all(); + return !columns.some(col => col.name === 'sire' || col.name === 'dam'); + } + }, + { + name: 'Parents table exists', + test: () => { + const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='parents'").all(); + return tables.length > 0; + } + }, + { + name: 'Litter_id column exists', + test: () => this.hasLitterIdColumn() + }, + { + name: 'Litters table exists', + test: () => { + const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='litters'").all(); + return tables.length > 0; + } + } + ]; + + let allPassed = true; + checks.forEach(check => { + const passed = check.test(); + const status = passed ? '✓' : '✗'; + console.log(`[Validation] ${status} ${check.name}`); + if (!passed) allPassed = false; + }); + + if (allPassed) { + console.log('[Validation] ✓ All schema checks passed!'); + } else { + console.warn('[Validation] ⚠ Some schema checks failed'); + } + + return allPassed; + } + + // Run all migrations + runMigrations() { + console.log('\n' + '='.repeat(60)); + console.log('BREEDR Database Migration System'); + console.log('='.repeat(60)); + console.log(`Database: ${this.dbPath}\n`); + + this.connect(); + + try { + const currentVersion = this.getSchemaVersion(); + console.log(`Current schema version: ${currentVersion}\n`); + + // Run migrations in order + if (currentVersion < 1) { + this.migration001_removeOldParentColumns(); + this.recordMigration(1, 'Migrate sire/dam columns to parents table'); + } + + if (currentVersion < 2) { + this.migration002_addLitterIdColumn(); + this.recordMigration(2, 'Add litter_id column to dogs table'); + } + + // Validate final schema + console.log(''); + const isValid = this.validateSchema(); + + const finalVersion = this.getSchemaVersion(); + console.log('\n' + '='.repeat(60)); + console.log(`Schema version: ${currentVersion} → ${finalVersion}`); + console.log('Migration system complete!'); + console.log('='.repeat(60) + '\n'); + + return isValid; + + } catch (error) { + console.error('\n✗ Migration system failed:', error.message); + console.error(error.stack); + throw error; + } finally { + this.close(); + } + } +} + +// Function to run migrations +function runMigrations(dbPath) { + const runner = new MigrationRunner(dbPath); + return runner.runMigrations(); +} + +module.exports = { MigrationRunner, runMigrations }; + +// Run migrations if called directly +if (require.main === module) { + const dbPath = process.env.DB_PATH || path.join(__dirname, '../../data/breedr.db'); + runMigrations(dbPath); +} -- 2.49.1 From a11dec6d2932dc2a368c5276918aa9c59df0b171 Mon Sep 17 00:00:00 2001 From: jason Date: Mon, 9 Mar 2026 00:58:39 -0500 Subject: [PATCH 2/3] Add automatic migration system on startup --- server/index.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/server/index.js b/server/index.js index 230646d..8c22137 100644 --- a/server/index.js +++ b/server/index.js @@ -4,6 +4,7 @@ const helmet = require('helmet'); const path = require('path'); const fs = require('fs'); const { initDatabase } = require('./db/init'); +const { runMigrations } = require('./db/migrations'); const app = express(); const PORT = process.env.PORT || 3000; @@ -19,9 +20,22 @@ if (!fs.existsSync(UPLOAD_PATH)) { fs.mkdirSync(UPLOAD_PATH, { recursive: true }); } -// Initialize database +// Initialize database schema (creates tables if they don't exist) initDatabase(DB_PATH); +// Run migrations to ensure schema is up-to-date +try { + console.log('Running database migrations...'); + runMigrations(DB_PATH); + console.log('Database migrations complete!\n'); +} catch (error) { + console.error('\n⚠️ Database migration failed!'); + console.error('Error:', error.message); + console.error('\nThe application may not function correctly.'); + console.error('Please check the database and try again.\n'); + // Don't exit - let the app try to start anyway +} + // Middleware app.use(helmet({ contentSecurityPolicy: false, // Allow inline scripts for React @@ -67,13 +81,13 @@ app.use((err, req, res, next) => { // Start server app.listen(PORT, '0.0.0.0', () => { console.log(`\n🐕 BREEDR Server Running`); - console.log(`================================`); + console.log(`=============================`); console.log(`Environment: ${process.env.NODE_ENV || 'development'}`); console.log(`Port: ${PORT}`); console.log(`Database: ${DB_PATH}`); console.log(`Uploads: ${UPLOAD_PATH}`); console.log(`Access: http://localhost:${PORT}`); - console.log(`================================\n`); + console.log(`=============================\n`); }); -module.exports = app; \ No newline at end of file +module.exports = app; -- 2.49.1 From 9c5d06d964465ed9512b796e49941555dc1b6aa6 Mon Sep 17 00:00:00 2001 From: jason Date: Mon, 9 Mar 2026 00:59:26 -0500 Subject: [PATCH 3/3] Add comprehensive database migration documentation --- DATABASE_MIGRATIONS.md | 345 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 DATABASE_MIGRATIONS.md diff --git a/DATABASE_MIGRATIONS.md b/DATABASE_MIGRATIONS.md new file mode 100644 index 0000000..28642ee --- /dev/null +++ b/DATABASE_MIGRATIONS.md @@ -0,0 +1,345 @@ +# BREEDR Database Migrations + +## Automatic Migration System + +BREEDR now includes an **automatic migration system** that runs on every startup to ensure your database schema is always correct and up-to-date. + +## How It Works + +### On Every Startup + +1. **Initialize Database** - Creates tables if they don't exist +2. **Run Migrations** - Automatically fixes schema issues +3. **Validate Schema** - Verifies everything is correct +4. **Start Application** - Server begins accepting requests + +You don't need to do anything manually! + +--- + +## Migrations Included + +### Migration 001: Remove Old Parent Columns + +**Problem**: Old schema had `sire` and `dam` columns in the `dogs` table causing "no such column: sire" errors. + +**Solution**: +- Creates `parents` table for relationships +- Migrates existing sire/dam data to `parents` table +- Recreates `dogs` table without sire/dam columns +- Preserves all existing dog data + +**Automatic**: Runs only if old schema detected + +### Migration 002: Add Litter ID Column + +**Problem**: Dogs table missing `litter_id` column for linking puppies to litters. + +**Solution**: +- Adds `litter_id` column to `dogs` table +- Creates foreign key to `litters` table + +**Automatic**: Runs only if column is missing + +--- + +## Schema Version Tracking + +The migration system uses a `schema_version` table to track which migrations have been applied: + +```sql +CREATE TABLE schema_version ( + version INTEGER PRIMARY KEY, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP, + description TEXT +); +``` + +Each migration runs only once, even if you restart the server multiple times. + +--- + +## Correct Schema (Current) + +### Dogs Table (No sire/dam columns!) + +```sql +CREATE TABLE dogs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + registration_number TEXT, + microchip TEXT, + sex TEXT CHECK(sex IN ('male', 'female')), + birth_date DATE, + breed TEXT, + color TEXT, + weight REAL, + height REAL, + notes TEXT, + litter_id INTEGER, + photo_urls TEXT, + is_active INTEGER DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (litter_id) REFERENCES litters(id) ON DELETE SET NULL +); +``` + +### Parents Table (Relationships) + +```sql +CREATE TABLE parents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dog_id INTEGER NOT NULL, + parent_id INTEGER NOT NULL, + parent_type TEXT NOT NULL CHECK(parent_type IN ('sire', 'dam')), + FOREIGN KEY (dog_id) REFERENCES dogs(id) ON DELETE CASCADE, + FOREIGN KEY (parent_id) REFERENCES dogs(id) ON DELETE CASCADE, + UNIQUE(dog_id, parent_type) +); +``` + +--- + +## How to Use + +### Normal Startup (Automatic) + +Just start your application normally: + +```bash +# With Docker +docker-compose up -d + +# Without Docker +cd server && npm start +``` + +Migrations run automatically! + +### Check Migration Logs + +Look at the server console output: + +``` +============================================================ +BREEDR Database Migration System +============================================================ +Database: /app/data/breedr.db + +Current schema version: 0 + +[Migration 001] Checking for old sire/dam columns... +[Migration 001] Found old schema with sire/dam columns +[Migration 001] Migrating to parents table... +[Migration 001] Backed up 15 dogs +[Migration 001] Migrated 8 sire relationships +[Migration 001] Migrated 8 dam relationships +[Migration 001] Dropped old dogs table +[Migration 001] Created new dogs table +[Migration 001] Restored 15 dogs +[Migration 001] ✓ Migration complete! + +[Migration 002] Checking for litter_id column... +[Migration 002] litter_id column already exists, skipping + +[Validation] ✓ Dogs table exists +[Validation] ✓ Dogs table has no sire/dam columns +[Validation] ✓ Parents table exists +[Validation] ✓ Litter_id column exists +[Validation] ✓ Litters table exists +[Validation] ✓ All schema checks passed! + +============================================================ +Schema version: 0 → 2 +Migration system complete! +============================================================ +``` + +### Manual Migration (If Needed) + +You can also run migrations manually: + +```bash +node server/db/migrations.js +``` + +--- + +## Fresh Install + +For a fresh installation: + +1. **No database file exists** → `init.js` creates correct schema +2. **Migrations check schema** → Everything already correct, no migration needed +3. **Application starts** → Ready to use! + +**Result**: Fresh installs automatically have the correct schema. + +--- + +## Upgrading from Old Version + +For existing installations with old schema: + +1. **Old database detected** → Migration system kicks in +2. **Data is backed up** → Safety first! +3. **Schema is updated** → Sire/dam data moved to parents table +4. **Data is restored** → All your dogs are preserved +5. **Application starts** → Now using correct schema! + +**Result**: Existing data is preserved and schema is fixed automatically. + +--- + +## Docker Integration + +### Dockerfile + +No changes needed! Migrations run automatically when the container starts. + +### docker-compose.yml + +No changes needed! Just restart: + +```bash +docker-compose restart +``` + +Or rebuild: + +```bash +docker-compose down +docker-compose up --build -d +``` + +--- + +## Troubleshooting + +### Migration Failed + +If you see an error: + +``` +⚠️ Database migration failed! +Error: [error message] +``` + +1. **Check the error message** - It will tell you what went wrong +2. **Check database file permissions** - Make sure the file is writable +3. **Check disk space** - Ensure you have enough space +4. **Try manual migration**: + ```bash + node server/db/migrations.js + ``` + +### Database is Locked + +If migrations fail with "database is locked": + +1. Stop all running instances of BREEDR +2. Check for zombie processes: `ps aux | grep node` +3. Kill any old processes: `kill ` +4. Restart BREEDR + +### Migration Keeps Running + +If the same migration runs every time: + +1. Check `schema_version` table: + ```sql + SELECT * FROM schema_version; + ``` +2. If empty, migration isn't being recorded +3. Check for database transaction issues +4. Manually add version: + ```sql + INSERT INTO schema_version (version, description) VALUES (1, 'Manual fix'); + ``` + +### Want to Start Fresh + +To completely reset the database: + +1. **Stop BREEDR** +2. **Backup your data** (optional): + ```bash + cp data/breedr.db data/breedr.db.backup + ``` +3. **Delete database**: + ```bash + rm data/breedr.db + ``` +4. **Restart BREEDR** - Fresh database will be created + +--- + +## Validation Checks + +The migration system validates your schema: + +- ✓ Dogs table exists +- ✓ Dogs table has no sire/dam columns +- ✓ Parents table exists +- ✓ Litter_id column exists +- ✓ Litters table exists + +If any check fails, you'll see a warning. + +--- + +## Adding New Migrations + +If you need to add a new migration: + +1. **Edit `server/db/migrations.js`** +2. **Add new migration function**: + ```javascript + migration003_yourNewMigration() { + console.log('[Migration 003] Doing something...'); + // Your migration code here + } + ``` +3. **Add to runMigrations()**: + ```javascript + if (currentVersion < 3) { + this.migration003_yourNewMigration(); + this.recordMigration(3, 'Description of migration'); + } + ``` +4. **Test thoroughly** before deploying + +--- + +## Best Practices + +1. **Let migrations run automatically** - Don't skip them +2. **Check logs on startup** - Verify migrations succeeded +3. **Backup before major updates** - Safety first +4. **Test in development** - Before deploying to production +5. **Monitor schema_version** - Know what version you're on + +--- + +## Schema Version History + +| Version | Description | Date | +|---------|-------------|------| +| 0 | Initial schema (may have sire/dam columns) | - | +| 1 | Migrated to parents table | March 2026 | +| 2 | Added litter_id column | March 2026 | + +--- + +## Summary + +✅ **Migrations run automatically on every startup** +✅ **No manual intervention needed** +✅ **Data is preserved during migrations** +✅ **Schema is validated after migrations** +✅ **Works with Docker and standalone** +✅ **Fresh installs get correct schema** +✅ **Old installs are automatically upgraded** + +**The "no such column: sire" error is now fixed automatically!** 🎉 -- 2.49.1