marketing-app/server/migrate-data.js

#!/usr/bin/env node
/**
 * migrate-data.js — One-time migration from SQLite to NocoDB.
 * Run: node migrate-data.js
 */

require('dotenv').config({ path: __dirname + '/.env' });
const Database = require('better-sqlite3');
const bcrypt = require('bcrypt');
const path = require('path');
const nocodb = require('./nocodb');
const { authDb } = require('./auth-db');

const sqliteDb = new Database(path.join(__dirname, 'marketing.db'), { readonly: true });

// ID mapping: { tableName: { oldId: newNocoDbId } }
const idMap = {};
function mapId(table, oldId, newId) {
  if (!idMap[table]) idMap[table] = {};
  idMap[table][oldId] = newId;
}
function getMappedId(table, oldId) {
  if (!oldId) return null;
  return idMap[table]?.[oldId] || null;
}

// Build a lookup: team_member_id → user row
function buildTeamMemberToUserMap() {
  const users = sqliteDb.prepare('SELECT id, team_member_id FROM users').all();
  const map = {};
  for (const u of users) {
    if (u.team_member_id) map[u.team_member_id] = u.id;
  }
  return map;
}

async function migrateBrands() {
  console.log('Migrating Brands...');
  const rows = sqliteDb.prepare('SELECT * FROM brands ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('Brands', {
      name: row.name,
      priority: row.priority,
      color: row.color,
      icon: row.icon,
    });
    mapId('brands', row.id, created.Id);
  }
  console.log(`  ✅ ${rows.length} brands migrated`);
}

async function migrateUsers() {
  console.log('Migrating Users...');
  const rows = sqliteDb.prepare('SELECT * FROM users ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('Users', {
      name: row.name,
      email: row.email,
      role: row.role,
      team_role: row.team_role || null,
      brands: row.brands || '[]',
      phone: row.phone || null,
      avatar: row.avatar || null,
      tutorial_completed: !!row.tutorial_completed,
    });
    mapId('users', row.id, created.Id);

    // Create auth credentials entry
    authDb.prepare(
      'INSERT OR REPLACE INTO auth_credentials (email, password_hash, nocodb_user_id) VALUES (?, ?, ?)'
    ).run(row.email, row.password_hash, created.Id);
  }
  console.log(`  ✅ ${rows.length} users migrated (+ auth_credentials)`);
}

async function migrateCampaigns() {
  console.log('Migrating Campaigns...');
  const rows = sqliteDb.prepare('SELECT * FROM campaigns ORDER BY id').all();
  for (const row of rows) {
    const data = {
      name: row.name,
      description: row.description,
      start_date: row.start_date,
      end_date: row.end_date,
      status: row.status,
      color: row.color,
      budget: row.budget,
      goals: row.goals,
      platforms: row.platforms || '[]',
      budget_spent: row.budget_spent || 0,
      revenue: row.revenue || 0,
      impressions: row.impressions || 0,
      clicks: row.clicks || 0,
      conversions: row.conversions || 0,
      cost_per_click: row.cost_per_click || 0,
      notes: row.notes || '',
    };

    const created = await nocodb.create('Campaigns', data);
    mapId('campaigns', row.id, created.Id);

    // Link Brand
    if (row.brand_id && getMappedId('brands', row.brand_id)) {
      await linkRecord('Campaigns', created.Id, 'Brand', getMappedId('brands', row.brand_id));
    }
    // Link CreatedByUser
    if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
      await linkRecord('Campaigns', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
    }
  }
  console.log(`  ✅ ${rows.length} campaigns migrated`);
}

async function migrateCampaignTracks() {
  console.log('Migrating CampaignTracks...');
  const rows = sqliteDb.prepare('SELECT * FROM campaign_tracks ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('CampaignTracks', {
      name: row.name,
      type: row.type,
      platform: row.platform,
      budget_allocated: row.budget_allocated || 0,
      budget_spent: row.budget_spent || 0,
      revenue: row.revenue || 0,
      impressions: row.impressions || 0,
      clicks: row.clicks || 0,
      conversions: row.conversions || 0,
      notes: row.notes || '',
      status: row.status,
    });
    mapId('campaign_tracks', row.id, created.Id);

    if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
      await linkRecord('CampaignTracks', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
    }
  }
  console.log(`  ✅ ${rows.length} campaign tracks migrated`);
}

async function migrateCampaignAssignments() {
  console.log('Migrating CampaignAssignments...');
  const rows = sqliteDb.prepare('SELECT * FROM campaign_assignments ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('CampaignAssignments', {
      assigned_at: row.assigned_at,
    });
    mapId('campaign_assignments', row.id, created.Id);

    if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
      await linkRecord('CampaignAssignments', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
    }
    if (row.user_id && getMappedId('users', row.user_id)) {
      await linkRecord('CampaignAssignments', created.Id, 'Member', getMappedId('users', row.user_id));
    }
    if (row.assigned_by && getMappedId('users', row.assigned_by)) {
      await linkRecord('CampaignAssignments', created.Id, 'Assigner', getMappedId('users', row.assigned_by));
    }
  }
  console.log(`  ✅ ${rows.length} campaign assignments migrated`);
}

async function migrateProjects() {
  console.log('Migrating Projects...');
  const rows = sqliteDb.prepare('SELECT * FROM projects ORDER BY id').all();
  const tmToUser = buildTeamMemberToUserMap();
  for (const row of rows) {
    const created = await nocodb.create('Projects', {
      name: row.name,
      description: row.description,
      status: row.status,
      priority: row.priority,
      due_date: row.due_date,
    });
    mapId('projects', row.id, created.Id);

    if (row.brand_id && getMappedId('brands', row.brand_id)) {
      await linkRecord('Projects', created.Id, 'Brand', getMappedId('brands', row.brand_id));
    }
    // owner_id references team_members, map through to users
    if (row.owner_id) {
      const userId = tmToUser[row.owner_id];
      if (userId && getMappedId('users', userId)) {
        await linkRecord('Projects', created.Id, 'Owner', getMappedId('users', userId));
      }
    }
    if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
      await linkRecord('Projects', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
    }
  }
  console.log(`  ✅ ${rows.length} projects migrated`);
}

async function migrateTasks() {
  console.log('Migrating Tasks...');
  const rows = sqliteDb.prepare('SELECT * FROM tasks ORDER BY id').all();
  const tmToUser = buildTeamMemberToUserMap();
  for (const row of rows) {
    const created = await nocodb.create('Tasks', {
      title: row.title,
      description: row.description,
      status: row.status,
      priority: row.priority,
      due_date: row.due_date,
      is_personal: !!row.is_personal,
      completed_at: row.completed_at,
    });
    mapId('tasks', row.id, created.Id);

    if (row.project_id && getMappedId('projects', row.project_id)) {
      await linkRecord('Tasks', created.Id, 'Project', getMappedId('projects', row.project_id));
    }
    // assigned_to references team_members
    if (row.assigned_to) {
      const userId = tmToUser[row.assigned_to];
      if (userId && getMappedId('users', userId)) {
        await linkRecord('Tasks', created.Id, 'AssignedTo', getMappedId('users', userId));
      }
    }
    if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
      await linkRecord('Tasks', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
    }
  }
  console.log(`  ✅ ${rows.length} tasks migrated`);
}

async function migratePosts() {
  console.log('Migrating Posts...');
  const rows = sqliteDb.prepare('SELECT * FROM posts ORDER BY id').all();
  const tmToUser = buildTeamMemberToUserMap();
  for (const row of rows) {
    const created = await nocodb.create('Posts', {
      title: row.title,
      description: row.description,
      status: row.status,
      platform: row.platform,
      platforms: row.platforms || '[]',
      content_type: row.content_type,
      scheduled_date: row.scheduled_date,
      published_date: row.published_date,
      notes: row.notes,
      publication_links: row.publication_links || '[]',
    });
    mapId('posts', row.id, created.Id);

    if (row.brand_id && getMappedId('brands', row.brand_id)) {
      await linkRecord('Posts', created.Id, 'Brand', getMappedId('brands', row.brand_id));
    }
    if (row.assigned_to) {
      const userId = tmToUser[row.assigned_to];
      if (userId && getMappedId('users', userId)) {
        await linkRecord('Posts', created.Id, 'AssignedTo', getMappedId('users', userId));
      }
    }
    if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
      await linkRecord('Posts', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
    }
    if (row.track_id && getMappedId('campaign_tracks', row.track_id)) {
      await linkRecord('Posts', created.Id, 'Track', getMappedId('campaign_tracks', row.track_id));
    }
    if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
      await linkRecord('Posts', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
    }
  }
  console.log(`  ✅ ${rows.length} posts migrated`);
}

async function migratePostAttachments() {
  console.log('Migrating PostAttachments...');
  const rows = sqliteDb.prepare('SELECT * FROM post_attachments ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('PostAttachments', {
      filename: row.filename,
      original_name: row.original_name,
      mime_type: row.mime_type,
      size: row.size,
      url: row.url,
    });
    mapId('post_attachments', row.id, created.Id);

    if (row.post_id && getMappedId('posts', row.post_id)) {
      await linkRecord('PostAttachments', created.Id, 'Post', getMappedId('posts', row.post_id));
    }
  }
  console.log(`  ✅ ${rows.length} post attachments migrated`);
}

async function migrateAssets() {
  console.log('Migrating Assets...');
  const rows = sqliteDb.prepare('SELECT * FROM assets ORDER BY id').all();
  const tmToUser = buildTeamMemberToUserMap();
  for (const row of rows) {
    const created = await nocodb.create('Assets', {
      filename: row.filename,
      original_name: row.original_name,
      mime_type: row.mime_type,
      size: row.size,
      tags: row.tags || '[]',
      folder: row.folder,
    });
    mapId('assets', row.id, created.Id);

    if (row.brand_id && getMappedId('brands', row.brand_id)) {
      await linkRecord('Assets', created.Id, 'Brand', getMappedId('brands', row.brand_id));
    }
    if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
      await linkRecord('Assets', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
    }
    if (row.uploaded_by) {
      const userId = tmToUser[row.uploaded_by];
      if (userId && getMappedId('users', userId)) {
        await linkRecord('Assets', created.Id, 'Uploader', getMappedId('users', userId));
      }
    }
  }
  console.log(`  ✅ ${rows.length} assets migrated`);
}

async function migrateComments() {
  console.log('Migrating Comments...');
  const rows = sqliteDb.prepare('SELECT * FROM comments ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('Comments', {
      entity_type: row.entity_type,
      entity_id: row.entity_id,
      content: row.content,
    });
    mapId('comments', row.id, created.Id);

    if (row.user_id && getMappedId('users', row.user_id)) {
      await linkRecord('Comments', created.Id, 'User', getMappedId('users', row.user_id));
    }
  }
  console.log(`  ✅ ${rows.length} comments migrated`);
}

async function migrateBudgetEntries() {
  console.log('Migrating BudgetEntries...');
  const rows = sqliteDb.prepare('SELECT * FROM budget_entries ORDER BY id').all();
  for (const row of rows) {
    const created = await nocodb.create('BudgetEntries', {
      label: row.label,
      amount: row.amount,
      source: row.source,
      category: row.category,
      date_received: row.date_received,
      notes: row.notes || '',
    });
    mapId('budget_entries', row.id, created.Id);

    if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
      await linkRecord('BudgetEntries', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
    }
  }
  console.log(`  ✅ ${rows.length} budget entries migrated`);
}

// Helper: link a record to another via NocoDB link API
async function linkRecord(table, recordId, linkField, linkedRecordId) {
  try {
    const tableId = await nocodb.resolveTableId(table);
    // Get columns to find the link column ID
    const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
      headers: { 'xc-token': nocodb.token },
    });
    const tableMeta = await res.json();
    const linkCol = tableMeta.columns.find(c => c.title === linkField && c.uidt === 'Links');
    if (!linkCol) {
      console.warn(`    ⚠ Link column "${linkField}" not found in ${table}`);
      return;
    }
    await fetch(`${nocodb.url}/api/v2/tables/${tableId}/links/${linkCol.id}/records/${recordId}`, {
      method: 'POST',
      headers: { 'xc-token': nocodb.token, 'Content-Type': 'application/json' },
      body: JSON.stringify([{ Id: linkedRecordId }]),
    });
  } catch (err) {
    console.warn(`    ⚠ Failed to link ${table}.${linkField}: ${err.message}`);
  }
}

// Cache for link column metadata (avoid re-fetching per record)
const linkColCache = {};
async function getLinkColId(table, linkField) {
  const key = `${table}.${linkField}`;
  if (linkColCache[key]) return linkColCache[key];
  const tableId = await nocodb.resolveTableId(table);
  const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
    headers: { 'xc-token': nocodb.token },
  });
  const tableMeta = await res.json();
  for (const c of tableMeta.columns) {
    if (c.uidt === 'Links') {
      linkColCache[`${table}.${c.title}`] = { colId: c.id, tableId };
    }
  }
  return linkColCache[key] || null;
}

// Optimized linkRecord using cache
const origLinkRecord = linkRecord;
linkRecord = async function(table, recordId, linkField, linkedRecordId) {
  let cached = linkColCache[`${table}.${linkField}`];
  if (!cached) {
    // Populate cache for this table
    const tableId = await nocodb.resolveTableId(table);
    const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
      headers: { 'xc-token': nocodb.token },
    });
    const tableMeta = await res.json();
    for (const c of tableMeta.columns) {
      if (c.uidt === 'Links') {
        linkColCache[`${table}.${c.title}`] = { colId: c.id, tableId };
      }
    }
    cached = linkColCache[`${table}.${linkField}`];
  }
  if (!cached) {
    console.warn(`    ⚠ Link column "${linkField}" not found in ${table}`);
    return;
  }
  try {
    await fetch(`${nocodb.url}/api/v2/tables/${cached.tableId}/links/${cached.colId}/records/${recordId}`, {
      method: 'POST',
      headers: { 'xc-token': nocodb.token, 'Content-Type': 'application/json' },
      body: JSON.stringify([{ Id: linkedRecordId }]),
    });
  } catch (err) {
    console.warn(`    ⚠ Failed to link ${table}.${linkField}: ${err.message}`);
  }
};

async function main() {
  console.log('Starting migration from SQLite → NocoDB...\n');

  await migrateBrands();
  await migrateUsers();
  await migrateCampaigns();
  await migrateCampaignTracks();
  await migrateCampaignAssignments();
  await migrateProjects();
  await migrateTasks();
  await migratePosts();
  await migratePostAttachments();
  await migrateAssets();
  await migrateComments();
  await migrateBudgetEntries();

  console.log('\n✅ Migration complete!');
  console.log('ID mapping summary:');
  for (const [table, map] of Object.entries(idMap)) {
    console.log(`  ${table}: ${Object.keys(map).length} records`);
  }
}

main().catch(err => {
  console.error('Migration failed:', err);
  process.exit(1);
});