update on timeline on portfolio view + some corrections
This commit is contained in:
fahed
458
server/migrate-data.js
Normal file
458
server/migrate-data.js
Normal file
@@ -0,0 +1,458 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* migrate-data.js — One-time migration from SQLite to NocoDB.
|
||||
* Run: node migrate-data.js
|
||||
*/
|
||||
|
||||
require('dotenv').config({ path: __dirname + '/.env' });
|
||||
const Database = require('better-sqlite3');
|
||||
const bcrypt = require('bcrypt');
|
||||
const path = require('path');
|
||||
const nocodb = require('./nocodb');
|
||||
const { authDb } = require('./auth-db');
|
||||
|
||||
const sqliteDb = new Database(path.join(__dirname, 'marketing.db'), { readonly: true });
|
||||
|
||||
// ID mapping: { tableName: { oldId: newNocoDbId } }
|
||||
const idMap = {};
|
||||
function mapId(table, oldId, newId) {
|
||||
if (!idMap[table]) idMap[table] = {};
|
||||
idMap[table][oldId] = newId;
|
||||
}
|
||||
function getMappedId(table, oldId) {
|
||||
if (!oldId) return null;
|
||||
return idMap[table]?.[oldId] || null;
|
||||
}
|
||||
|
||||
// Build a lookup: team_member_id → user row
|
||||
function buildTeamMemberToUserMap() {
|
||||
const users = sqliteDb.prepare('SELECT id, team_member_id FROM users').all();
|
||||
const map = {};
|
||||
for (const u of users) {
|
||||
if (u.team_member_id) map[u.team_member_id] = u.id;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function migrateBrands() {
|
||||
console.log('Migrating Brands...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM brands ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Brands', {
|
||||
name: row.name,
|
||||
priority: row.priority,
|
||||
color: row.color,
|
||||
icon: row.icon,
|
||||
});
|
||||
mapId('brands', row.id, created.Id);
|
||||
}
|
||||
console.log(` ✅ ${rows.length} brands migrated`);
|
||||
}
|
||||
|
||||
async function migrateUsers() {
|
||||
console.log('Migrating Users...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM users ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Users', {
|
||||
name: row.name,
|
||||
email: row.email,
|
||||
role: row.role,
|
||||
team_role: row.team_role || null,
|
||||
brands: row.brands || '[]',
|
||||
phone: row.phone || null,
|
||||
avatar: row.avatar || null,
|
||||
tutorial_completed: !!row.tutorial_completed,
|
||||
});
|
||||
mapId('users', row.id, created.Id);
|
||||
|
||||
// Create auth credentials entry
|
||||
authDb.prepare(
|
||||
'INSERT OR REPLACE INTO auth_credentials (email, password_hash, nocodb_user_id) VALUES (?, ?, ?)'
|
||||
).run(row.email, row.password_hash, created.Id);
|
||||
}
|
||||
console.log(` ✅ ${rows.length} users migrated (+ auth_credentials)`);
|
||||
}
|
||||
|
||||
async function migrateCampaigns() {
|
||||
console.log('Migrating Campaigns...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM campaigns ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const data = {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
start_date: row.start_date,
|
||||
end_date: row.end_date,
|
||||
status: row.status,
|
||||
color: row.color,
|
||||
budget: row.budget,
|
||||
goals: row.goals,
|
||||
platforms: row.platforms || '[]',
|
||||
budget_spent: row.budget_spent || 0,
|
||||
revenue: row.revenue || 0,
|
||||
impressions: row.impressions || 0,
|
||||
clicks: row.clicks || 0,
|
||||
conversions: row.conversions || 0,
|
||||
cost_per_click: row.cost_per_click || 0,
|
||||
notes: row.notes || '',
|
||||
};
|
||||
|
||||
const created = await nocodb.create('Campaigns', data);
|
||||
mapId('campaigns', row.id, created.Id);
|
||||
|
||||
// Link Brand
|
||||
if (row.brand_id && getMappedId('brands', row.brand_id)) {
|
||||
await linkRecord('Campaigns', created.Id, 'Brand', getMappedId('brands', row.brand_id));
|
||||
}
|
||||
// Link CreatedByUser
|
||||
if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
|
||||
await linkRecord('Campaigns', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} campaigns migrated`);
|
||||
}
|
||||
|
||||
async function migrateCampaignTracks() {
|
||||
console.log('Migrating CampaignTracks...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM campaign_tracks ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('CampaignTracks', {
|
||||
name: row.name,
|
||||
type: row.type,
|
||||
platform: row.platform,
|
||||
budget_allocated: row.budget_allocated || 0,
|
||||
budget_spent: row.budget_spent || 0,
|
||||
revenue: row.revenue || 0,
|
||||
impressions: row.impressions || 0,
|
||||
clicks: row.clicks || 0,
|
||||
conversions: row.conversions || 0,
|
||||
notes: row.notes || '',
|
||||
status: row.status,
|
||||
});
|
||||
mapId('campaign_tracks', row.id, created.Id);
|
||||
|
||||
if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
|
||||
await linkRecord('CampaignTracks', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} campaign tracks migrated`);
|
||||
}
|
||||
|
||||
async function migrateCampaignAssignments() {
|
||||
console.log('Migrating CampaignAssignments...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM campaign_assignments ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('CampaignAssignments', {
|
||||
assigned_at: row.assigned_at,
|
||||
});
|
||||
mapId('campaign_assignments', row.id, created.Id);
|
||||
|
||||
if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
|
||||
await linkRecord('CampaignAssignments', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
|
||||
}
|
||||
if (row.user_id && getMappedId('users', row.user_id)) {
|
||||
await linkRecord('CampaignAssignments', created.Id, 'Member', getMappedId('users', row.user_id));
|
||||
}
|
||||
if (row.assigned_by && getMappedId('users', row.assigned_by)) {
|
||||
await linkRecord('CampaignAssignments', created.Id, 'Assigner', getMappedId('users', row.assigned_by));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} campaign assignments migrated`);
|
||||
}
|
||||
|
||||
async function migrateProjects() {
|
||||
console.log('Migrating Projects...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM projects ORDER BY id').all();
|
||||
const tmToUser = buildTeamMemberToUserMap();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Projects', {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
status: row.status,
|
||||
priority: row.priority,
|
||||
due_date: row.due_date,
|
||||
});
|
||||
mapId('projects', row.id, created.Id);
|
||||
|
||||
if (row.brand_id && getMappedId('brands', row.brand_id)) {
|
||||
await linkRecord('Projects', created.Id, 'Brand', getMappedId('brands', row.brand_id));
|
||||
}
|
||||
// owner_id references team_members, map through to users
|
||||
if (row.owner_id) {
|
||||
const userId = tmToUser[row.owner_id];
|
||||
if (userId && getMappedId('users', userId)) {
|
||||
await linkRecord('Projects', created.Id, 'Owner', getMappedId('users', userId));
|
||||
}
|
||||
}
|
||||
if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
|
||||
await linkRecord('Projects', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} projects migrated`);
|
||||
}
|
||||
|
||||
async function migrateTasks() {
|
||||
console.log('Migrating Tasks...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM tasks ORDER BY id').all();
|
||||
const tmToUser = buildTeamMemberToUserMap();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Tasks', {
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
status: row.status,
|
||||
priority: row.priority,
|
||||
due_date: row.due_date,
|
||||
is_personal: !!row.is_personal,
|
||||
completed_at: row.completed_at,
|
||||
});
|
||||
mapId('tasks', row.id, created.Id);
|
||||
|
||||
if (row.project_id && getMappedId('projects', row.project_id)) {
|
||||
await linkRecord('Tasks', created.Id, 'Project', getMappedId('projects', row.project_id));
|
||||
}
|
||||
// assigned_to references team_members
|
||||
if (row.assigned_to) {
|
||||
const userId = tmToUser[row.assigned_to];
|
||||
if (userId && getMappedId('users', userId)) {
|
||||
await linkRecord('Tasks', created.Id, 'AssignedTo', getMappedId('users', userId));
|
||||
}
|
||||
}
|
||||
if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
|
||||
await linkRecord('Tasks', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} tasks migrated`);
|
||||
}
|
||||
|
||||
async function migratePosts() {
|
||||
console.log('Migrating Posts...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM posts ORDER BY id').all();
|
||||
const tmToUser = buildTeamMemberToUserMap();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Posts', {
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
status: row.status,
|
||||
platform: row.platform,
|
||||
platforms: row.platforms || '[]',
|
||||
content_type: row.content_type,
|
||||
scheduled_date: row.scheduled_date,
|
||||
published_date: row.published_date,
|
||||
notes: row.notes,
|
||||
publication_links: row.publication_links || '[]',
|
||||
});
|
||||
mapId('posts', row.id, created.Id);
|
||||
|
||||
if (row.brand_id && getMappedId('brands', row.brand_id)) {
|
||||
await linkRecord('Posts', created.Id, 'Brand', getMappedId('brands', row.brand_id));
|
||||
}
|
||||
if (row.assigned_to) {
|
||||
const userId = tmToUser[row.assigned_to];
|
||||
if (userId && getMappedId('users', userId)) {
|
||||
await linkRecord('Posts', created.Id, 'AssignedTo', getMappedId('users', userId));
|
||||
}
|
||||
}
|
||||
if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
|
||||
await linkRecord('Posts', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
|
||||
}
|
||||
if (row.track_id && getMappedId('campaign_tracks', row.track_id)) {
|
||||
await linkRecord('Posts', created.Id, 'Track', getMappedId('campaign_tracks', row.track_id));
|
||||
}
|
||||
if (row.created_by_user_id && getMappedId('users', row.created_by_user_id)) {
|
||||
await linkRecord('Posts', created.Id, 'CreatedByUser', getMappedId('users', row.created_by_user_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} posts migrated`);
|
||||
}
|
||||
|
||||
async function migratePostAttachments() {
|
||||
console.log('Migrating PostAttachments...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM post_attachments ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('PostAttachments', {
|
||||
filename: row.filename,
|
||||
original_name: row.original_name,
|
||||
mime_type: row.mime_type,
|
||||
size: row.size,
|
||||
url: row.url,
|
||||
});
|
||||
mapId('post_attachments', row.id, created.Id);
|
||||
|
||||
if (row.post_id && getMappedId('posts', row.post_id)) {
|
||||
await linkRecord('PostAttachments', created.Id, 'Post', getMappedId('posts', row.post_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} post attachments migrated`);
|
||||
}
|
||||
|
||||
async function migrateAssets() {
|
||||
console.log('Migrating Assets...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM assets ORDER BY id').all();
|
||||
const tmToUser = buildTeamMemberToUserMap();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Assets', {
|
||||
filename: row.filename,
|
||||
original_name: row.original_name,
|
||||
mime_type: row.mime_type,
|
||||
size: row.size,
|
||||
tags: row.tags || '[]',
|
||||
folder: row.folder,
|
||||
});
|
||||
mapId('assets', row.id, created.Id);
|
||||
|
||||
if (row.brand_id && getMappedId('brands', row.brand_id)) {
|
||||
await linkRecord('Assets', created.Id, 'Brand', getMappedId('brands', row.brand_id));
|
||||
}
|
||||
if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
|
||||
await linkRecord('Assets', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
|
||||
}
|
||||
if (row.uploaded_by) {
|
||||
const userId = tmToUser[row.uploaded_by];
|
||||
if (userId && getMappedId('users', userId)) {
|
||||
await linkRecord('Assets', created.Id, 'Uploader', getMappedId('users', userId));
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} assets migrated`);
|
||||
}
|
||||
|
||||
async function migrateComments() {
|
||||
console.log('Migrating Comments...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM comments ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('Comments', {
|
||||
entity_type: row.entity_type,
|
||||
entity_id: row.entity_id,
|
||||
content: row.content,
|
||||
});
|
||||
mapId('comments', row.id, created.Id);
|
||||
|
||||
if (row.user_id && getMappedId('users', row.user_id)) {
|
||||
await linkRecord('Comments', created.Id, 'User', getMappedId('users', row.user_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} comments migrated`);
|
||||
}
|
||||
|
||||
async function migrateBudgetEntries() {
|
||||
console.log('Migrating BudgetEntries...');
|
||||
const rows = sqliteDb.prepare('SELECT * FROM budget_entries ORDER BY id').all();
|
||||
for (const row of rows) {
|
||||
const created = await nocodb.create('BudgetEntries', {
|
||||
label: row.label,
|
||||
amount: row.amount,
|
||||
source: row.source,
|
||||
category: row.category,
|
||||
date_received: row.date_received,
|
||||
notes: row.notes || '',
|
||||
});
|
||||
mapId('budget_entries', row.id, created.Id);
|
||||
|
||||
if (row.campaign_id && getMappedId('campaigns', row.campaign_id)) {
|
||||
await linkRecord('BudgetEntries', created.Id, 'Campaign', getMappedId('campaigns', row.campaign_id));
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${rows.length} budget entries migrated`);
|
||||
}
|
||||
|
||||
// Helper: link a record to another via NocoDB link API
|
||||
async function linkRecord(table, recordId, linkField, linkedRecordId) {
|
||||
try {
|
||||
const tableId = await nocodb.resolveTableId(table);
|
||||
// Get columns to find the link column ID
|
||||
const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
|
||||
headers: { 'xc-token': nocodb.token },
|
||||
});
|
||||
const tableMeta = await res.json();
|
||||
const linkCol = tableMeta.columns.find(c => c.title === linkField && c.uidt === 'Links');
|
||||
if (!linkCol) {
|
||||
console.warn(` ⚠ Link column "${linkField}" not found in ${table}`);
|
||||
return;
|
||||
}
|
||||
await fetch(`${nocodb.url}/api/v2/tables/${tableId}/links/${linkCol.id}/records/${recordId}`, {
|
||||
method: 'POST',
|
||||
headers: { 'xc-token': nocodb.token, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify([{ Id: linkedRecordId }]),
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn(` ⚠ Failed to link ${table}.${linkField}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Cache for link column metadata (avoid re-fetching per record)
|
||||
const linkColCache = {};
|
||||
async function getLinkColId(table, linkField) {
|
||||
const key = `${table}.${linkField}`;
|
||||
if (linkColCache[key]) return linkColCache[key];
|
||||
const tableId = await nocodb.resolveTableId(table);
|
||||
const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
|
||||
headers: { 'xc-token': nocodb.token },
|
||||
});
|
||||
const tableMeta = await res.json();
|
||||
for (const c of tableMeta.columns) {
|
||||
if (c.uidt === 'Links') {
|
||||
linkColCache[`${table}.${c.title}`] = { colId: c.id, tableId };
|
||||
}
|
||||
}
|
||||
return linkColCache[key] || null;
|
||||
}
|
||||
|
||||
// Optimized linkRecord using cache
|
||||
const origLinkRecord = linkRecord;
|
||||
linkRecord = async function(table, recordId, linkField, linkedRecordId) {
|
||||
let cached = linkColCache[`${table}.${linkField}`];
|
||||
if (!cached) {
|
||||
// Populate cache for this table
|
||||
const tableId = await nocodb.resolveTableId(table);
|
||||
const res = await fetch(`${nocodb.url}/api/v2/meta/tables/${tableId}`, {
|
||||
headers: { 'xc-token': nocodb.token },
|
||||
});
|
||||
const tableMeta = await res.json();
|
||||
for (const c of tableMeta.columns) {
|
||||
if (c.uidt === 'Links') {
|
||||
linkColCache[`${table}.${c.title}`] = { colId: c.id, tableId };
|
||||
}
|
||||
}
|
||||
cached = linkColCache[`${table}.${linkField}`];
|
||||
}
|
||||
if (!cached) {
|
||||
console.warn(` ⚠ Link column "${linkField}" not found in ${table}`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await fetch(`${nocodb.url}/api/v2/tables/${cached.tableId}/links/${cached.colId}/records/${recordId}`, {
|
||||
method: 'POST',
|
||||
headers: { 'xc-token': nocodb.token, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify([{ Id: linkedRecordId }]),
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn(` ⚠ Failed to link ${table}.${linkField}: ${err.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
async function main() {
|
||||
console.log('Starting migration from SQLite → NocoDB...\n');
|
||||
|
||||
await migrateBrands();
|
||||
await migrateUsers();
|
||||
await migrateCampaigns();
|
||||
await migrateCampaignTracks();
|
||||
await migrateCampaignAssignments();
|
||||
await migrateProjects();
|
||||
await migrateTasks();
|
||||
await migratePosts();
|
||||
await migratePostAttachments();
|
||||
await migrateAssets();
|
||||
await migrateComments();
|
||||
await migrateBudgetEntries();
|
||||
|
||||
console.log('\n✅ Migration complete!');
|
||||
console.log('ID mapping summary:');
|
||||
for (const [table, map] of Object.entries(idMap)) {
|
||||
console.log(` ${table}: ${Object.keys(map).length} records`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Migration failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user