feat: add server-side ETL pipeline, revert client to NocoDB reads
ETL Pipeline (server): - POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB - nocodbClient.ts: table discovery, paginated delete/insert - etlSync.ts: orchestrates fetch → aggregate → upsert - museumMapping.ts moved from client to server - Auth via ETL_SECRET bearer token Client: - dataService.ts reverts to reading NocoDB DailySales table - Paginated fetch via fetchNocoDBTable (handles >1000 rows) - Suspicious data check: prefers cache if NocoDB returns <10 rows - Deleted erpService.ts and client-side museumMapping.ts First full sync: 391K transactions → 5,760 daily records in 108s. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { etl } from '../config';
|
||||
import { runSync } from '../services/etlSync';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// POST /api/etl/sync?mode=full|incremental
|
||||
router.post('/sync', async (req: Request, res: Response) => {
|
||||
// Auth check
|
||||
const auth = req.headers.authorization;
|
||||
if (etl.secret && auth !== `Bearer ${etl.secret}`) {
|
||||
res.status(401).json({ error: 'Unauthorized' });
|
||||
return;
|
||||
}
|
||||
|
||||
const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental';
|
||||
|
||||
try {
|
||||
console.log(`\nETL sync started (${mode})...`);
|
||||
const result = await runSync(mode);
|
||||
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
console.error('ETL sync failed:', (err as Error).message);
|
||||
res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/etl/status
|
||||
router.get('/status', (_req: Request, res: Response) => {
|
||||
res.json({ configured: !!etl.secret });
|
||||
});
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user