feat: fire-and-forget ETL sync with progress status endpoint
Deploy HiHala Dashboard / deploy (push) Successful in 8s
Deploy HiHala Dashboard / deploy (push) Successful in 8s
POST /api/etl/sync now returns immediately (202-style). GET /api/etl/status shows running state, current month being processed, and final result or error when done. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+38
-14
@@ -1,34 +1,58 @@
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { etl } from '../config';
|
||||
import { runSync } from '../services/etlSync';
|
||||
import { runSync, SyncResult } from '../services/etlSync';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// POST /api/etl/sync?mode=full|incremental
|
||||
router.post('/sync', async (req: Request, res: Response) => {
|
||||
// Auth check
|
||||
const auth = req.headers.authorization;
|
||||
if (etl.secret && auth !== `Bearer ${etl.secret}`) {
|
||||
type SyncState =
|
||||
| { status: 'idle' }
|
||||
| { status: 'running'; mode: string; startedAt: string; currentMonth?: string }
|
||||
| { status: 'done'; result: SyncResult; finishedAt: string }
|
||||
| { status: 'error'; error: string; finishedAt: string };
|
||||
|
||||
let syncState: SyncState = { status: 'idle' };
|
||||
|
||||
function auth(req: Request, res: Response): boolean {
|
||||
const header = req.headers.authorization;
|
||||
if (etl.secret && header !== `Bearer ${etl.secret}`) {
|
||||
res.status(401).json({ error: 'Unauthorized' });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// POST /api/etl/sync?mode=full|incremental — fires and returns immediately
|
||||
router.post('/sync', (req: Request, res: Response) => {
|
||||
if (!auth(req, res)) return;
|
||||
|
||||
if (syncState.status === 'running') {
|
||||
res.status(409).json({ error: 'Sync already running', state: syncState });
|
||||
return;
|
||||
}
|
||||
|
||||
const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental';
|
||||
syncState = { status: 'running', mode, startedAt: new Date().toISOString() };
|
||||
|
||||
res.json({ accepted: true, mode, message: 'Sync started — poll GET /api/etl/status for progress' });
|
||||
|
||||
try {
|
||||
console.log(`\nETL sync started (${mode})...`);
|
||||
const result = await runSync(mode);
|
||||
runSync(mode, (month) => {
|
||||
if (syncState.status === 'running') syncState = { ...syncState, currentMonth: month };
|
||||
})
|
||||
.then(result => {
|
||||
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
syncState = { status: 'done', result, finishedAt: new Date().toISOString() };
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('ETL sync failed:', (err as Error).message);
|
||||
res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message });
|
||||
}
|
||||
syncState = { status: 'error', error: (err as Error).message, finishedAt: new Date().toISOString() };
|
||||
});
|
||||
});
|
||||
|
||||
// GET /api/etl/status
|
||||
router.get('/status', (_req: Request, res: Response) => {
|
||||
res.json({ configured: !!etl.secret });
|
||||
router.get('/status', (req: Request, res: Response) => {
|
||||
if (!auth(req, res)) return;
|
||||
res.json(syncState);
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -96,7 +96,7 @@ export interface SyncResult {
|
||||
duration: string;
|
||||
}
|
||||
|
||||
export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Promise<SyncResult> {
|
||||
export async function runSync(mode: 'full' | 'incremental' = 'incremental', onMonth?: (month: string) => void): Promise<SyncResult> {
|
||||
const start = Date.now();
|
||||
|
||||
const tables = await discoverTableIds();
|
||||
@@ -113,7 +113,9 @@ export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Pro
|
||||
// Fetch from ERP sequentially (API can't handle concurrent requests)
|
||||
const allSales: ERPSaleRecord[] = [];
|
||||
for (const [startDate, endDate] of months) {
|
||||
console.log(` Fetching ${startDate.slice(0, 7)}...`);
|
||||
const monthLabel = startDate.slice(0, 7);
|
||||
console.log(` Fetching ${monthLabel}...`);
|
||||
onMonth?.(monthLabel);
|
||||
const chunk = await fetchSales(startDate, endDate) as ERPSaleRecord[];
|
||||
allSales.push(...chunk);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user