feat: fire-and-forget ETL sync with progress status endpoint
Deploy HiHala Dashboard / deploy (push) Successful in 8s

POST /api/etl/sync now returns immediately (202-style).
GET /api/etl/status shows running state, current month being
processed, and final result or error when done.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
fahed
2026-04-19 13:40:19 +03:00
parent ac32a541a1
commit 9064df82be
2 changed files with 45 additions and 19 deletions
+41 -17
View File
@@ -1,34 +1,58 @@
import { Router, Request, Response } from 'express';
import { etl } from '../config';
import { runSync } from '../services/etlSync';
import { runSync, SyncResult } from '../services/etlSync';
const router = Router();
// POST /api/etl/sync?mode=full|incremental
router.post('/sync', async (req: Request, res: Response) => {
// Auth check
const auth = req.headers.authorization;
if (etl.secret && auth !== `Bearer ${etl.secret}`) {
type SyncState =
| { status: 'idle' }
| { status: 'running'; mode: string; startedAt: string; currentMonth?: string }
| { status: 'done'; result: SyncResult; finishedAt: string }
| { status: 'error'; error: string; finishedAt: string };
let syncState: SyncState = { status: 'idle' };
function auth(req: Request, res: Response): boolean {
const header = req.headers.authorization;
if (etl.secret && header !== `Bearer ${etl.secret}`) {
res.status(401).json({ error: 'Unauthorized' });
return false;
}
return true;
}
// POST /api/etl/sync?mode=full|incremental — fires and returns immediately
router.post('/sync', (req: Request, res: Response) => {
if (!auth(req, res)) return;
if (syncState.status === 'running') {
res.status(409).json({ error: 'Sync already running', state: syncState });
return;
}
const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental';
syncState = { status: 'running', mode, startedAt: new Date().toISOString() };
try {
console.log(`\nETL sync started (${mode})...`);
const result = await runSync(mode);
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
res.json(result);
} catch (err) {
console.error('ETL sync failed:', (err as Error).message);
res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message });
}
res.json({ accepted: true, mode, message: 'Sync started — poll GET /api/etl/status for progress' });
console.log(`\nETL sync started (${mode})...`);
runSync(mode, (month) => {
if (syncState.status === 'running') syncState = { ...syncState, currentMonth: month };
})
.then(result => {
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
syncState = { status: 'done', result, finishedAt: new Date().toISOString() };
})
.catch(err => {
console.error('ETL sync failed:', (err as Error).message);
syncState = { status: 'error', error: (err as Error).message, finishedAt: new Date().toISOString() };
});
});
// GET /api/etl/status
router.get('/status', (_req: Request, res: Response) => {
res.json({ configured: !!etl.secret });
router.get('/status', (req: Request, res: Response) => {
if (!auth(req, res)) return;
res.json(syncState);
});
export default router;
+4 -2
View File
@@ -96,7 +96,7 @@ export interface SyncResult {
duration: string;
}
export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Promise<SyncResult> {
export async function runSync(mode: 'full' | 'incremental' = 'incremental', onMonth?: (month: string) => void): Promise<SyncResult> {
const start = Date.now();
const tables = await discoverTableIds();
@@ -113,7 +113,9 @@ export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Pro
// Fetch from ERP sequentially (API can't handle concurrent requests)
const allSales: ERPSaleRecord[] = [];
for (const [startDate, endDate] of months) {
console.log(` Fetching ${startDate.slice(0, 7)}...`);
const monthLabel = startDate.slice(0, 7);
console.log(` Fetching ${monthLabel}...`);
onMonth?.(monthLabel);
const chunk = await fetchSales(startDate, endDate) as ERPSaleRecord[];
allSales.push(...chunk);
}