feat: add server-side ETL pipeline, revert client to NocoDB reads

ETL Pipeline (server):
- POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB
- nocodbClient.ts: table discovery, paginated delete/insert
- etlSync.ts: orchestrates fetch → aggregate → upsert
- museumMapping.ts moved from client to server
- Auth via ETL_SECRET bearer token

Client:
- dataService.ts reverts to reading NocoDB DailySales table
- Paginated fetch via fetchNocoDBTable (handles >1000 rows)
- Suspicious data check: prefers cache if NocoDB returns <10 rows
- Deleted erpService.ts and client-side museumMapping.ts

First full sync: 391K transactions → 5,760 daily records in 108s.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
fahed
2026-03-31 13:25:50 +03:00
parent 9c0ffa5721
commit 1f1e0756d0
12 changed files with 455 additions and 141 deletions
-38
View File
@@ -1,38 +0,0 @@
// Definitive mapping of ERP product descriptions to museum names.
// Priority order matters — first match wins (handles combo tickets).
const MUSEUM_KEYWORDS: [string, string[]][] = [
['Revelation Exhibition', ['Revelation', 'الوحي']],
['Creation Story Museum', ['Creation Story', 'قصة الخلق']],
['Holy Quraan Museum', ['Holy Quraan', 'القرآن الكريم']],
['Trail To Hira Cave', ['Trail To Hira', 'غار حراء']],
['Makkah Greets Us', ['Makkah Greets']],
['VIP Experience', ['VIP Experience']],
];
export const MUSEUM_NAMES = MUSEUM_KEYWORDS.map(([name]) => name);
export function getMuseumFromProduct(productDescription: string): string {
const desc = productDescription.trim();
for (const [museum, keywords] of MUSEUM_KEYWORDS) {
for (const kw of keywords) {
if (desc.includes(kw)) return museum;
}
}
return 'Other';
}
export const CHANNEL_LABELS: Record<string, string> = {
'B2C': 'HiHala Website/App',
'B2B': 'B2B',
'POS': 'POS',
'Safiyyah POS': 'Safiyyah POS',
'Standalone': 'Standalone',
'Mobile': 'Mobile',
'Viva': 'Viva',
'IT': 'IT',
};
export function getChannelLabel(operatingAreaName: string): string {
return CHANNEL_LABELS[operatingAreaName] || operatingAreaName;
}