feat: add server-side ETL pipeline, revert client to NocoDB reads

ETL Pipeline (server):
- POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB
- nocodbClient.ts: table discovery, paginated delete/insert
- etlSync.ts: orchestrates fetch → aggregate → upsert
- museumMapping.ts moved from client to server
- Auth via ETL_SECRET bearer token

Client:
- dataService.ts reverts to reading NocoDB DailySales table
- Paginated fetch via fetchNocoDBTable (handles >1000 rows)
- Suspicious data check: prefers cache if NocoDB returns <10 rows
- Deleted erpService.ts and client-side museumMapping.ts

First full sync: 391K transactions → 5,760 daily records in 108s.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src/config/museumMapping.ts

@@ -1,38 +0,0 @@
-// Definitive mapping of ERP product descriptions to museum names.
-// Priority order matters — first match wins (handles combo tickets).
-
-const MUSEUM_KEYWORDS: [string, string[]][] = [
-  ['Revelation Exhibition', ['Revelation', 'الوحي']],
-  ['Creation Story Museum', ['Creation Story', 'قصة الخلق']],
-  ['Holy Quraan Museum', ['Holy Quraan', 'القرآن الكريم']],
-  ['Trail To Hira Cave', ['Trail To Hira', 'غار حراء']],
-  ['Makkah Greets Us', ['Makkah Greets']],
-  ['VIP Experience', ['VIP Experience']],
-];
-
-export const MUSEUM_NAMES = MUSEUM_KEYWORDS.map(([name]) => name);
-
-export function getMuseumFromProduct(productDescription: string): string {
-  const desc = productDescription.trim();
-  for (const [museum, keywords] of MUSEUM_KEYWORDS) {
-    for (const kw of keywords) {
-      if (desc.includes(kw)) return museum;
-    }
-  }
-  return 'Other';
-}
-
-export const CHANNEL_LABELS: Record<string, string> = {
-  'B2C': 'HiHala Website/App',
-  'B2B': 'B2B',
-  'POS': 'POS',
-  'Safiyyah POS': 'Safiyyah POS',
-  'Standalone': 'Standalone',
-  'Mobile': 'Mobile',
-  'Viva': 'Viva',
-  'IT': 'IT',
-};
-
-export function getChannelLabel(operatingAreaName: string): string {
-  return CHANNEL_LABELS[operatingAreaName] || operatingAreaName;
-}

src/services/dataService.ts

@@ -1,5 +1,4 @@
-// Data source: Hono ERP API (via server proxy) for museum sales
-// NocoDB: PilgrimStats only
+// Data source: NocoDB (DailySales populated by server-side ETL, PilgrimStats)
 // Offline mode: caches data to localStorage for resilience
 
 import type {
@@ -12,13 +11,12 @@ import type {
   FetchResult,
   GroupedData,
   UmrahData,
+  NocoDBDailySale,
   DataErrorType
 } from '../types';
 import { DataError } from '../types';
 import { fetchWithRetry } from '../utils/fetchHelpers';
-import { fetchFromERP } from './erpService';
 
-// NocoDB config (PilgrimStats only)
 const NOCODB_URL = import.meta.env.VITE_NOCODB_URL || '';
 const NOCODB_TOKEN = import.meta.env.VITE_NOCODB_TOKEN || '';
 const NOCODB_BASE_ID = import.meta.env.VITE_NOCODB_BASE_ID || '';
@@ -28,7 +26,7 @@ let discoveredTables: Record<string, string> | null = null;
 
 async function discoverTableIds(): Promise<Record<string, string>> {
   if (discoveredTables) return discoveredTables;
-  if (!NOCODB_BASE_ID) throw new Error('VITE_NOCODB_BASE_ID not configured');
+  if (!NOCODB_BASE_ID) throw new Error('NocoDB not configured');
 
   const res = await fetchWithRetry(
     `${NOCODB_URL}/api/v2/meta/bases/${NOCODB_BASE_ID}/tables`,
@@ -45,6 +43,27 @@ async function discoverTableIds(): Promise<Record<string, string>> {
   return tables;
 }
 
+async function fetchNocoDBTable<T>(tableId: string, limit: number = 1000): Promise<T[]> {
+  let allRecords: T[] = [];
+  let offset = 0;
+
+  while (true) {
+    const response = await fetchWithRetry(
+      `${NOCODB_URL}/api/v2/tables/${tableId}/records?limit=${limit}&offset=${offset}`,
+      { headers: { 'xc-token': NOCODB_TOKEN } }
+    );
+
+    const json = await response.json();
+    const records: T[] = json.list || [];
+    allRecords = allRecords.concat(records);
+
+    if (records.length < limit) break;
+    offset += limit;
+  }
+
+  return allRecords;
+}
+
 // Cache keys
 const CACHE_KEY = 'hihala_data_cache';
 const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
@@ -89,6 +108,41 @@ export async function fetchPilgrimStats(): Promise<UmrahData> {
   }
 }
 
+// ============================================
+// NocoDB DailySales Fetching
+// ============================================
+
+async function fetchFromNocoDB(): Promise<MuseumRecord[]> {
+  console.log('Fetching from NocoDB DailySales...');
+
+  const tables = await discoverTableIds();
+  if (!tables['DailySales']) throw new Error("NocoDB table 'DailySales' not found — run ETL sync first");
+
+  const rows = await fetchNocoDBTable<NocoDBDailySale>(tables['DailySales']);
+
+  const data: MuseumRecord[] = rows.map(row => {
+    const date = row.Date;
+    const year = date ? date.substring(0, 4) : '';
+    const month = date ? parseInt(date.substring(5, 7)) : 0;
+    const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
+
+    return {
+      date,
+      museum_name: row.MuseumName,
+      channel: row.Channel,
+      visits: row.Visits,
+      tickets: row.Tickets,
+      revenue_gross: row.GrossRevenue,
+      revenue_net: row.NetRevenue,
+      year,
+      quarter,
+    };
+  }).filter(r => r.date && r.museum_name);
+
+  console.log(`Loaded ${data.length} rows from NocoDB DailySales`);
+  return data;
+}
+
 // ============================================
 // Offline Cache Functions
 // ============================================
@@ -167,12 +221,29 @@ function classifyError(err: Error): DataErrorType {
 // ============================================
 
 export async function fetchData(): Promise<FetchResult> {
+  if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
+    const cached = loadFromCache();
+    if (cached) {
+      console.warn('NocoDB not configured, using cached data');
+      return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
+    }
+    throw new DataError('NocoDB not configured', 'config');
+  }
+
   try {
-    const data = await fetchFromERP();
+    const data = await fetchFromNocoDB();
+
+    // Suspicious data check — prefer cache if NocoDB returns too few rows
+    const cached = loadFromCache();
+    if (data.length < 10 && cached && cached.data.length > 10) {
+      console.warn('NocoDB returned suspiciously few rows, using cache');
+      return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
+    }
+
     saveToCache(data);
     return { data, fromCache: false };
   } catch (err) {
-    console.error('ERP fetch failed:', (err as Error).message);
+    console.error('NocoDB fetch failed:', (err as Error).message);
 
     const cached = loadFromCache();
     if (cached) {
@@ -186,7 +257,11 @@ export async function fetchData(): Promise<FetchResult> {
 }
 
 export async function refreshData(): Promise<FetchResult> {
-  const data = await fetchFromERP();
+  if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
+    throw new DataError('NocoDB not configured', 'config');
+  }
+
+  const data = await fetchFromNocoDB();
   saveToCache(data);
   return { data, fromCache: false };
 }

src/services/erpService.ts

@@ -1,101 +0,0 @@
-import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
-import type { ERPSaleRecord, MuseumRecord } from '../types';
-
-function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
-  const now = new Date();
-  const endYear = now.getFullYear();
-  const endMonth = now.getMonth() + 1;
-  const boundaries: Array<[string, string]> = [];
-
-  let y = startYear;
-  let m = startMonth;
-  while (y < endYear || (y === endYear && m <= endMonth)) {
-    const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
-    const nextM = m === 12 ? 1 : m + 1;
-    const nextY = m === 12 ? y + 1 : y;
-    const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
-    boundaries.push([start, end]);
-    y = nextY;
-    m = nextM;
-  }
-
-  return boundaries;
-}
-
-async function fetchChunk(startDate: string, endDate: string): Promise<ERPSaleRecord[]> {
-  const params = new URLSearchParams({ startDate, endDate });
-  const res = await fetch(`/api/erp/sales?${params}`);
-
-  if (!res.ok) {
-    const body = await res.json().catch(() => ({}));
-    throw new Error((body as { error?: string }).error || `ERP proxy returned ${res.status}`);
-  }
-
-  return res.json();
-}
-
-export function aggregateTransactions(sales: ERPSaleRecord[]): MuseumRecord[] {
-  const map = new Map<string, { visits: number; tickets: number; revenue_gross: number; revenue_net: number }>();
-
-  for (const sale of sales) {
-    const date = sale.TransactionDate.split(' ')[0];
-    const channel = getChannelLabel(sale.OperatingAreaName);
-
-    for (const product of sale.Products) {
-      const museum = getMuseumFromProduct(product.ProductDescription);
-      const key = `${date}|${museum}|${channel}`;
-
-      let entry = map.get(key);
-      if (!entry) {
-        entry = { visits: 0, tickets: 0, revenue_gross: 0, revenue_net: 0 };
-        map.set(key, entry);
-      }
-
-      entry.visits += product.PeopleCount;
-      entry.tickets += product.UnitQuantity;
-      entry.revenue_gross += product.TotalPrice;
-      entry.revenue_net += product.TotalPrice - product.TaxAmount;
-    }
-  }
-
-  const records: MuseumRecord[] = [];
-  for (const [key, entry] of map) {
-    const [date, museum_name, channel] = key.split('|');
-    const year = date.substring(0, 4);
-    const month = parseInt(date.substring(5, 7));
-    const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
-
-    records.push({
-      date,
-      museum_name,
-      channel,
-      visits: entry.visits,
-      tickets: entry.tickets,
-      revenue_gross: entry.revenue_gross,
-      revenue_net: entry.revenue_net,
-      year,
-      quarter,
-    });
-  }
-
-  return records;
-}
-
-export async function fetchFromERP(): Promise<MuseumRecord[]> {
-  console.log('Fetching from ERP API via proxy...');
-  const months = generateMonthBoundaries(2024, 1);
-
-  // Fetch months sequentially — the ERP API doesn't handle concurrent requests well
-  const allSales: ERPSaleRecord[] = [];
-
-  for (const [start, end] of months) {
-    const chunk = await fetchChunk(start, end);
-    allSales.push(...chunk);
-  }
-
-  console.log(`Fetched ${allSales.length} transactions, aggregating...`);
-  const records = aggregateTransactions(allSales);
-  console.log(`Aggregated into ${records.length} daily records`);
-
-  return records;
-}

src/types/index.ts

@@ -73,27 +73,16 @@ export interface UmrahData {
   };
 }
 
-// ERP API types
-export interface ERPProduct {
-  ProductDescription: string;
-  SiteDescription: string | null;
-  UnitQuantity: number;
-  PeopleCount: number;
-  TaxAmount: number;
-  TotalPrice: number;
-}
-
-export interface ERPPayment {
-  PaymentMethodDescription: string;
-}
-
-export interface ERPSaleRecord {
-  SaleId: number;
-  TransactionDate: string;
-  CustIdentification: string;
-  OperatingAreaName: string;
-  Payments: ERPPayment[];
-  Products: ERPProduct[];
+// NocoDB DailySales row (populated by server-side ETL)
+export interface NocoDBDailySale {
+  Id: number;
+  Date: string;
+  MuseumName: string;
+  Channel: string;
+  Visits: number;
+  Tickets: number;
+  GrossRevenue: number;
+  NetRevenue: number;
 }
 
 // Chart data types