feat: add server-side ETL pipeline, revert client to NocoDB reads
ETL Pipeline (server): - POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB - nocodbClient.ts: table discovery, paginated delete/insert - etlSync.ts: orchestrates fetch → aggregate → upsert - museumMapping.ts moved from client to server - Auth via ETL_SECRET bearer token Client: - dataService.ts reverts to reading NocoDB DailySales table - Paginated fetch via fetchNocoDBTable (handles >1000 rows) - Suspicious data check: prefers cache if NocoDB returns <10 rows - Deleted erpService.ts and client-side museumMapping.ts First full sync: 391K transactions → 5,760 daily records in 108s. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
fahed
@@ -1,5 +1,4 @@
|
||||
// Data source: Hono ERP API (via server proxy) for museum sales
|
||||
// NocoDB: PilgrimStats only
|
||||
// Data source: NocoDB (DailySales populated by server-side ETL, PilgrimStats)
|
||||
// Offline mode: caches data to localStorage for resilience
|
||||
|
||||
import type {
|
||||
@@ -12,13 +11,12 @@ import type {
|
||||
FetchResult,
|
||||
GroupedData,
|
||||
UmrahData,
|
||||
NocoDBDailySale,
|
||||
DataErrorType
|
||||
} from '../types';
|
||||
import { DataError } from '../types';
|
||||
import { fetchWithRetry } from '../utils/fetchHelpers';
|
||||
import { fetchFromERP } from './erpService';
|
||||
|
||||
// NocoDB config (PilgrimStats only)
|
||||
const NOCODB_URL = import.meta.env.VITE_NOCODB_URL || '';
|
||||
const NOCODB_TOKEN = import.meta.env.VITE_NOCODB_TOKEN || '';
|
||||
const NOCODB_BASE_ID = import.meta.env.VITE_NOCODB_BASE_ID || '';
|
||||
@@ -28,7 +26,7 @@ let discoveredTables: Record<string, string> | null = null;
|
||||
|
||||
async function discoverTableIds(): Promise<Record<string, string>> {
|
||||
if (discoveredTables) return discoveredTables;
|
||||
if (!NOCODB_BASE_ID) throw new Error('VITE_NOCODB_BASE_ID not configured');
|
||||
if (!NOCODB_BASE_ID) throw new Error('NocoDB not configured');
|
||||
|
||||
const res = await fetchWithRetry(
|
||||
`${NOCODB_URL}/api/v2/meta/bases/${NOCODB_BASE_ID}/tables`,
|
||||
@@ -45,6 +43,27 @@ async function discoverTableIds(): Promise<Record<string, string>> {
|
||||
return tables;
|
||||
}
|
||||
|
||||
async function fetchNocoDBTable<T>(tableId: string, limit: number = 1000): Promise<T[]> {
|
||||
let allRecords: T[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (true) {
|
||||
const response = await fetchWithRetry(
|
||||
`${NOCODB_URL}/api/v2/tables/${tableId}/records?limit=${limit}&offset=${offset}`,
|
||||
{ headers: { 'xc-token': NOCODB_TOKEN } }
|
||||
);
|
||||
|
||||
const json = await response.json();
|
||||
const records: T[] = json.list || [];
|
||||
allRecords = allRecords.concat(records);
|
||||
|
||||
if (records.length < limit) break;
|
||||
offset += limit;
|
||||
}
|
||||
|
||||
return allRecords;
|
||||
}
|
||||
|
||||
// Cache keys
|
||||
const CACHE_KEY = 'hihala_data_cache';
|
||||
const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
|
||||
@@ -89,6 +108,41 @@ export async function fetchPilgrimStats(): Promise<UmrahData> {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// NocoDB DailySales Fetching
|
||||
// ============================================
|
||||
|
||||
async function fetchFromNocoDB(): Promise<MuseumRecord[]> {
|
||||
console.log('Fetching from NocoDB DailySales...');
|
||||
|
||||
const tables = await discoverTableIds();
|
||||
if (!tables['DailySales']) throw new Error("NocoDB table 'DailySales' not found — run ETL sync first");
|
||||
|
||||
const rows = await fetchNocoDBTable<NocoDBDailySale>(tables['DailySales']);
|
||||
|
||||
const data: MuseumRecord[] = rows.map(row => {
|
||||
const date = row.Date;
|
||||
const year = date ? date.substring(0, 4) : '';
|
||||
const month = date ? parseInt(date.substring(5, 7)) : 0;
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
return {
|
||||
date,
|
||||
museum_name: row.MuseumName,
|
||||
channel: row.Channel,
|
||||
visits: row.Visits,
|
||||
tickets: row.Tickets,
|
||||
revenue_gross: row.GrossRevenue,
|
||||
revenue_net: row.NetRevenue,
|
||||
year,
|
||||
quarter,
|
||||
};
|
||||
}).filter(r => r.date && r.museum_name);
|
||||
|
||||
console.log(`Loaded ${data.length} rows from NocoDB DailySales`);
|
||||
return data;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Offline Cache Functions
|
||||
// ============================================
|
||||
@@ -167,12 +221,29 @@ function classifyError(err: Error): DataErrorType {
|
||||
// ============================================
|
||||
|
||||
export async function fetchData(): Promise<FetchResult> {
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
console.warn('NocoDB not configured, using cached data');
|
||||
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
|
||||
}
|
||||
throw new DataError('NocoDB not configured', 'config');
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fetchFromERP();
|
||||
const data = await fetchFromNocoDB();
|
||||
|
||||
// Suspicious data check — prefer cache if NocoDB returns too few rows
|
||||
const cached = loadFromCache();
|
||||
if (data.length < 10 && cached && cached.data.length > 10) {
|
||||
console.warn('NocoDB returned suspiciously few rows, using cache');
|
||||
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
|
||||
}
|
||||
|
||||
saveToCache(data);
|
||||
return { data, fromCache: false };
|
||||
} catch (err) {
|
||||
console.error('ERP fetch failed:', (err as Error).message);
|
||||
console.error('NocoDB fetch failed:', (err as Error).message);
|
||||
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
@@ -186,7 +257,11 @@ export async function fetchData(): Promise<FetchResult> {
|
||||
}
|
||||
|
||||
export async function refreshData(): Promise<FetchResult> {
|
||||
const data = await fetchFromERP();
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
|
||||
throw new DataError('NocoDB not configured', 'config');
|
||||
}
|
||||
|
||||
const data = await fetchFromNocoDB();
|
||||
saveToCache(data);
|
||||
return { data, fromCache: false };
|
||||
}
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
|
||||
import type { ERPSaleRecord, MuseumRecord } from '../types';
|
||||
|
||||
function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
|
||||
const now = new Date();
|
||||
const endYear = now.getFullYear();
|
||||
const endMonth = now.getMonth() + 1;
|
||||
const boundaries: Array<[string, string]> = [];
|
||||
|
||||
let y = startYear;
|
||||
let m = startMonth;
|
||||
while (y < endYear || (y === endYear && m <= endMonth)) {
|
||||
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
|
||||
const nextM = m === 12 ? 1 : m + 1;
|
||||
const nextY = m === 12 ? y + 1 : y;
|
||||
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
|
||||
boundaries.push([start, end]);
|
||||
y = nextY;
|
||||
m = nextM;
|
||||
}
|
||||
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
async function fetchChunk(startDate: string, endDate: string): Promise<ERPSaleRecord[]> {
|
||||
const params = new URLSearchParams({ startDate, endDate });
|
||||
const res = await fetch(`/api/erp/sales?${params}`);
|
||||
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error((body as { error?: string }).error || `ERP proxy returned ${res.status}`);
|
||||
}
|
||||
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export function aggregateTransactions(sales: ERPSaleRecord[]): MuseumRecord[] {
|
||||
const map = new Map<string, { visits: number; tickets: number; revenue_gross: number; revenue_net: number }>();
|
||||
|
||||
for (const sale of sales) {
|
||||
const date = sale.TransactionDate.split(' ')[0];
|
||||
const channel = getChannelLabel(sale.OperatingAreaName);
|
||||
|
||||
for (const product of sale.Products) {
|
||||
const museum = getMuseumFromProduct(product.ProductDescription);
|
||||
const key = `${date}|${museum}|${channel}`;
|
||||
|
||||
let entry = map.get(key);
|
||||
if (!entry) {
|
||||
entry = { visits: 0, tickets: 0, revenue_gross: 0, revenue_net: 0 };
|
||||
map.set(key, entry);
|
||||
}
|
||||
|
||||
entry.visits += product.PeopleCount;
|
||||
entry.tickets += product.UnitQuantity;
|
||||
entry.revenue_gross += product.TotalPrice;
|
||||
entry.revenue_net += product.TotalPrice - product.TaxAmount;
|
||||
}
|
||||
}
|
||||
|
||||
const records: MuseumRecord[] = [];
|
||||
for (const [key, entry] of map) {
|
||||
const [date, museum_name, channel] = key.split('|');
|
||||
const year = date.substring(0, 4);
|
||||
const month = parseInt(date.substring(5, 7));
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
records.push({
|
||||
date,
|
||||
museum_name,
|
||||
channel,
|
||||
visits: entry.visits,
|
||||
tickets: entry.tickets,
|
||||
revenue_gross: entry.revenue_gross,
|
||||
revenue_net: entry.revenue_net,
|
||||
year,
|
||||
quarter,
|
||||
});
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
export async function fetchFromERP(): Promise<MuseumRecord[]> {
|
||||
console.log('Fetching from ERP API via proxy...');
|
||||
const months = generateMonthBoundaries(2024, 1);
|
||||
|
||||
// Fetch months sequentially — the ERP API doesn't handle concurrent requests well
|
||||
const allSales: ERPSaleRecord[] = [];
|
||||
|
||||
for (const [start, end] of months) {
|
||||
const chunk = await fetchChunk(start, end);
|
||||
allSales.push(...chunk);
|
||||
}
|
||||
|
||||
console.log(`Fetched ${allSales.length} transactions, aggregating...`);
|
||||
const records = aggregateTransactions(allSales);
|
||||
console.log(`Aggregated into ${records.length} daily records`);
|
||||
|
||||
return records;
|
||||
}
|
||||
Reference in New Issue
Block a user