feat: add server-side ETL pipeline, revert client to NocoDB reads

ETL Pipeline (server):
- POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB
- nocodbClient.ts: table discovery, paginated delete/insert
- etlSync.ts: orchestrates fetch → aggregate → upsert
- museumMapping.ts moved from client to server
- Auth via ETL_SECRET bearer token

Client:
- dataService.ts reverts to reading NocoDB DailySales table
- Paginated fetch via fetchNocoDBTable (handles >1000 rows)
- Suspicious data check: prefers cache if NocoDB returns <10 rows
- Deleted erpService.ts and client-side museumMapping.ts

First full sync: 391K transactions → 5,760 daily records in 108s.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
fahed
2026-03-31 13:25:50 +03:00
parent 9c0ffa5721
commit 1f1e0756d0
12 changed files with 455 additions and 141 deletions

View File

@@ -7,11 +7,10 @@ ERP_API_CODE=your-api-function-key
ERP_USERNAME=your-username
ERP_PASSWORD=your-password
# Salla OAuth Credentials (from Salla Partners dashboard)
SALLA_CLIENT_ID=your_client_id_here
SALLA_CLIENT_SECRET=your_client_secret_here
SALLA_REDIRECT_URI=http://localhost:3001/auth/callback
# NocoDB (for ETL writes)
NOCODB_URL=http://localhost:8090
NOCODB_TOKEN=your-token
NOCODB_BASE_ID=your-base-id
# After OAuth, these will be populated automatically
# SALLA_ACCESS_TOKEN=
# SALLA_REFRESH_TOKEN=
# ETL sync secret (for cron auth)
ETL_SECRET=your-secret-here

View File

@@ -23,3 +23,13 @@ export const erp = {
username: process.env.ERP_USERNAME || '',
password: process.env.ERP_PASSWORD || '',
};
export const nocodb = {
url: process.env.NOCODB_URL || '',
token: process.env.NOCODB_TOKEN || '',
baseId: process.env.NOCODB_BASE_ID || '',
};
export const etl = {
secret: process.env.ETL_SECRET || '',
};

View File

@@ -1,7 +1,8 @@
import express from 'express';
import cors from 'cors';
import { server, erp } from './config';
import { server, erp, nocodb } from './config';
import erpRoutes from './routes/erp';
import etlRoutes from './routes/etl';
const app = express();
app.use(cors());
@@ -9,14 +10,21 @@ app.use(express.json());
// Mount routes
app.use('/api/erp', erpRoutes);
app.use('/api/etl', etlRoutes);
app.listen(server.port, () => {
console.log(`\nServer running on http://localhost:${server.port}`);
if (erp.apiUrl && erp.username) {
console.log(' GET /api/erp/sales?startDate=...&endDate=...');
console.log(' GET /api/erp/status');
console.log(' ERP: configured');
} else {
console.log(' WARNING: ERP_API_URL / ERP_USERNAME not set in .env');
console.log(' ERP: WARNING — not configured');
}
if (nocodb.url && nocodb.token) {
console.log(' NocoDB: configured');
console.log(' POST /api/etl/sync?mode=full|incremental');
} else {
console.log(' NocoDB: WARNING — not configured');
}
});

34
server/src/routes/etl.ts Normal file
View File

@@ -0,0 +1,34 @@
import { Router, Request, Response } from 'express';
import { etl } from '../config';
import { runSync } from '../services/etlSync';
const router = Router();
// POST /api/etl/sync?mode=full|incremental
router.post('/sync', async (req: Request, res: Response) => {
// Auth check
const auth = req.headers.authorization;
if (etl.secret && auth !== `Bearer ${etl.secret}`) {
res.status(401).json({ error: 'Unauthorized' });
return;
}
const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental';
try {
console.log(`\nETL sync started (${mode})...`);
const result = await runSync(mode);
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
res.json(result);
} catch (err) {
console.error('ETL sync failed:', (err as Error).message);
res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message });
}
});
// GET /api/etl/status
router.get('/status', (_req: Request, res: Response) => {
res.json({ configured: !!etl.secret });
});
export default router;

View File

@@ -0,0 +1,119 @@
import { fetchSales } from './erpClient';
import { discoverTableIds, deleteRowsByMonth, deleteAllRows, insertRecords } from './nocodbClient';
import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
import type { ERPSaleRecord, AggregatedRecord } from '../types';
function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
const now = new Date();
const endYear = now.getFullYear();
const endMonth = now.getMonth() + 1;
const boundaries: Array<[string, string]> = [];
let y = startYear;
let m = startMonth;
while (y < endYear || (y === endYear && m <= endMonth)) {
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
const nextM = m === 12 ? 1 : m + 1;
const nextY = m === 12 ? y + 1 : y;
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
boundaries.push([start, end]);
y = nextY;
m = nextM;
}
return boundaries;
}
function currentMonthBoundary(): [string, string] {
const now = new Date();
const y = now.getFullYear();
const m = now.getMonth() + 1;
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
const nextM = m === 12 ? 1 : m + 1;
const nextY = m === 12 ? y + 1 : y;
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
return [start, end];
}
export function aggregateTransactions(sales: ERPSaleRecord[]): AggregatedRecord[] {
const map = new Map<string, AggregatedRecord>();
for (const sale of sales) {
const date = sale.TransactionDate.split(' ')[0];
const channel = getChannelLabel(sale.OperatingAreaName);
for (const product of sale.Products) {
const museum = getMuseumFromProduct(product.ProductDescription);
const key = `${date}|${museum}|${channel}`;
let entry = map.get(key);
if (!entry) {
entry = { Date: date, MuseumName: museum, Channel: channel, Visits: 0, Tickets: 0, GrossRevenue: 0, NetRevenue: 0 };
map.set(key, entry);
}
entry.Visits += product.PeopleCount;
entry.Tickets += product.UnitQuantity;
entry.GrossRevenue += product.TotalPrice;
entry.NetRevenue += product.TotalPrice - product.TaxAmount;
}
}
return Array.from(map.values());
}
export interface SyncResult {
status: string;
mode: string;
transactionsFetched: number;
recordsWritten: number;
duration: string;
}
export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Promise<SyncResult> {
const start = Date.now();
const tables = await discoverTableIds();
const tableId = tables['DailySales'];
if (!tableId) throw new Error("NocoDB table 'DailySales' not found");
let months: Array<[string, string]>;
if (mode === 'full') {
months = generateMonthBoundaries(2024, 1);
} else {
months = [currentMonthBoundary()];
}
// Fetch from ERP sequentially (API can't handle concurrent requests)
const allSales: ERPSaleRecord[] = [];
for (const [startDate, endDate] of months) {
console.log(` Fetching ${startDate.slice(0, 7)}...`);
const chunk = await fetchSales(startDate, endDate) as ERPSaleRecord[];
allSales.push(...chunk);
}
const records = aggregateTransactions(allSales);
// Write to NocoDB
if (mode === 'full') {
console.log(' Clearing all DailySales rows...');
await deleteAllRows(tableId);
} else {
const yearMonth = months[0][0].slice(0, 7);
console.log(` Clearing ${yearMonth} rows...`);
await deleteRowsByMonth(tableId, yearMonth);
}
console.log(` Inserting ${records.length} records...`);
const written = await insertRecords(tableId, records);
const duration = ((Date.now() - start) / 1000).toFixed(1) + 's';
return {
status: 'ok',
mode,
transactionsFetched: allSales.length,
recordsWritten: written,
duration,
};
}

View File

@@ -0,0 +1,109 @@
import { nocodb } from '../config';
import type { AggregatedRecord } from '../types';
let discoveredTables: Record<string, string> | null = null;
async function fetchJson(url: string, options: RequestInit = {}): Promise<unknown> {
const res = await fetch(url, {
...options,
headers: {
'xc-token': nocodb.token,
'Content-Type': 'application/json',
...options.headers,
},
});
if (!res.ok) {
const text = await res.text().catch(() => '');
throw new Error(`NocoDB ${res.status}: ${text.slice(0, 200)}`);
}
return res.json();
}
export async function discoverTableIds(): Promise<Record<string, string>> {
if (discoveredTables) return discoveredTables;
if (!nocodb.baseId) throw new Error('NOCODB_BASE_ID not configured');
const json = await fetchJson(
`${nocodb.url}/api/v2/meta/bases/${nocodb.baseId}/tables`
) as { list: Array<{ title: string; id: string }> };
const tables: Record<string, string> = {};
for (const t of json.list) {
tables[t.title] = t.id;
}
discoveredTables = tables;
return tables;
}
export async function deleteRowsByMonth(tableId: string, yearMonth: string): Promise<number> {
// Fetch all row IDs for the given month using a where filter
const where = `(Date,like,${yearMonth}%)`;
let deleted = 0;
let hasMore = true;
while (hasMore) {
const json = await fetchJson(
`${nocodb.url}/api/v2/tables/${tableId}/records?where=${encodeURIComponent(where)}&limit=100&fields=Id`
) as { list: Array<{ Id: number }> };
const ids = json.list.map(r => r.Id);
if (ids.length === 0) {
hasMore = false;
break;
}
// Bulk delete
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
method: 'DELETE',
body: JSON.stringify(ids.map(Id => ({ Id }))),
});
deleted += ids.length;
}
return deleted;
}
export async function deleteAllRows(tableId: string): Promise<number> {
let deleted = 0;
let hasMore = true;
while (hasMore) {
const json = await fetchJson(
`${nocodb.url}/api/v2/tables/${tableId}/records?limit=100&fields=Id`
) as { list: Array<{ Id: number }> };
const ids = json.list.map(r => r.Id);
if (ids.length === 0) {
hasMore = false;
break;
}
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
method: 'DELETE',
body: JSON.stringify(ids.map(Id => ({ Id }))),
});
deleted += ids.length;
}
return deleted;
}
export async function insertRecords(tableId: string, records: AggregatedRecord[]): Promise<number> {
// NocoDB bulk insert accepts max 100 records at a time
const batchSize = 100;
let inserted = 0;
for (let i = 0; i < records.length; i += batchSize) {
const batch = records.slice(i, i + batchSize);
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
method: 'POST',
body: JSON.stringify(batch),
});
inserted += batch.length;
}
return inserted;
}

27
server/src/types.ts Normal file
View File

@@ -0,0 +1,27 @@
export interface ERPProduct {
ProductDescription: string;
SiteDescription: string | null;
UnitQuantity: number;
PeopleCount: number;
TaxAmount: number;
TotalPrice: number;
}
export interface ERPSaleRecord {
SaleId: number;
TransactionDate: string;
CustIdentification: string;
OperatingAreaName: string;
Payments: Array<{ PaymentMethodDescription: string }>;
Products: ERPProduct[];
}
export interface AggregatedRecord {
Date: string;
MuseumName: string;
Channel: string;
Visits: number;
Tickets: number;
GrossRevenue: number;
NetRevenue: number;
}

View File

@@ -1,5 +1,4 @@
// Data source: Hono ERP API (via server proxy) for museum sales
// NocoDB: PilgrimStats only
// Data source: NocoDB (DailySales populated by server-side ETL, PilgrimStats)
// Offline mode: caches data to localStorage for resilience
import type {
@@ -12,13 +11,12 @@ import type {
FetchResult,
GroupedData,
UmrahData,
NocoDBDailySale,
DataErrorType
} from '../types';
import { DataError } from '../types';
import { fetchWithRetry } from '../utils/fetchHelpers';
import { fetchFromERP } from './erpService';
// NocoDB config (PilgrimStats only)
const NOCODB_URL = import.meta.env.VITE_NOCODB_URL || '';
const NOCODB_TOKEN = import.meta.env.VITE_NOCODB_TOKEN || '';
const NOCODB_BASE_ID = import.meta.env.VITE_NOCODB_BASE_ID || '';
@@ -28,7 +26,7 @@ let discoveredTables: Record<string, string> | null = null;
async function discoverTableIds(): Promise<Record<string, string>> {
if (discoveredTables) return discoveredTables;
if (!NOCODB_BASE_ID) throw new Error('VITE_NOCODB_BASE_ID not configured');
if (!NOCODB_BASE_ID) throw new Error('NocoDB not configured');
const res = await fetchWithRetry(
`${NOCODB_URL}/api/v2/meta/bases/${NOCODB_BASE_ID}/tables`,
@@ -45,6 +43,27 @@ async function discoverTableIds(): Promise<Record<string, string>> {
return tables;
}
async function fetchNocoDBTable<T>(tableId: string, limit: number = 1000): Promise<T[]> {
let allRecords: T[] = [];
let offset = 0;
while (true) {
const response = await fetchWithRetry(
`${NOCODB_URL}/api/v2/tables/${tableId}/records?limit=${limit}&offset=${offset}`,
{ headers: { 'xc-token': NOCODB_TOKEN } }
);
const json = await response.json();
const records: T[] = json.list || [];
allRecords = allRecords.concat(records);
if (records.length < limit) break;
offset += limit;
}
return allRecords;
}
// Cache keys
const CACHE_KEY = 'hihala_data_cache';
const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
@@ -89,6 +108,41 @@ export async function fetchPilgrimStats(): Promise<UmrahData> {
}
}
// ============================================
// NocoDB DailySales Fetching
// ============================================
async function fetchFromNocoDB(): Promise<MuseumRecord[]> {
console.log('Fetching from NocoDB DailySales...');
const tables = await discoverTableIds();
if (!tables['DailySales']) throw new Error("NocoDB table 'DailySales' not found — run ETL sync first");
const rows = await fetchNocoDBTable<NocoDBDailySale>(tables['DailySales']);
const data: MuseumRecord[] = rows.map(row => {
const date = row.Date;
const year = date ? date.substring(0, 4) : '';
const month = date ? parseInt(date.substring(5, 7)) : 0;
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
return {
date,
museum_name: row.MuseumName,
channel: row.Channel,
visits: row.Visits,
tickets: row.Tickets,
revenue_gross: row.GrossRevenue,
revenue_net: row.NetRevenue,
year,
quarter,
};
}).filter(r => r.date && r.museum_name);
console.log(`Loaded ${data.length} rows from NocoDB DailySales`);
return data;
}
// ============================================
// Offline Cache Functions
// ============================================
@@ -167,12 +221,29 @@ function classifyError(err: Error): DataErrorType {
// ============================================
export async function fetchData(): Promise<FetchResult> {
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
const cached = loadFromCache();
if (cached) {
console.warn('NocoDB not configured, using cached data');
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
}
throw new DataError('NocoDB not configured', 'config');
}
try {
const data = await fetchFromERP();
const data = await fetchFromNocoDB();
// Suspicious data check — prefer cache if NocoDB returns too few rows
const cached = loadFromCache();
if (data.length < 10 && cached && cached.data.length > 10) {
console.warn('NocoDB returned suspiciously few rows, using cache');
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
}
saveToCache(data);
return { data, fromCache: false };
} catch (err) {
console.error('ERP fetch failed:', (err as Error).message);
console.error('NocoDB fetch failed:', (err as Error).message);
const cached = loadFromCache();
if (cached) {
@@ -186,7 +257,11 @@ export async function fetchData(): Promise<FetchResult> {
}
export async function refreshData(): Promise<FetchResult> {
const data = await fetchFromERP();
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
throw new DataError('NocoDB not configured', 'config');
}
const data = await fetchFromNocoDB();
saveToCache(data);
return { data, fromCache: false };
}

View File

@@ -1,101 +0,0 @@
import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
import type { ERPSaleRecord, MuseumRecord } from '../types';
function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
const now = new Date();
const endYear = now.getFullYear();
const endMonth = now.getMonth() + 1;
const boundaries: Array<[string, string]> = [];
let y = startYear;
let m = startMonth;
while (y < endYear || (y === endYear && m <= endMonth)) {
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
const nextM = m === 12 ? 1 : m + 1;
const nextY = m === 12 ? y + 1 : y;
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
boundaries.push([start, end]);
y = nextY;
m = nextM;
}
return boundaries;
}
async function fetchChunk(startDate: string, endDate: string): Promise<ERPSaleRecord[]> {
const params = new URLSearchParams({ startDate, endDate });
const res = await fetch(`/api/erp/sales?${params}`);
if (!res.ok) {
const body = await res.json().catch(() => ({}));
throw new Error((body as { error?: string }).error || `ERP proxy returned ${res.status}`);
}
return res.json();
}
export function aggregateTransactions(sales: ERPSaleRecord[]): MuseumRecord[] {
const map = new Map<string, { visits: number; tickets: number; revenue_gross: number; revenue_net: number }>();
for (const sale of sales) {
const date = sale.TransactionDate.split(' ')[0];
const channel = getChannelLabel(sale.OperatingAreaName);
for (const product of sale.Products) {
const museum = getMuseumFromProduct(product.ProductDescription);
const key = `${date}|${museum}|${channel}`;
let entry = map.get(key);
if (!entry) {
entry = { visits: 0, tickets: 0, revenue_gross: 0, revenue_net: 0 };
map.set(key, entry);
}
entry.visits += product.PeopleCount;
entry.tickets += product.UnitQuantity;
entry.revenue_gross += product.TotalPrice;
entry.revenue_net += product.TotalPrice - product.TaxAmount;
}
}
const records: MuseumRecord[] = [];
for (const [key, entry] of map) {
const [date, museum_name, channel] = key.split('|');
const year = date.substring(0, 4);
const month = parseInt(date.substring(5, 7));
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
records.push({
date,
museum_name,
channel,
visits: entry.visits,
tickets: entry.tickets,
revenue_gross: entry.revenue_gross,
revenue_net: entry.revenue_net,
year,
quarter,
});
}
return records;
}
export async function fetchFromERP(): Promise<MuseumRecord[]> {
console.log('Fetching from ERP API via proxy...');
const months = generateMonthBoundaries(2024, 1);
// Fetch months sequentially — the ERP API doesn't handle concurrent requests well
const allSales: ERPSaleRecord[] = [];
for (const [start, end] of months) {
const chunk = await fetchChunk(start, end);
allSales.push(...chunk);
}
console.log(`Fetched ${allSales.length} transactions, aggregating...`);
const records = aggregateTransactions(allSales);
console.log(`Aggregated into ${records.length} daily records`);
return records;
}

View File

@@ -73,27 +73,16 @@ export interface UmrahData {
};
}
// ERP API types
export interface ERPProduct {
ProductDescription: string;
SiteDescription: string | null;
UnitQuantity: number;
PeopleCount: number;
TaxAmount: number;
TotalPrice: number;
}
export interface ERPPayment {
PaymentMethodDescription: string;
}
export interface ERPSaleRecord {
SaleId: number;
TransactionDate: string;
CustIdentification: string;
OperatingAreaName: string;
Payments: ERPPayment[];
Products: ERPProduct[];
// NocoDB DailySales row (populated by server-side ETL)
export interface NocoDBDailySale {
Id: number;
Date: string;
MuseumName: string;
Channel: string;
Visits: number;
Tickets: number;
GrossRevenue: number;
NetRevenue: number;
}
// Chart data types

45
start-dev.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Temporary dev script for ERP migration — starts NocoDB + Express server + Vite
set -e
cleanup() {
echo ""
echo "Shutting down..."
kill $SERVER_PID $CLIENT_PID 2>/dev/null
docker stop nocodb 2>/dev/null
echo "Done."
}
trap cleanup EXIT INT TERM
cd "$(dirname "$0")"
# Start NocoDB
if docker ps --format '{{.Names}}' | grep -q '^nocodb$'; then
echo "NocoDB already running on port 8090"
else
echo "Starting NocoDB..."
docker start nocodb 2>/dev/null || docker run -d \
--name nocodb -p 8090:8080 nocodb/nocodb:latest
fi
echo "Waiting for NocoDB..."
for i in $(seq 1 30); do
curl -s http://localhost:8090/api/v1/health >/dev/null 2>&1 && echo "NocoDB ready" && break
sleep 1
done
# Start Express server (port 3002)
echo "Starting Express server..."
(cd server && npm run dev) &
SERVER_PID=$!
sleep 2
# Start Vite (port 3000)
echo "Starting Vite..."
npx vite &
CLIENT_PID=$!
wait $CLIENT_PID