feat: add server-side ETL pipeline, revert client to NocoDB reads
ETL Pipeline (server): - POST /api/etl/sync?mode=full|incremental — fetches ERP, aggregates, writes NocoDB - nocodbClient.ts: table discovery, paginated delete/insert - etlSync.ts: orchestrates fetch → aggregate → upsert - museumMapping.ts moved from client to server - Auth via ETL_SECRET bearer token Client: - dataService.ts reverts to reading NocoDB DailySales table - Paginated fetch via fetchNocoDBTable (handles >1000 rows) - Suspicious data check: prefers cache if NocoDB returns <10 rows - Deleted erpService.ts and client-side museumMapping.ts First full sync: 391K transactions → 5,760 daily records in 108s. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,11 +7,10 @@ ERP_API_CODE=your-api-function-key
|
||||
ERP_USERNAME=your-username
|
||||
ERP_PASSWORD=your-password
|
||||
|
||||
# Salla OAuth Credentials (from Salla Partners dashboard)
|
||||
SALLA_CLIENT_ID=your_client_id_here
|
||||
SALLA_CLIENT_SECRET=your_client_secret_here
|
||||
SALLA_REDIRECT_URI=http://localhost:3001/auth/callback
|
||||
# NocoDB (for ETL writes)
|
||||
NOCODB_URL=http://localhost:8090
|
||||
NOCODB_TOKEN=your-token
|
||||
NOCODB_BASE_ID=your-base-id
|
||||
|
||||
# After OAuth, these will be populated automatically
|
||||
# SALLA_ACCESS_TOKEN=
|
||||
# SALLA_REFRESH_TOKEN=
|
||||
# ETL sync secret (for cron auth)
|
||||
ETL_SECRET=your-secret-here
|
||||
|
||||
@@ -23,3 +23,13 @@ export const erp = {
|
||||
username: process.env.ERP_USERNAME || '',
|
||||
password: process.env.ERP_PASSWORD || '',
|
||||
};
|
||||
|
||||
export const nocodb = {
|
||||
url: process.env.NOCODB_URL || '',
|
||||
token: process.env.NOCODB_TOKEN || '',
|
||||
baseId: process.env.NOCODB_BASE_ID || '',
|
||||
};
|
||||
|
||||
export const etl = {
|
||||
secret: process.env.ETL_SECRET || '',
|
||||
};
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import { server, erp } from './config';
|
||||
import { server, erp, nocodb } from './config';
|
||||
import erpRoutes from './routes/erp';
|
||||
import etlRoutes from './routes/etl';
|
||||
|
||||
const app = express();
|
||||
app.use(cors());
|
||||
@@ -9,14 +10,21 @@ app.use(express.json());
|
||||
|
||||
// Mount routes
|
||||
app.use('/api/erp', erpRoutes);
|
||||
app.use('/api/etl', etlRoutes);
|
||||
|
||||
app.listen(server.port, () => {
|
||||
console.log(`\nServer running on http://localhost:${server.port}`);
|
||||
|
||||
if (erp.apiUrl && erp.username) {
|
||||
console.log(' GET /api/erp/sales?startDate=...&endDate=...');
|
||||
console.log(' GET /api/erp/status');
|
||||
console.log(' ERP: configured');
|
||||
} else {
|
||||
console.log(' WARNING: ERP_API_URL / ERP_USERNAME not set in .env');
|
||||
console.log(' ERP: WARNING — not configured');
|
||||
}
|
||||
|
||||
if (nocodb.url && nocodb.token) {
|
||||
console.log(' NocoDB: configured');
|
||||
console.log(' POST /api/etl/sync?mode=full|incremental');
|
||||
} else {
|
||||
console.log(' NocoDB: WARNING — not configured');
|
||||
}
|
||||
});
|
||||
|
||||
34
server/src/routes/etl.ts
Normal file
34
server/src/routes/etl.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { etl } from '../config';
|
||||
import { runSync } from '../services/etlSync';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// POST /api/etl/sync?mode=full|incremental
|
||||
router.post('/sync', async (req: Request, res: Response) => {
|
||||
// Auth check
|
||||
const auth = req.headers.authorization;
|
||||
if (etl.secret && auth !== `Bearer ${etl.secret}`) {
|
||||
res.status(401).json({ error: 'Unauthorized' });
|
||||
return;
|
||||
}
|
||||
|
||||
const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental';
|
||||
|
||||
try {
|
||||
console.log(`\nETL sync started (${mode})...`);
|
||||
const result = await runSync(mode);
|
||||
console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`);
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
console.error('ETL sync failed:', (err as Error).message);
|
||||
res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/etl/status
|
||||
router.get('/status', (_req: Request, res: Response) => {
|
||||
res.json({ configured: !!etl.secret });
|
||||
});
|
||||
|
||||
export default router;
|
||||
119
server/src/services/etlSync.ts
Normal file
119
server/src/services/etlSync.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
import { fetchSales } from './erpClient';
|
||||
import { discoverTableIds, deleteRowsByMonth, deleteAllRows, insertRecords } from './nocodbClient';
|
||||
import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
|
||||
import type { ERPSaleRecord, AggregatedRecord } from '../types';
|
||||
|
||||
function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
|
||||
const now = new Date();
|
||||
const endYear = now.getFullYear();
|
||||
const endMonth = now.getMonth() + 1;
|
||||
const boundaries: Array<[string, string]> = [];
|
||||
|
||||
let y = startYear;
|
||||
let m = startMonth;
|
||||
while (y < endYear || (y === endYear && m <= endMonth)) {
|
||||
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
|
||||
const nextM = m === 12 ? 1 : m + 1;
|
||||
const nextY = m === 12 ? y + 1 : y;
|
||||
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
|
||||
boundaries.push([start, end]);
|
||||
y = nextY;
|
||||
m = nextM;
|
||||
}
|
||||
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
function currentMonthBoundary(): [string, string] {
|
||||
const now = new Date();
|
||||
const y = now.getFullYear();
|
||||
const m = now.getMonth() + 1;
|
||||
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
|
||||
const nextM = m === 12 ? 1 : m + 1;
|
||||
const nextY = m === 12 ? y + 1 : y;
|
||||
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
|
||||
return [start, end];
|
||||
}
|
||||
|
||||
export function aggregateTransactions(sales: ERPSaleRecord[]): AggregatedRecord[] {
|
||||
const map = new Map<string, AggregatedRecord>();
|
||||
|
||||
for (const sale of sales) {
|
||||
const date = sale.TransactionDate.split(' ')[0];
|
||||
const channel = getChannelLabel(sale.OperatingAreaName);
|
||||
|
||||
for (const product of sale.Products) {
|
||||
const museum = getMuseumFromProduct(product.ProductDescription);
|
||||
const key = `${date}|${museum}|${channel}`;
|
||||
|
||||
let entry = map.get(key);
|
||||
if (!entry) {
|
||||
entry = { Date: date, MuseumName: museum, Channel: channel, Visits: 0, Tickets: 0, GrossRevenue: 0, NetRevenue: 0 };
|
||||
map.set(key, entry);
|
||||
}
|
||||
|
||||
entry.Visits += product.PeopleCount;
|
||||
entry.Tickets += product.UnitQuantity;
|
||||
entry.GrossRevenue += product.TotalPrice;
|
||||
entry.NetRevenue += product.TotalPrice - product.TaxAmount;
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(map.values());
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
status: string;
|
||||
mode: string;
|
||||
transactionsFetched: number;
|
||||
recordsWritten: number;
|
||||
duration: string;
|
||||
}
|
||||
|
||||
export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Promise<SyncResult> {
|
||||
const start = Date.now();
|
||||
|
||||
const tables = await discoverTableIds();
|
||||
const tableId = tables['DailySales'];
|
||||
if (!tableId) throw new Error("NocoDB table 'DailySales' not found");
|
||||
|
||||
let months: Array<[string, string]>;
|
||||
if (mode === 'full') {
|
||||
months = generateMonthBoundaries(2024, 1);
|
||||
} else {
|
||||
months = [currentMonthBoundary()];
|
||||
}
|
||||
|
||||
// Fetch from ERP sequentially (API can't handle concurrent requests)
|
||||
const allSales: ERPSaleRecord[] = [];
|
||||
for (const [startDate, endDate] of months) {
|
||||
console.log(` Fetching ${startDate.slice(0, 7)}...`);
|
||||
const chunk = await fetchSales(startDate, endDate) as ERPSaleRecord[];
|
||||
allSales.push(...chunk);
|
||||
}
|
||||
|
||||
const records = aggregateTransactions(allSales);
|
||||
|
||||
// Write to NocoDB
|
||||
if (mode === 'full') {
|
||||
console.log(' Clearing all DailySales rows...');
|
||||
await deleteAllRows(tableId);
|
||||
} else {
|
||||
const yearMonth = months[0][0].slice(0, 7);
|
||||
console.log(` Clearing ${yearMonth} rows...`);
|
||||
await deleteRowsByMonth(tableId, yearMonth);
|
||||
}
|
||||
|
||||
console.log(` Inserting ${records.length} records...`);
|
||||
const written = await insertRecords(tableId, records);
|
||||
|
||||
const duration = ((Date.now() - start) / 1000).toFixed(1) + 's';
|
||||
|
||||
return {
|
||||
status: 'ok',
|
||||
mode,
|
||||
transactionsFetched: allSales.length,
|
||||
recordsWritten: written,
|
||||
duration,
|
||||
};
|
||||
}
|
||||
109
server/src/services/nocodbClient.ts
Normal file
109
server/src/services/nocodbClient.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import { nocodb } from '../config';
|
||||
import type { AggregatedRecord } from '../types';
|
||||
|
||||
let discoveredTables: Record<string, string> | null = null;
|
||||
|
||||
async function fetchJson(url: string, options: RequestInit = {}): Promise<unknown> {
|
||||
const res = await fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
'xc-token': nocodb.token,
|
||||
'Content-Type': 'application/json',
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => '');
|
||||
throw new Error(`NocoDB ${res.status}: ${text.slice(0, 200)}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function discoverTableIds(): Promise<Record<string, string>> {
|
||||
if (discoveredTables) return discoveredTables;
|
||||
if (!nocodb.baseId) throw new Error('NOCODB_BASE_ID not configured');
|
||||
|
||||
const json = await fetchJson(
|
||||
`${nocodb.url}/api/v2/meta/bases/${nocodb.baseId}/tables`
|
||||
) as { list: Array<{ title: string; id: string }> };
|
||||
|
||||
const tables: Record<string, string> = {};
|
||||
for (const t of json.list) {
|
||||
tables[t.title] = t.id;
|
||||
}
|
||||
|
||||
discoveredTables = tables;
|
||||
return tables;
|
||||
}
|
||||
|
||||
export async function deleteRowsByMonth(tableId: string, yearMonth: string): Promise<number> {
|
||||
// Fetch all row IDs for the given month using a where filter
|
||||
const where = `(Date,like,${yearMonth}%)`;
|
||||
let deleted = 0;
|
||||
let hasMore = true;
|
||||
|
||||
while (hasMore) {
|
||||
const json = await fetchJson(
|
||||
`${nocodb.url}/api/v2/tables/${tableId}/records?where=${encodeURIComponent(where)}&limit=100&fields=Id`
|
||||
) as { list: Array<{ Id: number }> };
|
||||
|
||||
const ids = json.list.map(r => r.Id);
|
||||
if (ids.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
// Bulk delete
|
||||
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
|
||||
method: 'DELETE',
|
||||
body: JSON.stringify(ids.map(Id => ({ Id }))),
|
||||
});
|
||||
|
||||
deleted += ids.length;
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
export async function deleteAllRows(tableId: string): Promise<number> {
|
||||
let deleted = 0;
|
||||
let hasMore = true;
|
||||
|
||||
while (hasMore) {
|
||||
const json = await fetchJson(
|
||||
`${nocodb.url}/api/v2/tables/${tableId}/records?limit=100&fields=Id`
|
||||
) as { list: Array<{ Id: number }> };
|
||||
|
||||
const ids = json.list.map(r => r.Id);
|
||||
if (ids.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
|
||||
method: 'DELETE',
|
||||
body: JSON.stringify(ids.map(Id => ({ Id }))),
|
||||
});
|
||||
|
||||
deleted += ids.length;
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
export async function insertRecords(tableId: string, records: AggregatedRecord[]): Promise<number> {
|
||||
// NocoDB bulk insert accepts max 100 records at a time
|
||||
const batchSize = 100;
|
||||
let inserted = 0;
|
||||
|
||||
for (let i = 0; i < records.length; i += batchSize) {
|
||||
const batch = records.slice(i, i + batchSize);
|
||||
await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(batch),
|
||||
});
|
||||
inserted += batch.length;
|
||||
}
|
||||
|
||||
return inserted;
|
||||
}
|
||||
27
server/src/types.ts
Normal file
27
server/src/types.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
export interface ERPProduct {
|
||||
ProductDescription: string;
|
||||
SiteDescription: string | null;
|
||||
UnitQuantity: number;
|
||||
PeopleCount: number;
|
||||
TaxAmount: number;
|
||||
TotalPrice: number;
|
||||
}
|
||||
|
||||
export interface ERPSaleRecord {
|
||||
SaleId: number;
|
||||
TransactionDate: string;
|
||||
CustIdentification: string;
|
||||
OperatingAreaName: string;
|
||||
Payments: Array<{ PaymentMethodDescription: string }>;
|
||||
Products: ERPProduct[];
|
||||
}
|
||||
|
||||
export interface AggregatedRecord {
|
||||
Date: string;
|
||||
MuseumName: string;
|
||||
Channel: string;
|
||||
Visits: number;
|
||||
Tickets: number;
|
||||
GrossRevenue: number;
|
||||
NetRevenue: number;
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
// Data source: Hono ERP API (via server proxy) for museum sales
|
||||
// NocoDB: PilgrimStats only
|
||||
// Data source: NocoDB (DailySales populated by server-side ETL, PilgrimStats)
|
||||
// Offline mode: caches data to localStorage for resilience
|
||||
|
||||
import type {
|
||||
@@ -12,13 +11,12 @@ import type {
|
||||
FetchResult,
|
||||
GroupedData,
|
||||
UmrahData,
|
||||
NocoDBDailySale,
|
||||
DataErrorType
|
||||
} from '../types';
|
||||
import { DataError } from '../types';
|
||||
import { fetchWithRetry } from '../utils/fetchHelpers';
|
||||
import { fetchFromERP } from './erpService';
|
||||
|
||||
// NocoDB config (PilgrimStats only)
|
||||
const NOCODB_URL = import.meta.env.VITE_NOCODB_URL || '';
|
||||
const NOCODB_TOKEN = import.meta.env.VITE_NOCODB_TOKEN || '';
|
||||
const NOCODB_BASE_ID = import.meta.env.VITE_NOCODB_BASE_ID || '';
|
||||
@@ -28,7 +26,7 @@ let discoveredTables: Record<string, string> | null = null;
|
||||
|
||||
async function discoverTableIds(): Promise<Record<string, string>> {
|
||||
if (discoveredTables) return discoveredTables;
|
||||
if (!NOCODB_BASE_ID) throw new Error('VITE_NOCODB_BASE_ID not configured');
|
||||
if (!NOCODB_BASE_ID) throw new Error('NocoDB not configured');
|
||||
|
||||
const res = await fetchWithRetry(
|
||||
`${NOCODB_URL}/api/v2/meta/bases/${NOCODB_BASE_ID}/tables`,
|
||||
@@ -45,6 +43,27 @@ async function discoverTableIds(): Promise<Record<string, string>> {
|
||||
return tables;
|
||||
}
|
||||
|
||||
async function fetchNocoDBTable<T>(tableId: string, limit: number = 1000): Promise<T[]> {
|
||||
let allRecords: T[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (true) {
|
||||
const response = await fetchWithRetry(
|
||||
`${NOCODB_URL}/api/v2/tables/${tableId}/records?limit=${limit}&offset=${offset}`,
|
||||
{ headers: { 'xc-token': NOCODB_TOKEN } }
|
||||
);
|
||||
|
||||
const json = await response.json();
|
||||
const records: T[] = json.list || [];
|
||||
allRecords = allRecords.concat(records);
|
||||
|
||||
if (records.length < limit) break;
|
||||
offset += limit;
|
||||
}
|
||||
|
||||
return allRecords;
|
||||
}
|
||||
|
||||
// Cache keys
|
||||
const CACHE_KEY = 'hihala_data_cache';
|
||||
const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
|
||||
@@ -89,6 +108,41 @@ export async function fetchPilgrimStats(): Promise<UmrahData> {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// NocoDB DailySales Fetching
|
||||
// ============================================
|
||||
|
||||
async function fetchFromNocoDB(): Promise<MuseumRecord[]> {
|
||||
console.log('Fetching from NocoDB DailySales...');
|
||||
|
||||
const tables = await discoverTableIds();
|
||||
if (!tables['DailySales']) throw new Error("NocoDB table 'DailySales' not found — run ETL sync first");
|
||||
|
||||
const rows = await fetchNocoDBTable<NocoDBDailySale>(tables['DailySales']);
|
||||
|
||||
const data: MuseumRecord[] = rows.map(row => {
|
||||
const date = row.Date;
|
||||
const year = date ? date.substring(0, 4) : '';
|
||||
const month = date ? parseInt(date.substring(5, 7)) : 0;
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
return {
|
||||
date,
|
||||
museum_name: row.MuseumName,
|
||||
channel: row.Channel,
|
||||
visits: row.Visits,
|
||||
tickets: row.Tickets,
|
||||
revenue_gross: row.GrossRevenue,
|
||||
revenue_net: row.NetRevenue,
|
||||
year,
|
||||
quarter,
|
||||
};
|
||||
}).filter(r => r.date && r.museum_name);
|
||||
|
||||
console.log(`Loaded ${data.length} rows from NocoDB DailySales`);
|
||||
return data;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Offline Cache Functions
|
||||
// ============================================
|
||||
@@ -167,12 +221,29 @@ function classifyError(err: Error): DataErrorType {
|
||||
// ============================================
|
||||
|
||||
export async function fetchData(): Promise<FetchResult> {
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
console.warn('NocoDB not configured, using cached data');
|
||||
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
|
||||
}
|
||||
throw new DataError('NocoDB not configured', 'config');
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await fetchFromERP();
|
||||
const data = await fetchFromNocoDB();
|
||||
|
||||
// Suspicious data check — prefer cache if NocoDB returns too few rows
|
||||
const cached = loadFromCache();
|
||||
if (data.length < 10 && cached && cached.data.length > 10) {
|
||||
console.warn('NocoDB returned suspiciously few rows, using cache');
|
||||
return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp };
|
||||
}
|
||||
|
||||
saveToCache(data);
|
||||
return { data, fromCache: false };
|
||||
} catch (err) {
|
||||
console.error('ERP fetch failed:', (err as Error).message);
|
||||
console.error('NocoDB fetch failed:', (err as Error).message);
|
||||
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
@@ -186,7 +257,11 @@ export async function fetchData(): Promise<FetchResult> {
|
||||
}
|
||||
|
||||
export async function refreshData(): Promise<FetchResult> {
|
||||
const data = await fetchFromERP();
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) {
|
||||
throw new DataError('NocoDB not configured', 'config');
|
||||
}
|
||||
|
||||
const data = await fetchFromNocoDB();
|
||||
saveToCache(data);
|
||||
return { data, fromCache: false };
|
||||
}
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping';
|
||||
import type { ERPSaleRecord, MuseumRecord } from '../types';
|
||||
|
||||
function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> {
|
||||
const now = new Date();
|
||||
const endYear = now.getFullYear();
|
||||
const endMonth = now.getMonth() + 1;
|
||||
const boundaries: Array<[string, string]> = [];
|
||||
|
||||
let y = startYear;
|
||||
let m = startMonth;
|
||||
while (y < endYear || (y === endYear && m <= endMonth)) {
|
||||
const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`;
|
||||
const nextM = m === 12 ? 1 : m + 1;
|
||||
const nextY = m === 12 ? y + 1 : y;
|
||||
const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`;
|
||||
boundaries.push([start, end]);
|
||||
y = nextY;
|
||||
m = nextM;
|
||||
}
|
||||
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
async function fetchChunk(startDate: string, endDate: string): Promise<ERPSaleRecord[]> {
|
||||
const params = new URLSearchParams({ startDate, endDate });
|
||||
const res = await fetch(`/api/erp/sales?${params}`);
|
||||
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error((body as { error?: string }).error || `ERP proxy returned ${res.status}`);
|
||||
}
|
||||
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export function aggregateTransactions(sales: ERPSaleRecord[]): MuseumRecord[] {
|
||||
const map = new Map<string, { visits: number; tickets: number; revenue_gross: number; revenue_net: number }>();
|
||||
|
||||
for (const sale of sales) {
|
||||
const date = sale.TransactionDate.split(' ')[0];
|
||||
const channel = getChannelLabel(sale.OperatingAreaName);
|
||||
|
||||
for (const product of sale.Products) {
|
||||
const museum = getMuseumFromProduct(product.ProductDescription);
|
||||
const key = `${date}|${museum}|${channel}`;
|
||||
|
||||
let entry = map.get(key);
|
||||
if (!entry) {
|
||||
entry = { visits: 0, tickets: 0, revenue_gross: 0, revenue_net: 0 };
|
||||
map.set(key, entry);
|
||||
}
|
||||
|
||||
entry.visits += product.PeopleCount;
|
||||
entry.tickets += product.UnitQuantity;
|
||||
entry.revenue_gross += product.TotalPrice;
|
||||
entry.revenue_net += product.TotalPrice - product.TaxAmount;
|
||||
}
|
||||
}
|
||||
|
||||
const records: MuseumRecord[] = [];
|
||||
for (const [key, entry] of map) {
|
||||
const [date, museum_name, channel] = key.split('|');
|
||||
const year = date.substring(0, 4);
|
||||
const month = parseInt(date.substring(5, 7));
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
records.push({
|
||||
date,
|
||||
museum_name,
|
||||
channel,
|
||||
visits: entry.visits,
|
||||
tickets: entry.tickets,
|
||||
revenue_gross: entry.revenue_gross,
|
||||
revenue_net: entry.revenue_net,
|
||||
year,
|
||||
quarter,
|
||||
});
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
export async function fetchFromERP(): Promise<MuseumRecord[]> {
|
||||
console.log('Fetching from ERP API via proxy...');
|
||||
const months = generateMonthBoundaries(2024, 1);
|
||||
|
||||
// Fetch months sequentially — the ERP API doesn't handle concurrent requests well
|
||||
const allSales: ERPSaleRecord[] = [];
|
||||
|
||||
for (const [start, end] of months) {
|
||||
const chunk = await fetchChunk(start, end);
|
||||
allSales.push(...chunk);
|
||||
}
|
||||
|
||||
console.log(`Fetched ${allSales.length} transactions, aggregating...`);
|
||||
const records = aggregateTransactions(allSales);
|
||||
console.log(`Aggregated into ${records.length} daily records`);
|
||||
|
||||
return records;
|
||||
}
|
||||
@@ -73,27 +73,16 @@ export interface UmrahData {
|
||||
};
|
||||
}
|
||||
|
||||
// ERP API types
|
||||
export interface ERPProduct {
|
||||
ProductDescription: string;
|
||||
SiteDescription: string | null;
|
||||
UnitQuantity: number;
|
||||
PeopleCount: number;
|
||||
TaxAmount: number;
|
||||
TotalPrice: number;
|
||||
}
|
||||
|
||||
export interface ERPPayment {
|
||||
PaymentMethodDescription: string;
|
||||
}
|
||||
|
||||
export interface ERPSaleRecord {
|
||||
SaleId: number;
|
||||
TransactionDate: string;
|
||||
CustIdentification: string;
|
||||
OperatingAreaName: string;
|
||||
Payments: ERPPayment[];
|
||||
Products: ERPProduct[];
|
||||
// NocoDB DailySales row (populated by server-side ETL)
|
||||
export interface NocoDBDailySale {
|
||||
Id: number;
|
||||
Date: string;
|
||||
MuseumName: string;
|
||||
Channel: string;
|
||||
Visits: number;
|
||||
Tickets: number;
|
||||
GrossRevenue: number;
|
||||
NetRevenue: number;
|
||||
}
|
||||
|
||||
// Chart data types
|
||||
|
||||
45
start-dev.sh
Executable file
45
start-dev.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Temporary dev script for ERP migration — starts NocoDB + Express server + Vite
|
||||
|
||||
set -e
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Shutting down..."
|
||||
kill $SERVER_PID $CLIENT_PID 2>/dev/null
|
||||
docker stop nocodb 2>/dev/null
|
||||
echo "Done."
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# Start NocoDB
|
||||
if docker ps --format '{{.Names}}' | grep -q '^nocodb$'; then
|
||||
echo "NocoDB already running on port 8090"
|
||||
else
|
||||
echo "Starting NocoDB..."
|
||||
docker start nocodb 2>/dev/null || docker run -d \
|
||||
--name nocodb -p 8090:8080 nocodb/nocodb:latest
|
||||
fi
|
||||
|
||||
echo "Waiting for NocoDB..."
|
||||
for i in $(seq 1 30); do
|
||||
curl -s http://localhost:8090/api/v1/health >/dev/null 2>&1 && echo "NocoDB ready" && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Start Express server (port 3002)
|
||||
echo "Starting Express server..."
|
||||
(cd server && npm run dev) &
|
||||
SERVER_PID=$!
|
||||
|
||||
sleep 2
|
||||
|
||||
# Start Vite (port 3000)
|
||||
echo "Starting Vite..."
|
||||
npx vite &
|
||||
CLIENT_PID=$!
|
||||
|
||||
wait $CLIENT_PID
|
||||
Reference in New Issue
Block a user