diff --git a/server/.env.example b/server/.env.example index 19f43f6..4745518 100644 --- a/server/.env.example +++ b/server/.env.example @@ -7,11 +7,10 @@ ERP_API_CODE=your-api-function-key ERP_USERNAME=your-username ERP_PASSWORD=your-password -# Salla OAuth Credentials (from Salla Partners dashboard) -SALLA_CLIENT_ID=your_client_id_here -SALLA_CLIENT_SECRET=your_client_secret_here -SALLA_REDIRECT_URI=http://localhost:3001/auth/callback +# NocoDB (for ETL writes) +NOCODB_URL=http://localhost:8090 +NOCODB_TOKEN=your-token +NOCODB_BASE_ID=your-base-id -# After OAuth, these will be populated automatically -# SALLA_ACCESS_TOKEN= -# SALLA_REFRESH_TOKEN= +# ETL sync secret (for cron auth) +ETL_SECRET=your-secret-here diff --git a/server/src/config.ts b/server/src/config.ts index b815c39..cb48ef4 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -23,3 +23,13 @@ export const erp = { username: process.env.ERP_USERNAME || '', password: process.env.ERP_PASSWORD || '', }; + +export const nocodb = { + url: process.env.NOCODB_URL || '', + token: process.env.NOCODB_TOKEN || '', + baseId: process.env.NOCODB_BASE_ID || '', +}; + +export const etl = { + secret: process.env.ETL_SECRET || '', +}; diff --git a/src/config/museumMapping.ts b/server/src/config/museumMapping.ts similarity index 100% rename from src/config/museumMapping.ts rename to server/src/config/museumMapping.ts diff --git a/server/src/index.ts b/server/src/index.ts index f1f13a8..4b54e08 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,7 +1,8 @@ import express from 'express'; import cors from 'cors'; -import { server, erp } from './config'; +import { server, erp, nocodb } from './config'; import erpRoutes from './routes/erp'; +import etlRoutes from './routes/etl'; const app = express(); app.use(cors()); @@ -9,14 +10,21 @@ app.use(express.json()); // Mount routes app.use('/api/erp', erpRoutes); +app.use('/api/etl', etlRoutes); app.listen(server.port, () => { console.log(`\nServer running on http://localhost:${server.port}`); if (erp.apiUrl && erp.username) { - console.log(' GET /api/erp/sales?startDate=...&endDate=...'); - console.log(' GET /api/erp/status'); + console.log(' ERP: configured'); } else { - console.log(' WARNING: ERP_API_URL / ERP_USERNAME not set in .env'); + console.log(' ERP: WARNING — not configured'); + } + + if (nocodb.url && nocodb.token) { + console.log(' NocoDB: configured'); + console.log(' POST /api/etl/sync?mode=full|incremental'); + } else { + console.log(' NocoDB: WARNING — not configured'); } }); diff --git a/server/src/routes/etl.ts b/server/src/routes/etl.ts new file mode 100644 index 0000000..53f065d --- /dev/null +++ b/server/src/routes/etl.ts @@ -0,0 +1,34 @@ +import { Router, Request, Response } from 'express'; +import { etl } from '../config'; +import { runSync } from '../services/etlSync'; + +const router = Router(); + +// POST /api/etl/sync?mode=full|incremental +router.post('/sync', async (req: Request, res: Response) => { + // Auth check + const auth = req.headers.authorization; + if (etl.secret && auth !== `Bearer ${etl.secret}`) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + const mode = (req.query.mode as string) === 'full' ? 'full' : 'incremental'; + + try { + console.log(`\nETL sync started (${mode})...`); + const result = await runSync(mode); + console.log(`ETL sync complete: ${result.transactionsFetched} transactions → ${result.recordsWritten} records in ${result.duration}`); + res.json(result); + } catch (err) { + console.error('ETL sync failed:', (err as Error).message); + res.status(500).json({ error: 'ETL sync failed', details: (err as Error).message }); + } +}); + +// GET /api/etl/status +router.get('/status', (_req: Request, res: Response) => { + res.json({ configured: !!etl.secret }); +}); + +export default router; diff --git a/server/src/services/etlSync.ts b/server/src/services/etlSync.ts new file mode 100644 index 0000000..505d18d --- /dev/null +++ b/server/src/services/etlSync.ts @@ -0,0 +1,119 @@ +import { fetchSales } from './erpClient'; +import { discoverTableIds, deleteRowsByMonth, deleteAllRows, insertRecords } from './nocodbClient'; +import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping'; +import type { ERPSaleRecord, AggregatedRecord } from '../types'; + +function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> { + const now = new Date(); + const endYear = now.getFullYear(); + const endMonth = now.getMonth() + 1; + const boundaries: Array<[string, string]> = []; + + let y = startYear; + let m = startMonth; + while (y < endYear || (y === endYear && m <= endMonth)) { + const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`; + const nextM = m === 12 ? 1 : m + 1; + const nextY = m === 12 ? y + 1 : y; + const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`; + boundaries.push([start, end]); + y = nextY; + m = nextM; + } + + return boundaries; +} + +function currentMonthBoundary(): [string, string] { + const now = new Date(); + const y = now.getFullYear(); + const m = now.getMonth() + 1; + const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`; + const nextM = m === 12 ? 1 : m + 1; + const nextY = m === 12 ? y + 1 : y; + const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`; + return [start, end]; +} + +export function aggregateTransactions(sales: ERPSaleRecord[]): AggregatedRecord[] { + const map = new Map(); + + for (const sale of sales) { + const date = sale.TransactionDate.split(' ')[0]; + const channel = getChannelLabel(sale.OperatingAreaName); + + for (const product of sale.Products) { + const museum = getMuseumFromProduct(product.ProductDescription); + const key = `${date}|${museum}|${channel}`; + + let entry = map.get(key); + if (!entry) { + entry = { Date: date, MuseumName: museum, Channel: channel, Visits: 0, Tickets: 0, GrossRevenue: 0, NetRevenue: 0 }; + map.set(key, entry); + } + + entry.Visits += product.PeopleCount; + entry.Tickets += product.UnitQuantity; + entry.GrossRevenue += product.TotalPrice; + entry.NetRevenue += product.TotalPrice - product.TaxAmount; + } + } + + return Array.from(map.values()); +} + +export interface SyncResult { + status: string; + mode: string; + transactionsFetched: number; + recordsWritten: number; + duration: string; +} + +export async function runSync(mode: 'full' | 'incremental' = 'incremental'): Promise { + const start = Date.now(); + + const tables = await discoverTableIds(); + const tableId = tables['DailySales']; + if (!tableId) throw new Error("NocoDB table 'DailySales' not found"); + + let months: Array<[string, string]>; + if (mode === 'full') { + months = generateMonthBoundaries(2024, 1); + } else { + months = [currentMonthBoundary()]; + } + + // Fetch from ERP sequentially (API can't handle concurrent requests) + const allSales: ERPSaleRecord[] = []; + for (const [startDate, endDate] of months) { + console.log(` Fetching ${startDate.slice(0, 7)}...`); + const chunk = await fetchSales(startDate, endDate) as ERPSaleRecord[]; + allSales.push(...chunk); + } + + const records = aggregateTransactions(allSales); + + // Write to NocoDB + if (mode === 'full') { + console.log(' Clearing all DailySales rows...'); + await deleteAllRows(tableId); + } else { + const yearMonth = months[0][0].slice(0, 7); + console.log(` Clearing ${yearMonth} rows...`); + await deleteRowsByMonth(tableId, yearMonth); + } + + console.log(` Inserting ${records.length} records...`); + const written = await insertRecords(tableId, records); + + const duration = ((Date.now() - start) / 1000).toFixed(1) + 's'; + + return { + status: 'ok', + mode, + transactionsFetched: allSales.length, + recordsWritten: written, + duration, + }; +} diff --git a/server/src/services/nocodbClient.ts b/server/src/services/nocodbClient.ts new file mode 100644 index 0000000..ebf203d --- /dev/null +++ b/server/src/services/nocodbClient.ts @@ -0,0 +1,109 @@ +import { nocodb } from '../config'; +import type { AggregatedRecord } from '../types'; + +let discoveredTables: Record | null = null; + +async function fetchJson(url: string, options: RequestInit = {}): Promise { + const res = await fetch(url, { + ...options, + headers: { + 'xc-token': nocodb.token, + 'Content-Type': 'application/json', + ...options.headers, + }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`NocoDB ${res.status}: ${text.slice(0, 200)}`); + } + return res.json(); +} + +export async function discoverTableIds(): Promise> { + if (discoveredTables) return discoveredTables; + if (!nocodb.baseId) throw new Error('NOCODB_BASE_ID not configured'); + + const json = await fetchJson( + `${nocodb.url}/api/v2/meta/bases/${nocodb.baseId}/tables` + ) as { list: Array<{ title: string; id: string }> }; + + const tables: Record = {}; + for (const t of json.list) { + tables[t.title] = t.id; + } + + discoveredTables = tables; + return tables; +} + +export async function deleteRowsByMonth(tableId: string, yearMonth: string): Promise { + // Fetch all row IDs for the given month using a where filter + const where = `(Date,like,${yearMonth}%)`; + let deleted = 0; + let hasMore = true; + + while (hasMore) { + const json = await fetchJson( + `${nocodb.url}/api/v2/tables/${tableId}/records?where=${encodeURIComponent(where)}&limit=100&fields=Id` + ) as { list: Array<{ Id: number }> }; + + const ids = json.list.map(r => r.Id); + if (ids.length === 0) { + hasMore = false; + break; + } + + // Bulk delete + await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, { + method: 'DELETE', + body: JSON.stringify(ids.map(Id => ({ Id }))), + }); + + deleted += ids.length; + } + + return deleted; +} + +export async function deleteAllRows(tableId: string): Promise { + let deleted = 0; + let hasMore = true; + + while (hasMore) { + const json = await fetchJson( + `${nocodb.url}/api/v2/tables/${tableId}/records?limit=100&fields=Id` + ) as { list: Array<{ Id: number }> }; + + const ids = json.list.map(r => r.Id); + if (ids.length === 0) { + hasMore = false; + break; + } + + await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, { + method: 'DELETE', + body: JSON.stringify(ids.map(Id => ({ Id }))), + }); + + deleted += ids.length; + } + + return deleted; +} + +export async function insertRecords(tableId: string, records: AggregatedRecord[]): Promise { + // NocoDB bulk insert accepts max 100 records at a time + const batchSize = 100; + let inserted = 0; + + for (let i = 0; i < records.length; i += batchSize) { + const batch = records.slice(i, i + batchSize); + await fetchJson(`${nocodb.url}/api/v2/tables/${tableId}/records`, { + method: 'POST', + body: JSON.stringify(batch), + }); + inserted += batch.length; + } + + return inserted; +} diff --git a/server/src/types.ts b/server/src/types.ts new file mode 100644 index 0000000..bf1d9d9 --- /dev/null +++ b/server/src/types.ts @@ -0,0 +1,27 @@ +export interface ERPProduct { + ProductDescription: string; + SiteDescription: string | null; + UnitQuantity: number; + PeopleCount: number; + TaxAmount: number; + TotalPrice: number; +} + +export interface ERPSaleRecord { + SaleId: number; + TransactionDate: string; + CustIdentification: string; + OperatingAreaName: string; + Payments: Array<{ PaymentMethodDescription: string }>; + Products: ERPProduct[]; +} + +export interface AggregatedRecord { + Date: string; + MuseumName: string; + Channel: string; + Visits: number; + Tickets: number; + GrossRevenue: number; + NetRevenue: number; +} diff --git a/src/services/dataService.ts b/src/services/dataService.ts index 759d257..5e242ab 100644 --- a/src/services/dataService.ts +++ b/src/services/dataService.ts @@ -1,5 +1,4 @@ -// Data source: Hono ERP API (via server proxy) for museum sales -// NocoDB: PilgrimStats only +// Data source: NocoDB (DailySales populated by server-side ETL, PilgrimStats) // Offline mode: caches data to localStorage for resilience import type { @@ -12,13 +11,12 @@ import type { FetchResult, GroupedData, UmrahData, + NocoDBDailySale, DataErrorType } from '../types'; import { DataError } from '../types'; import { fetchWithRetry } from '../utils/fetchHelpers'; -import { fetchFromERP } from './erpService'; -// NocoDB config (PilgrimStats only) const NOCODB_URL = import.meta.env.VITE_NOCODB_URL || ''; const NOCODB_TOKEN = import.meta.env.VITE_NOCODB_TOKEN || ''; const NOCODB_BASE_ID = import.meta.env.VITE_NOCODB_BASE_ID || ''; @@ -28,7 +26,7 @@ let discoveredTables: Record | null = null; async function discoverTableIds(): Promise> { if (discoveredTables) return discoveredTables; - if (!NOCODB_BASE_ID) throw new Error('VITE_NOCODB_BASE_ID not configured'); + if (!NOCODB_BASE_ID) throw new Error('NocoDB not configured'); const res = await fetchWithRetry( `${NOCODB_URL}/api/v2/meta/bases/${NOCODB_BASE_ID}/tables`, @@ -45,6 +43,27 @@ async function discoverTableIds(): Promise> { return tables; } +async function fetchNocoDBTable(tableId: string, limit: number = 1000): Promise { + let allRecords: T[] = []; + let offset = 0; + + while (true) { + const response = await fetchWithRetry( + `${NOCODB_URL}/api/v2/tables/${tableId}/records?limit=${limit}&offset=${offset}`, + { headers: { 'xc-token': NOCODB_TOKEN } } + ); + + const json = await response.json(); + const records: T[] = json.list || []; + allRecords = allRecords.concat(records); + + if (records.length < limit) break; + offset += limit; + } + + return allRecords; +} + // Cache keys const CACHE_KEY = 'hihala_data_cache'; const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp'; @@ -89,6 +108,41 @@ export async function fetchPilgrimStats(): Promise { } } +// ============================================ +// NocoDB DailySales Fetching +// ============================================ + +async function fetchFromNocoDB(): Promise { + console.log('Fetching from NocoDB DailySales...'); + + const tables = await discoverTableIds(); + if (!tables['DailySales']) throw new Error("NocoDB table 'DailySales' not found — run ETL sync first"); + + const rows = await fetchNocoDBTable(tables['DailySales']); + + const data: MuseumRecord[] = rows.map(row => { + const date = row.Date; + const year = date ? date.substring(0, 4) : ''; + const month = date ? parseInt(date.substring(5, 7)) : 0; + const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4'; + + return { + date, + museum_name: row.MuseumName, + channel: row.Channel, + visits: row.Visits, + tickets: row.Tickets, + revenue_gross: row.GrossRevenue, + revenue_net: row.NetRevenue, + year, + quarter, + }; + }).filter(r => r.date && r.museum_name); + + console.log(`Loaded ${data.length} rows from NocoDB DailySales`); + return data; +} + // ============================================ // Offline Cache Functions // ============================================ @@ -167,12 +221,29 @@ function classifyError(err: Error): DataErrorType { // ============================================ export async function fetchData(): Promise { + if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) { + const cached = loadFromCache(); + if (cached) { + console.warn('NocoDB not configured, using cached data'); + return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp }; + } + throw new DataError('NocoDB not configured', 'config'); + } + try { - const data = await fetchFromERP(); + const data = await fetchFromNocoDB(); + + // Suspicious data check — prefer cache if NocoDB returns too few rows + const cached = loadFromCache(); + if (data.length < 10 && cached && cached.data.length > 10) { + console.warn('NocoDB returned suspiciously few rows, using cache'); + return { data: cached.data, fromCache: true, cacheTimestamp: cached.timestamp }; + } + saveToCache(data); return { data, fromCache: false }; } catch (err) { - console.error('ERP fetch failed:', (err as Error).message); + console.error('NocoDB fetch failed:', (err as Error).message); const cached = loadFromCache(); if (cached) { @@ -186,7 +257,11 @@ export async function fetchData(): Promise { } export async function refreshData(): Promise { - const data = await fetchFromERP(); + if (!NOCODB_URL || !NOCODB_TOKEN || !NOCODB_BASE_ID) { + throw new DataError('NocoDB not configured', 'config'); + } + + const data = await fetchFromNocoDB(); saveToCache(data); return { data, fromCache: false }; } diff --git a/src/services/erpService.ts b/src/services/erpService.ts deleted file mode 100644 index fcb4a02..0000000 --- a/src/services/erpService.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { getMuseumFromProduct, getChannelLabel } from '../config/museumMapping'; -import type { ERPSaleRecord, MuseumRecord } from '../types'; - -function generateMonthBoundaries(startYear: number, startMonth: number): Array<[string, string]> { - const now = new Date(); - const endYear = now.getFullYear(); - const endMonth = now.getMonth() + 1; - const boundaries: Array<[string, string]> = []; - - let y = startYear; - let m = startMonth; - while (y < endYear || (y === endYear && m <= endMonth)) { - const start = `${y}-${String(m).padStart(2, '0')}-01T00:00:00`; - const nextM = m === 12 ? 1 : m + 1; - const nextY = m === 12 ? y + 1 : y; - const end = `${nextY}-${String(nextM).padStart(2, '0')}-01T00:00:00`; - boundaries.push([start, end]); - y = nextY; - m = nextM; - } - - return boundaries; -} - -async function fetchChunk(startDate: string, endDate: string): Promise { - const params = new URLSearchParams({ startDate, endDate }); - const res = await fetch(`/api/erp/sales?${params}`); - - if (!res.ok) { - const body = await res.json().catch(() => ({})); - throw new Error((body as { error?: string }).error || `ERP proxy returned ${res.status}`); - } - - return res.json(); -} - -export function aggregateTransactions(sales: ERPSaleRecord[]): MuseumRecord[] { - const map = new Map(); - - for (const sale of sales) { - const date = sale.TransactionDate.split(' ')[0]; - const channel = getChannelLabel(sale.OperatingAreaName); - - for (const product of sale.Products) { - const museum = getMuseumFromProduct(product.ProductDescription); - const key = `${date}|${museum}|${channel}`; - - let entry = map.get(key); - if (!entry) { - entry = { visits: 0, tickets: 0, revenue_gross: 0, revenue_net: 0 }; - map.set(key, entry); - } - - entry.visits += product.PeopleCount; - entry.tickets += product.UnitQuantity; - entry.revenue_gross += product.TotalPrice; - entry.revenue_net += product.TotalPrice - product.TaxAmount; - } - } - - const records: MuseumRecord[] = []; - for (const [key, entry] of map) { - const [date, museum_name, channel] = key.split('|'); - const year = date.substring(0, 4); - const month = parseInt(date.substring(5, 7)); - const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4'; - - records.push({ - date, - museum_name, - channel, - visits: entry.visits, - tickets: entry.tickets, - revenue_gross: entry.revenue_gross, - revenue_net: entry.revenue_net, - year, - quarter, - }); - } - - return records; -} - -export async function fetchFromERP(): Promise { - console.log('Fetching from ERP API via proxy...'); - const months = generateMonthBoundaries(2024, 1); - - // Fetch months sequentially — the ERP API doesn't handle concurrent requests well - const allSales: ERPSaleRecord[] = []; - - for (const [start, end] of months) { - const chunk = await fetchChunk(start, end); - allSales.push(...chunk); - } - - console.log(`Fetched ${allSales.length} transactions, aggregating...`); - const records = aggregateTransactions(allSales); - console.log(`Aggregated into ${records.length} daily records`); - - return records; -} diff --git a/src/types/index.ts b/src/types/index.ts index 5813ec5..d1308e5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -73,27 +73,16 @@ export interface UmrahData { }; } -// ERP API types -export interface ERPProduct { - ProductDescription: string; - SiteDescription: string | null; - UnitQuantity: number; - PeopleCount: number; - TaxAmount: number; - TotalPrice: number; -} - -export interface ERPPayment { - PaymentMethodDescription: string; -} - -export interface ERPSaleRecord { - SaleId: number; - TransactionDate: string; - CustIdentification: string; - OperatingAreaName: string; - Payments: ERPPayment[]; - Products: ERPProduct[]; +// NocoDB DailySales row (populated by server-side ETL) +export interface NocoDBDailySale { + Id: number; + Date: string; + MuseumName: string; + Channel: string; + Visits: number; + Tickets: number; + GrossRevenue: number; + NetRevenue: number; } // Chart data types diff --git a/start-dev.sh b/start-dev.sh new file mode 100755 index 0000000..d35cd17 --- /dev/null +++ b/start-dev.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Temporary dev script for ERP migration — starts NocoDB + Express server + Vite + +set -e + +cleanup() { + echo "" + echo "Shutting down..." + kill $SERVER_PID $CLIENT_PID 2>/dev/null + docker stop nocodb 2>/dev/null + echo "Done." +} + +trap cleanup EXIT INT TERM + +cd "$(dirname "$0")" + +# Start NocoDB +if docker ps --format '{{.Names}}' | grep -q '^nocodb$'; then + echo "NocoDB already running on port 8090" +else + echo "Starting NocoDB..." + docker start nocodb 2>/dev/null || docker run -d \ + --name nocodb -p 8090:8080 nocodb/nocodb:latest +fi + +echo "Waiting for NocoDB..." +for i in $(seq 1 30); do + curl -s http://localhost:8090/api/v1/health >/dev/null 2>&1 && echo "NocoDB ready" && break + sleep 1 +done + +# Start Express server (port 3002) +echo "Starting Express server..." +(cd server && npm run dev) & +SERVER_PID=$! + +sleep 2 + +# Start Vite (port 3000) +echo "Starting Vite..." +npx vite & +CLIENT_PID=$! + +wait $CLIENT_PID