feat: VAT toggle + offline mode
- Rename Revenue to GrossRevenue, add NetRevenue (excl. VAT) - Add VAT toggle (Incl/Excl) on Dashboard and Comparison pages - Add offline mode with localStorage caching (24h validity) - Add refresh button and offline indicator in nav - Remove Google Sheets fallback (archived to dataService.legacy.js) - Add AR/EN translations for new UI elements
This commit is contained in:
+191
-147
@@ -1,101 +1,96 @@
|
||||
// Data source configuration - all from environment variables
|
||||
// Set these in .env.local (never commit .env.local to git)
|
||||
// Data source: NocoDB only
|
||||
// Offline mode: caches data to localStorage for resilience
|
||||
|
||||
// NocoDB (primary/default)
|
||||
const NOCODB_URL = process.env.REACT_APP_NOCODB_URL || '';
|
||||
const NOCODB_TOKEN = process.env.REACT_APP_NOCODB_TOKEN || '';
|
||||
|
||||
// Google Sheets (fallback)
|
||||
const SPREADSHEET_ID = process.env.REACT_APP_SHEETS_ID || '';
|
||||
const SHEET_NAME = process.env.REACT_APP_SHEETS_NAME || 'Consolidated Data';
|
||||
const SHEET_URL = SPREADSHEET_ID
|
||||
? `https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/gviz/tq?tqx=out:csv&sheet=${encodeURIComponent(SHEET_NAME)}`
|
||||
: '';
|
||||
|
||||
// Table IDs (not sensitive - just identifiers)
|
||||
// Table IDs
|
||||
const NOCODB_TABLES = {
|
||||
districts: 'm8cup7lesbet0sa',
|
||||
museums: 'm1c7od7mdirffvu',
|
||||
dailyStats: 'mc7qhbdh3mjjwl8'
|
||||
};
|
||||
|
||||
// Cache keys
|
||||
const CACHE_KEY = 'hihala_data_cache';
|
||||
const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
|
||||
const CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
|
||||
|
||||
export const umrahData = {
|
||||
2024: { 1: 11574494, 2: 10521465, 3: 3364627, 4: 7435625 },
|
||||
2025: { 1: 15222497, 2: 5443393, 3: null, 4: null }
|
||||
};
|
||||
|
||||
// Convert Excel serial date to YYYY-MM-DD
|
||||
function excelDateToYMD(serial) {
|
||||
const num = parseInt(serial);
|
||||
if (isNaN(num) || num < 1) return null;
|
||||
|
||||
// Excel epoch is Dec 30, 1899
|
||||
const utcDays = Math.floor(num - 25569); // 25569 = days from 1899-12-30 to 1970-01-01
|
||||
const date = new Date(utcDays * 86400 * 1000);
|
||||
|
||||
const y = date.getUTCFullYear();
|
||||
const m = String(date.getUTCMonth() + 1).padStart(2, '0');
|
||||
const d = String(date.getUTCDate()).padStart(2, '0');
|
||||
|
||||
return `${y}-${m}-${d}`;
|
||||
}
|
||||
// ============================================
|
||||
// Offline Cache Functions
|
||||
// ============================================
|
||||
|
||||
function parseCSV(text) {
|
||||
const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
||||
const lines = normalizedText.trim().split('\n');
|
||||
const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
|
||||
|
||||
return lines.slice(1).map(line => {
|
||||
const values = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let char of line) {
|
||||
if (char === '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
values.push(current.trim().replace(/^"|"$/g, ''));
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
values.push(current.trim().replace(/^"|"$/g, ''));
|
||||
|
||||
const obj = {};
|
||||
headers.forEach((header, i) => {
|
||||
let val = values[i] || '';
|
||||
// Convert date serial to YYYY-MM-DD
|
||||
if (header === 'date' && /^\d+$/.test(val)) {
|
||||
val = excelDateToYMD(val);
|
||||
}
|
||||
obj[header] = val;
|
||||
});
|
||||
return obj;
|
||||
}).filter(row => row.date);
|
||||
}
|
||||
|
||||
export async function fetchSheetData() {
|
||||
function saveToCache(data) {
|
||||
try {
|
||||
console.log('Fetching from Google Sheets...');
|
||||
const response = await fetch(SHEET_URL);
|
||||
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const text = await response.text();
|
||||
if (text.includes('<!DOCTYPE') || text.includes('<html')) {
|
||||
throw new Error('Sheet is not public');
|
||||
}
|
||||
|
||||
const data = parseCSV(text);
|
||||
console.log(`Loaded ${data.length} rows from Google Sheets`);
|
||||
return data;
|
||||
localStorage.setItem(CACHE_KEY, JSON.stringify(data));
|
||||
localStorage.setItem(CACHE_TIMESTAMP_KEY, Date.now().toString());
|
||||
console.log(`Cached ${data.length} rows to localStorage`);
|
||||
} catch (err) {
|
||||
console.error('Fetch error:', err);
|
||||
throw new Error(`Failed to load data: ${err.message}`);
|
||||
console.warn('Failed to save to cache:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
function loadFromCache() {
|
||||
try {
|
||||
const cached = localStorage.getItem(CACHE_KEY);
|
||||
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
|
||||
|
||||
if (!cached) return null;
|
||||
|
||||
const data = JSON.parse(cached);
|
||||
const age = timestamp ? Date.now() - parseInt(timestamp) : Infinity;
|
||||
const isStale = age > CACHE_MAX_AGE_MS;
|
||||
|
||||
console.log(`Loaded ${data.length} rows from cache (age: ${Math.round(age / 1000 / 60)} min, stale: ${isStale})`);
|
||||
|
||||
return { data, isStale, timestamp: parseInt(timestamp) };
|
||||
} catch (err) {
|
||||
console.warn('Failed to load from cache:', err.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function getCacheAge() {
|
||||
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
|
||||
if (!timestamp) return null;
|
||||
return Date.now() - parseInt(timestamp);
|
||||
}
|
||||
|
||||
export function getCacheStatus() {
|
||||
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
|
||||
const cached = localStorage.getItem(CACHE_KEY);
|
||||
|
||||
if (!cached || !timestamp) {
|
||||
return { available: false, timestamp: null, age: null, rows: 0 };
|
||||
}
|
||||
|
||||
const ts = parseInt(timestamp);
|
||||
const data = JSON.parse(cached);
|
||||
|
||||
return {
|
||||
available: true,
|
||||
timestamp: new Date(ts).toISOString(),
|
||||
age: Date.now() - ts,
|
||||
rows: data.length,
|
||||
isStale: (Date.now() - ts) > CACHE_MAX_AGE_MS
|
||||
};
|
||||
}
|
||||
|
||||
export function clearCache() {
|
||||
localStorage.removeItem(CACHE_KEY);
|
||||
localStorage.removeItem(CACHE_TIMESTAMP_KEY);
|
||||
console.log('Cache cleared');
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// NocoDB Data Fetching
|
||||
// ============================================
|
||||
|
||||
async function fetchNocoDBTable(tableId, limit = 1000) {
|
||||
let allRecords = [];
|
||||
let offset = 0;
|
||||
@@ -119,78 +114,113 @@ async function fetchNocoDBTable(tableId, limit = 1000) {
|
||||
return allRecords;
|
||||
}
|
||||
|
||||
export async function fetchNocoDBData() {
|
||||
async function fetchFromNocoDB() {
|
||||
console.log('Fetching from NocoDB...');
|
||||
|
||||
// Fetch all three tables in parallel
|
||||
const [districts, museums, dailyStats] = await Promise.all([
|
||||
fetchNocoDBTable(NOCODB_TABLES.districts),
|
||||
fetchNocoDBTable(NOCODB_TABLES.museums),
|
||||
fetchNocoDBTable(NOCODB_TABLES.dailyStats)
|
||||
]);
|
||||
|
||||
// Build lookup maps
|
||||
const districtMap = {};
|
||||
districts.forEach(d => { districtMap[d.Id] = d.Name; });
|
||||
|
||||
const museumMap = {};
|
||||
museums.forEach(m => {
|
||||
museumMap[m.Id] = {
|
||||
code: m.Code,
|
||||
name: m.Name,
|
||||
district: districtMap[m['nc_epk____Districts_id']] || 'Unknown'
|
||||
};
|
||||
});
|
||||
|
||||
// Join data into flat structure
|
||||
const data = dailyStats.map(row => {
|
||||
const museum = museumMap[row['nc_epk____Museums_id']] || {};
|
||||
const date = row.Date;
|
||||
const year = date ? date.substring(0, 4) : '';
|
||||
const month = date ? parseInt(date.substring(5, 7)) : 0;
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
// GrossRevenue = including VAT, NetRevenue = excluding VAT
|
||||
const grossRevenue = row.GrossRevenue || 0;
|
||||
const netRevenue = row.NetRevenue || (grossRevenue / 1.15);
|
||||
|
||||
return {
|
||||
date: date,
|
||||
museum_code: museum.code,
|
||||
museum_name: museum.name,
|
||||
district: museum.district,
|
||||
visits: row.Visits,
|
||||
tickets: row.Tickets,
|
||||
revenue_gross: grossRevenue,
|
||||
revenue_net: netRevenue,
|
||||
revenue_incl_tax: grossRevenue, // Legacy compatibility
|
||||
year: year,
|
||||
quarter: quarter
|
||||
};
|
||||
}).filter(r => r.date && r.museum_name);
|
||||
|
||||
console.log(`Loaded ${data.length} rows from NocoDB`);
|
||||
return data;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Main Data Fetcher (with offline fallback)
|
||||
// ============================================
|
||||
|
||||
export async function fetchData() {
|
||||
// Check if NocoDB is configured
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN) {
|
||||
// Try cache
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
console.warn('NocoDB not configured, using cached data');
|
||||
return cached.data;
|
||||
}
|
||||
throw new Error('NocoDB not configured and no cached data available. Set REACT_APP_NOCODB_URL and REACT_APP_NOCODB_TOKEN in .env.local');
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('Fetching from NocoDB (normalized)...');
|
||||
// Try to fetch fresh data
|
||||
const data = await fetchFromNocoDB();
|
||||
|
||||
// Fetch all three tables in parallel
|
||||
const [districts, museums, dailyStats] = await Promise.all([
|
||||
fetchNocoDBTable(NOCODB_TABLES.districts),
|
||||
fetchNocoDBTable(NOCODB_TABLES.museums),
|
||||
fetchNocoDBTable(NOCODB_TABLES.dailyStats)
|
||||
]);
|
||||
// Save to cache on success
|
||||
saveToCache(data);
|
||||
|
||||
// Build lookup maps
|
||||
const districtMap = {};
|
||||
districts.forEach(d => { districtMap[d.Id] = d.Name; });
|
||||
|
||||
const museumMap = {};
|
||||
museums.forEach(m => {
|
||||
museumMap[m.Id] = {
|
||||
code: m.Code,
|
||||
name: m.Name,
|
||||
district: districtMap[m['nc_epk____Districts_id']] || 'Unknown'
|
||||
};
|
||||
});
|
||||
|
||||
// Join data into flat structure for dashboard
|
||||
const data = dailyStats.map(row => {
|
||||
const museum = museumMap[row['nc_epk____Museums_id']] || {};
|
||||
const date = row.Date;
|
||||
const year = date ? date.substring(0, 4) : '';
|
||||
const month = date ? parseInt(date.substring(5, 7)) : 0;
|
||||
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
|
||||
|
||||
return {
|
||||
date: date,
|
||||
museum_code: museum.code,
|
||||
museum_name: museum.name,
|
||||
district: museum.district,
|
||||
visits: row.Visits,
|
||||
tickets: row.Tickets,
|
||||
revenue_incl_tax: row.Revenue,
|
||||
year: year,
|
||||
quarter: quarter
|
||||
};
|
||||
}).filter(r => r.date && r.museum_name);
|
||||
|
||||
console.log(`Loaded ${data.length} rows from NocoDB (joined from ${districts.length} districts, ${museums.length} museums, ${dailyStats.length} stats)`);
|
||||
return data;
|
||||
} catch (err) {
|
||||
console.error('NocoDB fetch error:', err);
|
||||
throw new Error(`Failed to load from NocoDB: ${err.message}`);
|
||||
console.error('NocoDB fetch failed:', err.message);
|
||||
|
||||
// Try to load from cache
|
||||
const cached = loadFromCache();
|
||||
if (cached) {
|
||||
console.warn(`Using cached data from ${new Date(cached.timestamp).toLocaleString()} (offline mode)`);
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
throw new Error(`Database unavailable and no cached data: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Main data fetcher - tries NocoDB first, falls back to Sheets
|
||||
export async function fetchData() {
|
||||
// Try NocoDB if configured
|
||||
if (NOCODB_URL && NOCODB_TOKEN) {
|
||||
try {
|
||||
return await fetchNocoDBData();
|
||||
} catch (err) {
|
||||
console.warn('NocoDB failed, trying Google Sheets fallback...', err.message);
|
||||
}
|
||||
// Force refresh (bypass cache read, but still write to cache)
|
||||
export async function refreshData() {
|
||||
if (!NOCODB_URL || !NOCODB_TOKEN) {
|
||||
throw new Error('NocoDB not configured');
|
||||
}
|
||||
|
||||
// Fallback to Google Sheets if configured
|
||||
if (SHEET_URL) {
|
||||
return await fetchSheetData();
|
||||
}
|
||||
|
||||
throw new Error('No data source configured. Set REACT_APP_NOCODB_URL + REACT_APP_NOCODB_TOKEN, or REACT_APP_SHEETS_ID in .env.local');
|
||||
const data = await fetchFromNocoDB();
|
||||
saveToCache(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Data Filtering & Metrics
|
||||
// ============================================
|
||||
|
||||
export function filterData(data, filters) {
|
||||
return data.filter(row => {
|
||||
if (filters.year && filters.year !== 'all' && row.year !== filters.year) return false;
|
||||
@@ -211,14 +241,19 @@ export function filterDataByDateRange(data, startDate, endDate, filters = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
export function calculateMetrics(data) {
|
||||
const revenue = data.reduce((sum, row) => sum + parseFloat(row.revenue_incl_tax || 0), 0);
|
||||
export function calculateMetrics(data, includeVAT = true) {
|
||||
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
|
||||
const revenue = data.reduce((sum, row) => sum + parseFloat(row[revenueField] || row.revenue_incl_tax || 0), 0);
|
||||
const visitors = data.reduce((sum, row) => sum + parseInt(row.visits || 0), 0);
|
||||
const tickets = data.reduce((sum, row) => sum + parseInt(row.tickets || 0), 0);
|
||||
const avgRevPerVisitor = visitors > 0 ? revenue / visitors : 0;
|
||||
return { revenue, visitors, tickets, avgRevPerVisitor };
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Formatting Functions
|
||||
// ============================================
|
||||
|
||||
export function formatCurrency(num) {
|
||||
if (isNaN(num)) return 'SAR 0';
|
||||
return new Intl.NumberFormat('en-US', {
|
||||
@@ -249,6 +284,10 @@ export function formatCompactCurrency(num) {
|
||||
return formatCurrency(num);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Grouping Functions
|
||||
// ============================================
|
||||
|
||||
export function getWeekStart(dateStr) {
|
||||
if (!dateStr || !dateStr.match(/^\d{4}-\d{2}-\d{2}$/)) return null;
|
||||
|
||||
@@ -265,45 +304,51 @@ export function getWeekStart(dateStr) {
|
||||
return `${y}-${m}-${d}`;
|
||||
}
|
||||
|
||||
export function groupByWeek(data) {
|
||||
export function groupByWeek(data, includeVAT = true) {
|
||||
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
|
||||
const grouped = {};
|
||||
data.forEach(row => {
|
||||
if (!row.date) return;
|
||||
const weekStart = getWeekStart(row.date);
|
||||
if (!weekStart) return;
|
||||
if (!grouped[weekStart]) grouped[weekStart] = { revenue: 0, visitors: 0, tickets: 0 };
|
||||
grouped[weekStart].revenue += parseFloat(row.revenue_incl_tax || 0);
|
||||
grouped[weekStart].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
|
||||
grouped[weekStart].visitors += parseInt(row.visits || 0);
|
||||
grouped[weekStart].tickets += parseInt(row.tickets || 0);
|
||||
});
|
||||
return grouped;
|
||||
}
|
||||
|
||||
export function groupByMuseum(data) {
|
||||
export function groupByMuseum(data, includeVAT = true) {
|
||||
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
|
||||
const grouped = {};
|
||||
data.forEach(row => {
|
||||
if (!row.museum_name) return;
|
||||
if (!grouped[row.museum_name]) grouped[row.museum_name] = { revenue: 0, visitors: 0, tickets: 0 };
|
||||
grouped[row.museum_name].revenue += parseFloat(row.revenue_incl_tax || 0);
|
||||
grouped[row.museum_name].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
|
||||
grouped[row.museum_name].visitors += parseInt(row.visits || 0);
|
||||
grouped[row.museum_name].tickets += parseInt(row.tickets || 0);
|
||||
});
|
||||
return grouped;
|
||||
}
|
||||
|
||||
export function groupByDistrict(data) {
|
||||
export function groupByDistrict(data, includeVAT = true) {
|
||||
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
|
||||
const grouped = {};
|
||||
data.forEach(row => {
|
||||
if (!row.district) return;
|
||||
if (!grouped[row.district]) grouped[row.district] = { revenue: 0, visitors: 0, tickets: 0 };
|
||||
grouped[row.district].revenue += parseFloat(row.revenue_incl_tax || 0);
|
||||
grouped[row.district].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
|
||||
grouped[row.district].visitors += parseInt(row.visits || 0);
|
||||
grouped[row.district].tickets += parseInt(row.tickets || 0);
|
||||
});
|
||||
return grouped;
|
||||
}
|
||||
|
||||
// Dynamic data extraction helpers
|
||||
// ============================================
|
||||
// Data Extraction Helpers
|
||||
// ============================================
|
||||
|
||||
export function getUniqueYears(data) {
|
||||
const years = [...new Set(data.map(r => r.year).filter(Boolean))];
|
||||
return years.sort((a, b) => parseInt(a) - parseInt(b));
|
||||
@@ -320,7 +365,6 @@ export function getDistrictMuseumMap(data) {
|
||||
if (!map[row.district]) map[row.district] = new Set();
|
||||
map[row.district].add(row.museum_name);
|
||||
});
|
||||
// Convert sets to sorted arrays
|
||||
Object.keys(map).forEach(d => {
|
||||
map[d] = [...map[d]].sort();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
// ============================================
|
||||
// ARCHIVED - Google Sheets Data Fetching
|
||||
// ============================================
|
||||
// Kept for reference only - NOT used in the app
|
||||
// The app now uses NocoDB exclusively with offline caching
|
||||
|
||||
const SPREADSHEET_ID = process.env.REACT_APP_SHEETS_ID || '';
|
||||
const SHEET_NAME = process.env.REACT_APP_SHEETS_NAME || 'Consolidated Data';
|
||||
const SHEET_URL = SPREADSHEET_ID
|
||||
? `https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/gviz/tq?tqx=out:csv&sheet=${encodeURIComponent(SHEET_NAME)}`
|
||||
: '';
|
||||
|
||||
// Convert Excel serial date to YYYY-MM-DD
|
||||
function excelDateToYMD(serial) {
|
||||
const num = parseInt(serial);
|
||||
if (isNaN(num) || num < 1) return null;
|
||||
|
||||
const utcDays = Math.floor(num - 25569);
|
||||
const date = new Date(utcDays * 86400 * 1000);
|
||||
|
||||
const y = date.getUTCFullYear();
|
||||
const m = String(date.getUTCMonth() + 1).padStart(2, '0');
|
||||
const d = String(date.getUTCDate()).padStart(2, '0');
|
||||
|
||||
return `${y}-${m}-${d}`;
|
||||
}
|
||||
|
||||
function parseCSV(text) {
|
||||
const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
||||
const lines = normalizedText.trim().split('\n');
|
||||
const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
|
||||
|
||||
return lines.slice(1).map(line => {
|
||||
const values = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let char of line) {
|
||||
if (char === '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
values.push(current.trim().replace(/^"|"$/g, ''));
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
values.push(current.trim().replace(/^"|"$/g, ''));
|
||||
|
||||
const obj = {};
|
||||
headers.forEach((header, i) => {
|
||||
let val = values[i] || '';
|
||||
if (header === 'date' && /^\d+$/.test(val)) {
|
||||
val = excelDateToYMD(val);
|
||||
}
|
||||
obj[header] = val;
|
||||
});
|
||||
return obj;
|
||||
}).filter(row => row.date);
|
||||
}
|
||||
|
||||
export async function fetchSheetData() {
|
||||
try {
|
||||
console.log('Fetching from Google Sheets...');
|
||||
const response = await fetch(SHEET_URL);
|
||||
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const text = await response.text();
|
||||
if (text.includes('<!DOCTYPE') || text.includes('<html')) {
|
||||
throw new Error('Sheet is not public');
|
||||
}
|
||||
|
||||
const data = parseCSV(text);
|
||||
console.log(`Loaded ${data.length} rows from Google Sheets`);
|
||||
return data;
|
||||
} catch (err) {
|
||||
console.error('Fetch error:', err);
|
||||
throw new Error(`Failed to load data: ${err.message}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user