feat: VAT toggle + offline mode

- Rename Revenue to GrossRevenue, add NetRevenue (excl. VAT)
- Add VAT toggle (Incl/Excl) on Dashboard and Comparison pages
- Add offline mode with localStorage caching (24h validity)
- Add refresh button and offline indicator in nav
- Remove Google Sheets fallback (archived to dataService.legacy.js)
- Add AR/EN translations for new UI elements
This commit is contained in:
fahed
2026-02-04 11:47:42 +03:00
parent afe276541f
commit 9044ab7da3
8 changed files with 477 additions and 280 deletions
+191 -147
View File
@@ -1,101 +1,96 @@
// Data source configuration - all from environment variables
// Set these in .env.local (never commit .env.local to git)
// Data source: NocoDB only
// Offline mode: caches data to localStorage for resilience
// NocoDB (primary/default)
const NOCODB_URL = process.env.REACT_APP_NOCODB_URL || '';
const NOCODB_TOKEN = process.env.REACT_APP_NOCODB_TOKEN || '';
// Google Sheets (fallback)
const SPREADSHEET_ID = process.env.REACT_APP_SHEETS_ID || '';
const SHEET_NAME = process.env.REACT_APP_SHEETS_NAME || 'Consolidated Data';
const SHEET_URL = SPREADSHEET_ID
? `https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/gviz/tq?tqx=out:csv&sheet=${encodeURIComponent(SHEET_NAME)}`
: '';
// Table IDs (not sensitive - just identifiers)
// Table IDs
const NOCODB_TABLES = {
districts: 'm8cup7lesbet0sa',
museums: 'm1c7od7mdirffvu',
dailyStats: 'mc7qhbdh3mjjwl8'
};
// Cache keys
const CACHE_KEY = 'hihala_data_cache';
const CACHE_TIMESTAMP_KEY = 'hihala_data_cache_timestamp';
const CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
export const umrahData = {
2024: { 1: 11574494, 2: 10521465, 3: 3364627, 4: 7435625 },
2025: { 1: 15222497, 2: 5443393, 3: null, 4: null }
};
// Convert Excel serial date to YYYY-MM-DD
function excelDateToYMD(serial) {
const num = parseInt(serial);
if (isNaN(num) || num < 1) return null;
// Excel epoch is Dec 30, 1899
const utcDays = Math.floor(num - 25569); // 25569 = days from 1899-12-30 to 1970-01-01
const date = new Date(utcDays * 86400 * 1000);
const y = date.getUTCFullYear();
const m = String(date.getUTCMonth() + 1).padStart(2, '0');
const d = String(date.getUTCDate()).padStart(2, '0');
return `${y}-${m}-${d}`;
}
// ============================================
// Offline Cache Functions
// ============================================
function parseCSV(text) {
const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
const lines = normalizedText.trim().split('\n');
const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
return lines.slice(1).map(line => {
const values = [];
let current = '';
let inQuotes = false;
for (let char of line) {
if (char === '"') {
inQuotes = !inQuotes;
} else if (char === ',' && !inQuotes) {
values.push(current.trim().replace(/^"|"$/g, ''));
current = '';
} else {
current += char;
}
}
values.push(current.trim().replace(/^"|"$/g, ''));
const obj = {};
headers.forEach((header, i) => {
let val = values[i] || '';
// Convert date serial to YYYY-MM-DD
if (header === 'date' && /^\d+$/.test(val)) {
val = excelDateToYMD(val);
}
obj[header] = val;
});
return obj;
}).filter(row => row.date);
}
export async function fetchSheetData() {
function saveToCache(data) {
try {
console.log('Fetching from Google Sheets...');
const response = await fetch(SHEET_URL);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const text = await response.text();
if (text.includes('<!DOCTYPE') || text.includes('<html')) {
throw new Error('Sheet is not public');
}
const data = parseCSV(text);
console.log(`Loaded ${data.length} rows from Google Sheets`);
return data;
localStorage.setItem(CACHE_KEY, JSON.stringify(data));
localStorage.setItem(CACHE_TIMESTAMP_KEY, Date.now().toString());
console.log(`Cached ${data.length} rows to localStorage`);
} catch (err) {
console.error('Fetch error:', err);
throw new Error(`Failed to load data: ${err.message}`);
console.warn('Failed to save to cache:', err.message);
}
}
function loadFromCache() {
try {
const cached = localStorage.getItem(CACHE_KEY);
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
if (!cached) return null;
const data = JSON.parse(cached);
const age = timestamp ? Date.now() - parseInt(timestamp) : Infinity;
const isStale = age > CACHE_MAX_AGE_MS;
console.log(`Loaded ${data.length} rows from cache (age: ${Math.round(age / 1000 / 60)} min, stale: ${isStale})`);
return { data, isStale, timestamp: parseInt(timestamp) };
} catch (err) {
console.warn('Failed to load from cache:', err.message);
return null;
}
}
function getCacheAge() {
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
if (!timestamp) return null;
return Date.now() - parseInt(timestamp);
}
export function getCacheStatus() {
const timestamp = localStorage.getItem(CACHE_TIMESTAMP_KEY);
const cached = localStorage.getItem(CACHE_KEY);
if (!cached || !timestamp) {
return { available: false, timestamp: null, age: null, rows: 0 };
}
const ts = parseInt(timestamp);
const data = JSON.parse(cached);
return {
available: true,
timestamp: new Date(ts).toISOString(),
age: Date.now() - ts,
rows: data.length,
isStale: (Date.now() - ts) > CACHE_MAX_AGE_MS
};
}
export function clearCache() {
localStorage.removeItem(CACHE_KEY);
localStorage.removeItem(CACHE_TIMESTAMP_KEY);
console.log('Cache cleared');
}
// ============================================
// NocoDB Data Fetching
// ============================================
async function fetchNocoDBTable(tableId, limit = 1000) {
let allRecords = [];
let offset = 0;
@@ -119,78 +114,113 @@ async function fetchNocoDBTable(tableId, limit = 1000) {
return allRecords;
}
export async function fetchNocoDBData() {
async function fetchFromNocoDB() {
console.log('Fetching from NocoDB...');
// Fetch all three tables in parallel
const [districts, museums, dailyStats] = await Promise.all([
fetchNocoDBTable(NOCODB_TABLES.districts),
fetchNocoDBTable(NOCODB_TABLES.museums),
fetchNocoDBTable(NOCODB_TABLES.dailyStats)
]);
// Build lookup maps
const districtMap = {};
districts.forEach(d => { districtMap[d.Id] = d.Name; });
const museumMap = {};
museums.forEach(m => {
museumMap[m.Id] = {
code: m.Code,
name: m.Name,
district: districtMap[m['nc_epk____Districts_id']] || 'Unknown'
};
});
// Join data into flat structure
const data = dailyStats.map(row => {
const museum = museumMap[row['nc_epk____Museums_id']] || {};
const date = row.Date;
const year = date ? date.substring(0, 4) : '';
const month = date ? parseInt(date.substring(5, 7)) : 0;
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
// GrossRevenue = including VAT, NetRevenue = excluding VAT
const grossRevenue = row.GrossRevenue || 0;
const netRevenue = row.NetRevenue || (grossRevenue / 1.15);
return {
date: date,
museum_code: museum.code,
museum_name: museum.name,
district: museum.district,
visits: row.Visits,
tickets: row.Tickets,
revenue_gross: grossRevenue,
revenue_net: netRevenue,
revenue_incl_tax: grossRevenue, // Legacy compatibility
year: year,
quarter: quarter
};
}).filter(r => r.date && r.museum_name);
console.log(`Loaded ${data.length} rows from NocoDB`);
return data;
}
// ============================================
// Main Data Fetcher (with offline fallback)
// ============================================
export async function fetchData() {
// Check if NocoDB is configured
if (!NOCODB_URL || !NOCODB_TOKEN) {
// Try cache
const cached = loadFromCache();
if (cached) {
console.warn('NocoDB not configured, using cached data');
return cached.data;
}
throw new Error('NocoDB not configured and no cached data available. Set REACT_APP_NOCODB_URL and REACT_APP_NOCODB_TOKEN in .env.local');
}
try {
console.log('Fetching from NocoDB (normalized)...');
// Try to fetch fresh data
const data = await fetchFromNocoDB();
// Fetch all three tables in parallel
const [districts, museums, dailyStats] = await Promise.all([
fetchNocoDBTable(NOCODB_TABLES.districts),
fetchNocoDBTable(NOCODB_TABLES.museums),
fetchNocoDBTable(NOCODB_TABLES.dailyStats)
]);
// Save to cache on success
saveToCache(data);
// Build lookup maps
const districtMap = {};
districts.forEach(d => { districtMap[d.Id] = d.Name; });
const museumMap = {};
museums.forEach(m => {
museumMap[m.Id] = {
code: m.Code,
name: m.Name,
district: districtMap[m['nc_epk____Districts_id']] || 'Unknown'
};
});
// Join data into flat structure for dashboard
const data = dailyStats.map(row => {
const museum = museumMap[row['nc_epk____Museums_id']] || {};
const date = row.Date;
const year = date ? date.substring(0, 4) : '';
const month = date ? parseInt(date.substring(5, 7)) : 0;
const quarter = month <= 3 ? '1' : month <= 6 ? '2' : month <= 9 ? '3' : '4';
return {
date: date,
museum_code: museum.code,
museum_name: museum.name,
district: museum.district,
visits: row.Visits,
tickets: row.Tickets,
revenue_incl_tax: row.Revenue,
year: year,
quarter: quarter
};
}).filter(r => r.date && r.museum_name);
console.log(`Loaded ${data.length} rows from NocoDB (joined from ${districts.length} districts, ${museums.length} museums, ${dailyStats.length} stats)`);
return data;
} catch (err) {
console.error('NocoDB fetch error:', err);
throw new Error(`Failed to load from NocoDB: ${err.message}`);
console.error('NocoDB fetch failed:', err.message);
// Try to load from cache
const cached = loadFromCache();
if (cached) {
console.warn(`Using cached data from ${new Date(cached.timestamp).toLocaleString()} (offline mode)`);
return cached.data;
}
throw new Error(`Database unavailable and no cached data: ${err.message}`);
}
}
// Main data fetcher - tries NocoDB first, falls back to Sheets
export async function fetchData() {
// Try NocoDB if configured
if (NOCODB_URL && NOCODB_TOKEN) {
try {
return await fetchNocoDBData();
} catch (err) {
console.warn('NocoDB failed, trying Google Sheets fallback...', err.message);
}
// Force refresh (bypass cache read, but still write to cache)
export async function refreshData() {
if (!NOCODB_URL || !NOCODB_TOKEN) {
throw new Error('NocoDB not configured');
}
// Fallback to Google Sheets if configured
if (SHEET_URL) {
return await fetchSheetData();
}
throw new Error('No data source configured. Set REACT_APP_NOCODB_URL + REACT_APP_NOCODB_TOKEN, or REACT_APP_SHEETS_ID in .env.local');
const data = await fetchFromNocoDB();
saveToCache(data);
return data;
}
// ============================================
// Data Filtering & Metrics
// ============================================
export function filterData(data, filters) {
return data.filter(row => {
if (filters.year && filters.year !== 'all' && row.year !== filters.year) return false;
@@ -211,14 +241,19 @@ export function filterDataByDateRange(data, startDate, endDate, filters = {}) {
});
}
export function calculateMetrics(data) {
const revenue = data.reduce((sum, row) => sum + parseFloat(row.revenue_incl_tax || 0), 0);
export function calculateMetrics(data, includeVAT = true) {
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
const revenue = data.reduce((sum, row) => sum + parseFloat(row[revenueField] || row.revenue_incl_tax || 0), 0);
const visitors = data.reduce((sum, row) => sum + parseInt(row.visits || 0), 0);
const tickets = data.reduce((sum, row) => sum + parseInt(row.tickets || 0), 0);
const avgRevPerVisitor = visitors > 0 ? revenue / visitors : 0;
return { revenue, visitors, tickets, avgRevPerVisitor };
}
// ============================================
// Formatting Functions
// ============================================
export function formatCurrency(num) {
if (isNaN(num)) return 'SAR 0';
return new Intl.NumberFormat('en-US', {
@@ -249,6 +284,10 @@ export function formatCompactCurrency(num) {
return formatCurrency(num);
}
// ============================================
// Grouping Functions
// ============================================
export function getWeekStart(dateStr) {
if (!dateStr || !dateStr.match(/^\d{4}-\d{2}-\d{2}$/)) return null;
@@ -265,45 +304,51 @@ export function getWeekStart(dateStr) {
return `${y}-${m}-${d}`;
}
export function groupByWeek(data) {
export function groupByWeek(data, includeVAT = true) {
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
const grouped = {};
data.forEach(row => {
if (!row.date) return;
const weekStart = getWeekStart(row.date);
if (!weekStart) return;
if (!grouped[weekStart]) grouped[weekStart] = { revenue: 0, visitors: 0, tickets: 0 };
grouped[weekStart].revenue += parseFloat(row.revenue_incl_tax || 0);
grouped[weekStart].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
grouped[weekStart].visitors += parseInt(row.visits || 0);
grouped[weekStart].tickets += parseInt(row.tickets || 0);
});
return grouped;
}
export function groupByMuseum(data) {
export function groupByMuseum(data, includeVAT = true) {
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
const grouped = {};
data.forEach(row => {
if (!row.museum_name) return;
if (!grouped[row.museum_name]) grouped[row.museum_name] = { revenue: 0, visitors: 0, tickets: 0 };
grouped[row.museum_name].revenue += parseFloat(row.revenue_incl_tax || 0);
grouped[row.museum_name].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
grouped[row.museum_name].visitors += parseInt(row.visits || 0);
grouped[row.museum_name].tickets += parseInt(row.tickets || 0);
});
return grouped;
}
export function groupByDistrict(data) {
export function groupByDistrict(data, includeVAT = true) {
const revenueField = includeVAT ? 'revenue_gross' : 'revenue_net';
const grouped = {};
data.forEach(row => {
if (!row.district) return;
if (!grouped[row.district]) grouped[row.district] = { revenue: 0, visitors: 0, tickets: 0 };
grouped[row.district].revenue += parseFloat(row.revenue_incl_tax || 0);
grouped[row.district].revenue += parseFloat(row[revenueField] || row.revenue_incl_tax || 0);
grouped[row.district].visitors += parseInt(row.visits || 0);
grouped[row.district].tickets += parseInt(row.tickets || 0);
});
return grouped;
}
// Dynamic data extraction helpers
// ============================================
// Data Extraction Helpers
// ============================================
export function getUniqueYears(data) {
const years = [...new Set(data.map(r => r.year).filter(Boolean))];
return years.sort((a, b) => parseInt(a) - parseInt(b));
@@ -320,7 +365,6 @@ export function getDistrictMuseumMap(data) {
if (!map[row.district]) map[row.district] = new Set();
map[row.district].add(row.museum_name);
});
// Convert sets to sorted arrays
Object.keys(map).forEach(d => {
map[d] = [...map[d]].sort();
});
+81
View File
@@ -0,0 +1,81 @@
// ============================================
// ARCHIVED - Google Sheets Data Fetching
// ============================================
// Kept for reference only - NOT used in the app
// The app now uses NocoDB exclusively with offline caching
const SPREADSHEET_ID = process.env.REACT_APP_SHEETS_ID || '';
const SHEET_NAME = process.env.REACT_APP_SHEETS_NAME || 'Consolidated Data';
const SHEET_URL = SPREADSHEET_ID
? `https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/gviz/tq?tqx=out:csv&sheet=${encodeURIComponent(SHEET_NAME)}`
: '';
// Convert Excel serial date to YYYY-MM-DD
function excelDateToYMD(serial) {
const num = parseInt(serial);
if (isNaN(num) || num < 1) return null;
const utcDays = Math.floor(num - 25569);
const date = new Date(utcDays * 86400 * 1000);
const y = date.getUTCFullYear();
const m = String(date.getUTCMonth() + 1).padStart(2, '0');
const d = String(date.getUTCDate()).padStart(2, '0');
return `${y}-${m}-${d}`;
}
function parseCSV(text) {
const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
const lines = normalizedText.trim().split('\n');
const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
return lines.slice(1).map(line => {
const values = [];
let current = '';
let inQuotes = false;
for (let char of line) {
if (char === '"') {
inQuotes = !inQuotes;
} else if (char === ',' && !inQuotes) {
values.push(current.trim().replace(/^"|"$/g, ''));
current = '';
} else {
current += char;
}
}
values.push(current.trim().replace(/^"|"$/g, ''));
const obj = {};
headers.forEach((header, i) => {
let val = values[i] || '';
if (header === 'date' && /^\d+$/.test(val)) {
val = excelDateToYMD(val);
}
obj[header] = val;
});
return obj;
}).filter(row => row.date);
}
export async function fetchSheetData() {
try {
console.log('Fetching from Google Sheets...');
const response = await fetch(SHEET_URL);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const text = await response.text();
if (text.includes('<!DOCTYPE') || text.includes('<html')) {
throw new Error('Sheet is not public');
}
const data = parseCSV(text);
console.log(`Loaded ${data.length} rows from Google Sheets`);
return data;
} catch (err) {
console.error('Fetch error:', err);
throw new Error(`Failed to load data: ${err.message}`);
}
}