feat: BI-CCC evolution — 6-phase platform upgrade (45→85 maturity)
Phase 1: Refactor queries.js (1787 lines) into domain modules with facade pattern
- src/queries/{helpers,payin,payout,corporate,bi,client,provider,compliance}.queries.js
- New provider performance + compliance data layer queries
- Health check endpoint (GET /health)
Phase 2: Provider Performance Dashboard (src/admin-providers.js)
- Hero cards, sortable tables, Chart.js charts, date range filter
- API routes: /admin/api/providers, /admin/api/providers/failed, /admin/api/providers/trend
Phase 3: Excel Export (exceljs)
- CambioReal-branded exports for BI, clients, providers, transactions
- Export buttons added to BI and Client 360 dashboards
Phase 4: Alert System (node-cron + nodemailer)
- 5 alert rules: volume spike, spread anomaly, large tx, failed tx spike, provider inactivity
- SQLite alerts table, bell icon UI with acknowledge workflow
- Email notifications via SMTP
Phase 5: Enhanced Analytics
- Churn prediction: weighted RFM model (src/services/churn-predictor.js)
- Volume forecasting: exponential smoothing with confidence bands (src/services/forecast.js)
- Forecast chart in BI dashboard, churn risk in Client 360
Phase 6: SQLite Analytics Store (ETL)
- src/db-analytics.js: daily_metrics, client_health_daily, monthly_revenue tables
- src/etl/daily-sync.js: MySQL RDS → SQLite daily sync at 1 AM + 90-day backfill
- src/etl/data-quality.js: post-sync validation (row counts, reconciliation)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
root
377
server.js
377
server.js
@@ -11,13 +11,20 @@ const express = require('express');
|
||||
const session = require('express-session');
|
||||
const path = require('path');
|
||||
const { authenticate, requireAuth, requireRole, createAgente, createUser } = require('./src/auth');
|
||||
const { fetchTransacoes, fetchAllTransacoes, serialize, fetchDailyStats, fetchKPIs, fetchTrend30Days, fetchTopAgentes, fetchTrendByPeriod, fetchKPIsByPeriod, fetchBIData, fetchRevenueAnalytics, fetchBIStrategic, fetchTopClients, fetchClientSearch, fetchClientProfile, fetchClientData, fetchMerchantProfile, fetchMerchantData } = require('./src/queries');
|
||||
const { fetchTransacoes, fetchAllTransacoes, serialize, fetchDailyStats, fetchKPIs, fetchTrend30Days, fetchTopAgentes, fetchTrendByPeriod, fetchKPIsByPeriod, fetchBIData, fetchRevenueAnalytics, fetchBIStrategic, fetchTopClients, fetchClientSearch, fetchClientProfile, fetchClientData, fetchMerchantProfile, fetchMerchantData, fetchProviderPerformance, fetchFailedTransactions, fetchProviderTrend } = require('./src/queries');
|
||||
const { buildAdminProvidersHTML } = require('./src/admin-providers');
|
||||
const pool = require('./src/db-rds');
|
||||
const { buildHTML } = require('./src/dashboard');
|
||||
const { buildAdminHTML } = require('./src/admin-panel');
|
||||
const { buildAdminHomeHTML } = require('./src/admin-home');
|
||||
const { buildAdminDashboardHTML } = require('./src/admin-dashboard');
|
||||
const { buildAdminBIHTML } = require('./src/admin-bi');
|
||||
const { buildAdminClienteHTML } = require('./src/admin-cliente');
|
||||
const { exportToExcel, createWorkbook, sendWorkbook } = require('./src/export/excel-export');
|
||||
const { startAlertEngine, getAlerts, acknowledgeAlert, getAlertHistory, getUnackedCount } = require('./src/alerts/alert-engine');
|
||||
const { predictChurnRisk } = require('./src/services/churn-predictor');
|
||||
const { forecastFromTrend } = require('./src/services/forecast');
|
||||
const { startETL } = require('./src/etl/daily-sync');
|
||||
const bcrypt = require('bcrypt');
|
||||
const db = require('./src/db-local');
|
||||
const cache = require('./src/cache');
|
||||
@@ -580,6 +587,368 @@ app.delete('/admin/agentes/:id', requireRole('admin'), (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// --- Excel Export Endpoints ---
|
||||
|
||||
app.get('/admin/api/export/bi-excel', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
if (!start || !end) return res.status(400).json({ error: 'start and end required' });
|
||||
const getAgenteName = (agenteId) => {
|
||||
const row = db.prepare('SELECT nome FROM agentes WHERE agente_id = ?').get(agenteId);
|
||||
return row ? row.nome : null;
|
||||
};
|
||||
const data = await fetchBIData(start, end, getAgenteName);
|
||||
|
||||
// Build multi-sheet workbook
|
||||
const ExcelJS = require('exceljs');
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = 'CambioReal BI-CCC';
|
||||
|
||||
// Sheet 1: KPI Summary
|
||||
const kpiSheet = workbook.addWorksheet('KPI Summary');
|
||||
kpiSheet.columns = [
|
||||
{ header: 'Metric', key: 'metric', width: 30 },
|
||||
{ header: 'Current', key: 'current', width: 18 },
|
||||
{ header: 'Previous', key: 'previous', width: 18 },
|
||||
{ header: 'Change %', key: 'change', width: 14 }
|
||||
];
|
||||
const k = data.kpis.total;
|
||||
const c = data.comparison;
|
||||
const pctChg = (curr, prev) => prev > 0 ? Math.round((curr - prev) / prev * 100) : 0;
|
||||
kpiSheet.addRows([
|
||||
{ metric: 'Total Transactions', current: k.qtd, previous: c.prev_qtd, change: pctChg(k.qtd, c.prev_qtd) },
|
||||
{ metric: 'Total Volume USD', current: k.vol_usd, previous: c.prev_vol_usd, change: pctChg(k.vol_usd, c.prev_vol_usd) },
|
||||
{ metric: 'Spread Revenue', current: k.spread_revenue, previous: c.prev_spread, change: pctChg(k.spread_revenue, c.prev_spread) },
|
||||
{ metric: 'Active Clients', current: k.clientes, previous: '-', change: '-' },
|
||||
{ metric: 'Avg Ticket', current: k.ticket_medio, previous: '-', change: '-' },
|
||||
{ metric: 'Retention Rate %', current: data.retention.rate, previous: '-', change: '-' }
|
||||
]);
|
||||
|
||||
// Sheet 2: Top Clients
|
||||
const clientSheet = workbook.addWorksheet('Top Clients');
|
||||
clientSheet.columns = [
|
||||
{ header: 'Client', key: 'nome', width: 30 },
|
||||
{ header: 'Volume USD', key: 'vol_usd', width: 18 },
|
||||
{ header: 'Transactions', key: 'qtd', width: 14 }
|
||||
];
|
||||
data.topClients.forEach(c => clientSheet.addRow(c));
|
||||
|
||||
// Sheet 3: Agent Ranking
|
||||
const agentSheet = workbook.addWorksheet('Agent Ranking');
|
||||
agentSheet.columns = [
|
||||
{ header: 'Rank', key: 'rank', width: 8 },
|
||||
{ header: 'Agent', key: 'nome', width: 25 },
|
||||
{ header: 'Volume USD', key: 'vol_usd', width: 18 },
|
||||
{ header: 'Transactions', key: 'qtd', width: 14 },
|
||||
{ header: 'Spread Revenue', key: 'spread_revenue', width: 18 },
|
||||
{ header: 'Clients', key: 'clientes', width: 12 }
|
||||
];
|
||||
data.agentRanking.forEach(a => agentSheet.addRow(a));
|
||||
|
||||
// Sheet 4: Clients at Risk
|
||||
const riskSheet = workbook.addWorksheet('Clients at Risk');
|
||||
riskSheet.columns = [
|
||||
{ header: 'Client', key: 'nome', width: 30 },
|
||||
{ header: 'Volume USD', key: 'vol_usd', width: 18 },
|
||||
{ header: 'Transactions', key: 'qtd', width: 14 },
|
||||
{ header: 'Last Activity', key: 'last_op', width: 18 },
|
||||
{ header: 'Days Inactive', key: 'days_inactive', width: 14 }
|
||||
];
|
||||
data.clientsAtRisk.forEach(c => riskSheet.addRow(c));
|
||||
|
||||
// Style all sheet headers
|
||||
workbook.eachSheet(sheet => {
|
||||
const hr = sheet.getRow(1);
|
||||
hr.eachCell(cell => {
|
||||
cell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FF7600BE' } };
|
||||
cell.font = { bold: true, color: { argb: 'FFFFFFFF' }, size: 11 };
|
||||
cell.alignment = { vertical: 'middle', horizontal: 'center' };
|
||||
});
|
||||
hr.height = 28;
|
||||
sheet.views = [{ state: 'frozen', ySplit: 1 }];
|
||||
});
|
||||
|
||||
await sendWorkbook(res, workbook, `BI_Executive_${start}_${end}`);
|
||||
} catch (err) {
|
||||
console.error('BI Export error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/export/clients-excel', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const clients = await cache.getOrFetch('top-clients', fetchTopClients, 15 * 60 * 1000);
|
||||
await exportToExcel(res, clients, [
|
||||
{ header: 'Client', key: 'nome', width: 30 },
|
||||
{ header: 'Volume USD', key: 'vol', width: 18, type: 'currency' },
|
||||
{ header: 'Operations', key: 'ops', width: 14, type: 'number' },
|
||||
{ header: 'Months Active', key: 'months', width: 14, type: 'number' },
|
||||
{ header: 'Last Activity', key: 'lastOp', width: 16, type: 'date' }
|
||||
], 'Top Clients', `Top_Clients_${new Date().toISOString().slice(0, 10)}`);
|
||||
} catch (err) {
|
||||
console.error('Clients Export error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/export/providers-excel', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
if (!start || !end) return res.status(400).json({ error: 'start and end required' });
|
||||
const data = await fetchProviderPerformance(start, end);
|
||||
await exportToExcel(res, data.providers, [
|
||||
{ header: 'Provider', key: 'provider', width: 18 },
|
||||
{ header: 'Flow', key: 'flow', width: 12 },
|
||||
{ header: 'Total Tx', key: 'total_tx', width: 12, type: 'number' },
|
||||
{ header: 'Success Tx', key: 'success_tx', width: 12, type: 'number' },
|
||||
{ header: 'Success Rate %', key: 'success_rate', width: 14, type: 'percentage' },
|
||||
{ header: 'Volume USD', key: 'vol_usd', width: 18, type: 'currency' },
|
||||
{ header: 'Avg Ticket', key: 'avg_ticket', width: 14, type: 'currency' },
|
||||
{ header: 'Spread %', key: 'avg_spread_pct', width: 12, type: 'percentage' },
|
||||
{ header: 'Settlement Hours', key: 'avg_settlement_hours', width: 16, type: 'number' }
|
||||
], 'Providers', `Providers_${start}_${end}`);
|
||||
} catch (err) {
|
||||
console.error('Providers Export error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/export/transactions-excel', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
const dias = start && end ? null : 90;
|
||||
let data;
|
||||
if (start && end) {
|
||||
// Use fetchBIData for date-filtered transactions
|
||||
const biData = await fetchBIData(start, end);
|
||||
// Combine trend data into flat rows
|
||||
const allTrend = [];
|
||||
['brlUsd', 'usdBrl', 'usdUsd'].forEach(flow => {
|
||||
(biData.trend[flow] || []).forEach(r => {
|
||||
allTrend.push({ date: r.dia, flow, transactions: r.qtd, volume_usd: r.vol_usd, avg_spread: r.avg_spread || 0 });
|
||||
});
|
||||
});
|
||||
data = allTrend;
|
||||
} else {
|
||||
const raw = await fetchAllTransacoes(dias || 90);
|
||||
data = serialize(raw.rowsBrlUsd, raw.rowsUsdBrl);
|
||||
}
|
||||
|
||||
await exportToExcel(res, data, data.length > 0 && data[0].fluxo ? [
|
||||
{ header: 'Flow', key: 'fluxo', width: 12 },
|
||||
{ header: 'Client', key: 'cliente', width: 28 },
|
||||
{ header: 'Date', key: 'data_operacao', width: 18 },
|
||||
{ header: 'BRL', key: 'valor_reais', width: 14, type: 'currency' },
|
||||
{ header: 'USD', key: 'valor_dolar', width: 14, type: 'currency' },
|
||||
{ header: 'PTAX', key: 'taxa_ptax', width: 12 },
|
||||
{ header: 'Rate', key: 'taxa_cobrada', width: 12 },
|
||||
{ header: 'Spread', key: 'spread_bruto', width: 12 },
|
||||
{ header: 'Spread %', key: 'spread_pct', width: 10, type: 'percentage' },
|
||||
{ header: 'IOF %', key: 'iof_pct', width: 8 },
|
||||
{ header: 'Status', key: 'status', width: 14 }
|
||||
] : [
|
||||
{ header: 'Date', key: 'date', width: 14 },
|
||||
{ header: 'Flow', key: 'flow', width: 12 },
|
||||
{ header: 'Transactions', key: 'transactions', width: 14, type: 'number' },
|
||||
{ header: 'Volume USD', key: 'volume_usd', width: 16, type: 'currency' },
|
||||
{ header: 'Avg Spread %', key: 'avg_spread', width: 14, type: 'percentage' }
|
||||
], 'Transactions', `Transactions_${start || 'last90d'}_${end || 'today'}`);
|
||||
} catch (err) {
|
||||
console.error('Transactions Export error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Forecast API ---
|
||||
app.get('/admin/api/bi/forecast', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const metric = req.query.metric || 'volume';
|
||||
const days = parseInt(req.query.days) || 30;
|
||||
// Get last 90 days of trend data for forecasting
|
||||
const now = new Date();
|
||||
const start = new Date(now.getTime() - 90 * 86400000).toISOString().slice(0, 10);
|
||||
const end = now.toISOString().slice(0, 10);
|
||||
const biData = await fetchBIData(start, end);
|
||||
|
||||
// Combine all flows into daily totals
|
||||
const dailyMap = {};
|
||||
['brlUsd', 'usdBrl', 'usdUsd'].forEach(flow => {
|
||||
(biData.trend[flow] || []).forEach(d => {
|
||||
if (!dailyMap[d.dia]) dailyMap[d.dia] = { dia: d.dia, vol_usd: 0, qtd: 0 };
|
||||
dailyMap[d.dia].vol_usd += d.vol_usd;
|
||||
dailyMap[d.dia].qtd += d.qtd;
|
||||
});
|
||||
});
|
||||
const trendData = Object.values(dailyMap).sort((a, b) => a.dia.localeCompare(b.dia));
|
||||
|
||||
const metricKey = metric === 'transactions' ? 'qtd' : 'vol_usd';
|
||||
const result = forecastFromTrend(trendData, metricKey, days);
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
console.error('Forecast API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Churn Risk API ---
|
||||
app.get('/admin/api/cliente/:id/churn', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const clienteId = parseInt(req.params.id);
|
||||
if (!clienteId) return res.status(400).json({ error: 'Invalid client ID' });
|
||||
|
||||
// Get profile and recent data
|
||||
const profile = await fetchClientProfile(clienteId);
|
||||
const now = new Date();
|
||||
const end = now.toISOString().slice(0, 10);
|
||||
const start30 = new Date(now.getTime() - 30 * 86400000).toISOString().slice(0, 10);
|
||||
const start60 = new Date(now.getTime() - 60 * 86400000).toISOString().slice(0, 10);
|
||||
|
||||
const clientData = await fetchClientData(clienteId, start60, end);
|
||||
|
||||
// Count operations in current vs previous 30-day windows
|
||||
const currOps = (clientData.trend.brlUsd || [])
|
||||
.filter(d => d.dia >= start30)
|
||||
.reduce((s, d) => s + d.qtd, 0) +
|
||||
(clientData.trend.usdBrl || [])
|
||||
.filter(d => d.dia >= start30)
|
||||
.reduce((s, d) => s + d.qtd, 0);
|
||||
|
||||
const prevOps = (clientData.trend.brlUsd || [])
|
||||
.filter(d => d.dia < start30 && d.dia >= start60)
|
||||
.reduce((s, d) => s + d.qtd, 0) +
|
||||
(clientData.trend.usdBrl || [])
|
||||
.filter(d => d.dia < start30 && d.dia >= start60)
|
||||
.reduce((s, d) => s + d.qtd, 0);
|
||||
|
||||
// Determine product count
|
||||
let productCount = 0;
|
||||
if (profile.brlUsd && profile.brlUsd.qtd > 0) productCount++;
|
||||
if (profile.usdBrl && profile.usdBrl.qtd > 0) productCount++;
|
||||
|
||||
const churn = predictChurnRisk({
|
||||
days_inactive: profile.days_inactive,
|
||||
avg_monthly_ops: profile.avg_monthly_ops,
|
||||
avg_monthly_vol: profile.avg_monthly_vol,
|
||||
months_active: profile.months_active,
|
||||
curr_ops: currOps,
|
||||
prev_ops: prevOps,
|
||||
product_count: productCount
|
||||
});
|
||||
|
||||
res.json(churn);
|
||||
} catch (err) {
|
||||
console.error('Churn API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Alert API Endpoints ---
|
||||
|
||||
app.get('/admin/api/alerts', requireRole('admin'), (req, res) => {
|
||||
try {
|
||||
const unacked = req.query.unacked === '1';
|
||||
const alerts = getAlerts(24, unacked);
|
||||
res.json({ alerts, unacked_count: getUnackedCount() });
|
||||
} catch (err) {
|
||||
console.error('Alerts API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.put('/admin/api/alerts/:id/ack', requireRole('admin'), (req, res) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id);
|
||||
acknowledgeAlert(id);
|
||||
res.json({ success: true });
|
||||
} catch (err) {
|
||||
console.error('Alert ack error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/alerts/history', requireRole('admin'), (req, res) => {
|
||||
try {
|
||||
const days = parseInt(req.query.days) || 7;
|
||||
const alerts = getAlertHistory(days);
|
||||
res.json({ alerts });
|
||||
} catch (err) {
|
||||
console.error('Alert history error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Health Check ---
|
||||
app.get('/health', async (req, res) => {
|
||||
const health = {
|
||||
status: 'ok',
|
||||
uptime: Math.round(process.uptime()),
|
||||
memory: {
|
||||
rss: Math.round(process.memoryUsage().rss / 1024 / 1024),
|
||||
heap: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)
|
||||
},
|
||||
cache: cache.stats(),
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
try {
|
||||
const conn = await pool.getConnection();
|
||||
await conn.execute('SELECT 1');
|
||||
conn.release();
|
||||
health.mysql = 'connected';
|
||||
} catch (err) {
|
||||
health.mysql = 'error: ' + err.message;
|
||||
health.status = 'degraded';
|
||||
}
|
||||
res.json(health);
|
||||
});
|
||||
|
||||
// --- Provider Dashboard (admin only) ---
|
||||
app.get('/admin/providers', requireRole('admin'), (req, res) => {
|
||||
try {
|
||||
res.set('Cache-Control', 'no-store, no-cache, must-revalidate');
|
||||
const html = buildAdminProvidersHTML(req.session.user);
|
||||
res.send(html);
|
||||
} catch (err) {
|
||||
console.error('Admin Providers error:', err);
|
||||
res.status(500).send('Erro ao carregar providers: ' + err.message);
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/providers', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
if (!start || !end) return res.status(400).json({ error: 'start and end required' });
|
||||
const data = await fetchProviderPerformance(start, end);
|
||||
res.json(data);
|
||||
} catch (err) {
|
||||
console.error('Provider API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/providers/failed', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
if (!start || !end) return res.status(400).json({ error: 'start and end required' });
|
||||
const data = await fetchFailedTransactions(start, end);
|
||||
res.json(data);
|
||||
} catch (err) {
|
||||
console.error('Failed TX API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/providers/trend', requireRole('admin'), async (req, res) => {
|
||||
try {
|
||||
const { start, end } = req.query;
|
||||
if (!start || !end) return res.status(400).json({ error: 'start and end required' });
|
||||
const data = await fetchProviderTrend(start, end);
|
||||
res.json(data);
|
||||
} catch (err) {
|
||||
console.error('Provider Trend API error:', err);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Start
|
||||
app.listen(PORT, () => {
|
||||
console.log(`BI - CCC rodando: http://localhost:${PORT}`);
|
||||
@@ -591,4 +960,10 @@ app.listen(PORT, () => {
|
||||
cache.registerAutoRefresh('top-agentes-30', () => fetchTopAgentes(30), 10 * 60 * 1000);
|
||||
cache.registerAutoRefresh('top-agentes-7', () => fetchTopAgentes(7), 10 * 60 * 1000);
|
||||
cache.registerAutoRefresh('top-agentes-90', () => fetchTopAgentes(90), 10 * 60 * 1000);
|
||||
|
||||
// Start alert engine
|
||||
startAlertEngine();
|
||||
|
||||
// Start ETL daily sync (MySQL RDS → SQLite analytics)
|
||||
startETL();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user