cassel/bi-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: parallelize BI queries to fix 504 timeout

Browse Source 
- fetchBIData: 12 sequential queries → 1 Promise.all (14 parallel)
- fetchBIStrategic: 6 sequential queries → 4 parallel sections
  (cohorts, expansion, cross-sell, maturity), each with internal
  parallelism via Promise.all
- Total response time ≈ slowest single query instead of sum of all

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
root
2026-02-16 23:01:44 -05:00
parent 6bfd21a111
commit ab7a886836
1 changed files with 323 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
src/queries/bi.queries.js
View File

@@ -10,10 +10,16 @@ async function fetchBIData(dataInicio, dataFim, getAgenteName = null) {
try {
const { prevStartStr, prevEndStr } = calcPrevPeriod(dataInicio, dataFim);
// Run all 12 queries in parallel
const [
[kpiBrlUsd], [kpiUsdBrl], [kpiUsdUsd], [uniqueClients],
[prevBrlUsd], [prevUsdBrl], [prevUsdUsd],
[trendBrlUsd], [trendUsdBrl], [trendUsdUsd],
[topClients], [retention], [clientsAtRisk], [agentRanking]
] = await Promise.all([
// 1. BRL→USD KPIs
const [kpiBrlUsd] = await conn.execute(`
SELECT
COUNT(*) as qtd,
conn.execute(`
SELECT COUNT(*) as qtd,
ROUND(COALESCE(SUM(amount_usd), 0), 2) as vol_usd,
ROUND(COALESCE(SUM(amount_brl), 0), 2) as vol_brl,
ROUND(COALESCE(SUM((exchange_rate - ptax) / exchange_rate * amount_usd), 0), 2) as spread_revenue,
@@ -21,12 +27,10 @@ async function fetchBIData(dataInicio, dataFim, getAgenteName = null) {
COUNT(DISTINCT id_conta) as clientes
FROM br_transaction_to_usa
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
`, [dataInicio, dataFim]);
`, [dataInicio, dataFim]),
// 2. USD→BRL KPIs
const [kpiUsdBrl] = await conn.execute(`
SELECT
COUNT(*) as qtd,
conn.execute(`
SELECT COUNT(*) as qtd,
ROUND(COALESCE(SUM(valor), 0), 2) as vol_usd,
ROUND(COALESCE(SUM(valor_sol), 0), 2) as vol_brl,
ROUND(COALESCE(SUM((ptax - cotacao) / ptax * valor), 0), 2) as spread_revenue,
@@ -36,158 +40,121 @@ async function fetchBIData(dataInicio, dataFim, getAgenteName = null) {
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND cotacao IS NOT NULL AND cotacao > 0
AND (pgto IS NULL OR pgto != 'balance')
`, [dataInicio, dataFim]);
`, [dataInicio, dataFim]),
// 3. USD→USD KPIs
const [kpiUsdUsd] = await conn.execute(`
SELECT
COUNT(*) as qtd,
conn.execute(`
SELECT COUNT(*) as qtd,
ROUND(COALESCE(SUM(valor), 0), 2) as vol_usd,
COUNT(DISTINCT id_conta) as clientes
FROM pagamento_br
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND (cotacao IS NULL OR cotacao = 0 OR pgto = 'balance')
`, [dataInicio, dataFim]);
// 4. Unique active clients across all flows
const [uniqueClients] = await conn.execute(`
`, [dataInicio, dataFim]),
// 4. Unique active clients
conn.execute(`
SELECT COUNT(DISTINCT id_conta) as total FROM (
SELECT id_conta FROM br_transaction_to_usa
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
SELECT id_conta FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
UNION
SELECT id_conta FROM pagamento_br
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
SELECT id_conta FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
) all_clients
`, [dataInicio, dataFim, dataInicio, dataFim]);
// 5. Previous period totals for comparison
const [prevBrlUsd] = await conn.execute(`
`, [dataInicio, dataFim, dataInicio, dataFim]),
// 5. Previous period
conn.execute(`
SELECT COUNT(*) as qtd, ROUND(COALESCE(SUM(amount_usd),0),2) as vol_usd,
ROUND(COALESCE(SUM((exchange_rate - ptax) / exchange_rate * amount_usd),0),2) as spread_revenue
FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
`, [prevStartStr, prevEndStr]);
const [prevUsdBrl] = await conn.execute(`
`, [prevStartStr, prevEndStr]),
conn.execute(`
SELECT COUNT(*) as qtd, ROUND(COALESCE(SUM(valor),0),2) as vol_usd,
ROUND(COALESCE(SUM((ptax - cotacao) / ptax * valor),0),2) as spread_revenue
FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
`, [prevStartStr, prevEndStr]);
const [prevUsdUsd] = await conn.execute(`
`, [prevStartStr, prevEndStr]),
conn.execute(`
SELECT COUNT(*) as qtd, ROUND(COALESCE(SUM(valor),0),2) as vol_usd
FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND (cotacao IS NULL OR cotacao = 0 OR pgto = 'balance')
`, [prevStartStr, prevEndStr]);
// 6. BRL→USD daily trend with spread
const [trendBrlUsd] = await conn.execute(`
`, [prevStartStr, prevEndStr]),
// 6-8. Trends
conn.execute(`
SELECT DATE(created_at) as dia, COUNT(*) as qtd,
ROUND(SUM(amount_usd), 2) as vol_usd,
ROUND(AVG((exchange_rate - ptax) / exchange_rate * 100), 2) as avg_spread
FROM br_transaction_to_usa
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
GROUP BY DATE(created_at) ORDER BY dia
`, [dataInicio, dataFim]);
// 7. USD→BRL daily trend with spread
const [trendUsdBrl] = await conn.execute(`
`, [dataInicio, dataFim]),
conn.execute(`
SELECT DATE(created_at) as dia, COUNT(*) as qtd,
ROUND(SUM(valor), 2) as vol_usd,
ROUND(AVG(CASE WHEN cotacao > 0 THEN (ptax - cotacao) / ptax * 100 ELSE 0 END), 2) as avg_spread
FROM pagamento_br
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND cotacao IS NOT NULL AND cotacao > 0
AND (pgto IS NULL OR pgto != 'balance')
FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
GROUP BY DATE(created_at) ORDER BY dia
`, [dataInicio, dataFim]);
// 8. USD→USD daily trend
const [trendUsdUsd] = await conn.execute(`
SELECT DATE(created_at) as dia, COUNT(*) as qtd,
ROUND(SUM(valor), 2) as vol_usd
FROM pagamento_br
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
`, [dataInicio, dataFim]),
conn.execute(`
SELECT DATE(created_at) as dia, COUNT(*) as qtd, ROUND(SUM(valor), 2) as vol_usd
FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
AND (cotacao IS NULL OR cotacao = 0 OR pgto = 'balance')
GROUP BY DATE(created_at) ORDER BY dia
`, [dataInicio, dataFim]);
// 9. Top 10 clients by volume
const [topClients] = await conn.execute(`
`, [dataInicio, dataFim]),
// 9. Top clients
conn.execute(`
SELECT nome, SUM(vol) as total_usd, SUM(qtd) as total_qtd FROM (
SELECT c.nome, SUM(t.amount_usd) as vol, COUNT(*) as qtd
FROM br_transaction_to_usa t
INNER JOIN conta c ON c.id_conta = t.id_conta
WHERE DATE(t.created_at) >= ? AND DATE(t.created_at) <= ?
GROUP BY c.nome
FROM br_transaction_to_usa t INNER JOIN conta c ON c.id_conta = t.id_conta
WHERE DATE(t.created_at) >= ? AND DATE(t.created_at) <= ? GROUP BY c.nome
UNION ALL
SELECT c.nome, SUM(p.valor) as vol, COUNT(*) as qtd
FROM pagamento_br p
INNER JOIN conta c ON c.id_conta = p.id_conta
WHERE DATE(p.created_at) >= ? AND DATE(p.created_at) <= ?
GROUP BY c.nome
) combined
GROUP BY nome ORDER BY total_usd DESC LIMIT 10
`, [dataInicio, dataFim, dataInicio, dataFim]);
// 10. Client retention
const [retention] = await conn.execute(`
SELECT
COUNT(DISTINCT prev.id_conta) as prev_clients,
FROM pagamento_br p INNER JOIN conta c ON c.id_conta = p.id_conta
WHERE DATE(p.created_at) >= ? AND DATE(p.created_at) <= ? GROUP BY c.nome
) combined GROUP BY nome ORDER BY total_usd DESC LIMIT 10
`, [dataInicio, dataFim, dataInicio, dataFim]),
// 10. Retention
conn.execute(`
SELECT COUNT(DISTINCT prev.id_conta) as prev_clients,
COUNT(DISTINCT CASE WHEN curr.id_conta IS NOT NULL THEN prev.id_conta END) as retained
FROM (
SELECT DISTINCT id_conta FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
UNION
SELECT DISTINCT id_conta FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
) prev
LEFT JOIN (
) prev LEFT JOIN (
SELECT DISTINCT id_conta FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
UNION
SELECT DISTINCT id_conta FROM pagamento_br WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
) curr ON prev.id_conta = curr.id_conta
`, [prevStartStr, prevEndStr, prevStartStr, prevEndStr, dataInicio, dataFim, dataInicio, dataFim]);
`, [prevStartStr, prevEndStr, prevStartStr, prevEndStr, dataInicio, dataFim, dataInicio, dataFim]),
// 11. Clients at risk
const [clientsAtRisk] = await conn.execute(`
conn.execute(`
SELECT nome, MAX(last_op) as last_op, SUM(vol) as total_usd, SUM(qtd) as total_qtd,
DATEDIFF(CURDATE(), MAX(last_op)) as days_inactive
FROM (
SELECT c.nome, MAX(t.created_at) as last_op, SUM(t.amount_usd) as vol, COUNT(*) as qtd
FROM br_transaction_to_usa t
INNER JOIN conta c ON c.id_conta = t.id_conta
GROUP BY c.nome
FROM br_transaction_to_usa t INNER JOIN conta c ON c.id_conta = t.id_conta GROUP BY c.nome
UNION ALL
SELECT c.nome, MAX(p.created_at) as last_op, SUM(p.valor) as vol, COUNT(*) as qtd
FROM pagamento_br p
INNER JOIN conta c ON c.id_conta = p.id_conta
GROUP BY c.nome
) combined
GROUP BY nome
HAVING MAX(last_op) < CURDATE()
ORDER BY total_usd DESC LIMIT 20
`);
// 12. Agent ranking with spread revenue
const [agentRanking] = await conn.execute(`
FROM pagamento_br p INNER JOIN conta c ON c.id_conta = p.id_conta GROUP BY c.nome
) combined GROUP BY nome HAVING MAX(last_op) < CURDATE() ORDER BY total_usd DESC LIMIT 20
`),
// 12. Agent ranking
conn.execute(`
SELECT agente_id, SUM(vol) as total_usd, SUM(qtd) as total_qtd,
ROUND(SUM(spread_rev), 2) as total_spread, COUNT(DISTINCT client_id) as clientes
FROM (
SELECT ac.agente_id, t.id_conta as client_id, SUM(t.amount_usd) as vol, COUNT(*) as qtd,
SUM((t.exchange_rate - t.ptax) / t.exchange_rate * t.amount_usd) as spread_rev
FROM br_transaction_to_usa t
INNER JOIN ag_contas ac ON ac.conta_id = t.id_conta
FROM br_transaction_to_usa t INNER JOIN ag_contas ac ON ac.conta_id = t.id_conta
WHERE DATE(t.created_at) >= ? AND DATE(t.created_at) <= ?
GROUP BY ac.agente_id, t.id_conta
UNION ALL
SELECT ac.agente_id, p.id_conta as client_id, SUM(p.valor) as vol, COUNT(*) as qtd,
SUM((p.ptax - p.cotacao) / p.ptax * p.valor) as spread_rev
FROM pagamento_br p
INNER JOIN ag_contas ac ON ac.conta_id = p.id_conta
FROM pagamento_br p INNER JOIN ag_contas ac ON ac.conta_id = p.id_conta
WHERE DATE(p.created_at) >= ? AND DATE(p.created_at) <= ?
AND p.cotacao IS NOT NULL AND p.cotacao > 0
AND (p.pgto IS NULL OR p.pgto != 'balance')
AND p.cotacao IS NOT NULL AND p.cotacao > 0 AND (p.pgto IS NULL OR p.pgto != 'balance')
GROUP BY ac.agente_id, p.id_conta
) combined
GROUP BY agente_id ORDER BY total_usd DESC LIMIT 10
`, [dataInicio, dataFim, dataInicio, dataFim]);
) combined GROUP BY agente_id ORDER BY total_usd DESC LIMIT 10
`, [dataInicio, dataFim, dataInicio, dataFim])
]);
// Resolve agent names
const agents = agentRanking.map((r, i) => {
@@ -444,8 +411,23 @@ async function fetchBIStrategic(dataInicio, dataFim) {
try {
const { prevStartStr, prevEndStr } = calcPrevPeriod(dataInicio, dataFim);
// === 1. COHORT RETENTION ===
const [cohortClients] = await conn.execute(`
// Run all 4 sections in parallel — each is independent
const [cohorts, expansion, crossSell, maturity] = await Promise.all([
_fetchCohorts(conn),
_fetchExpansion(conn, dataInicio, dataFim, prevStartStr, prevEndStr),
_fetchCrossSell(conn, dataInicio, dataFim),
_fetchMaturity(conn, dataInicio, dataFim, prevStartStr, prevEndStr)
]);
return { cohorts, expansion, crossSell, maturity };
} finally {
conn.release();
}
}
async function _fetchCohorts(conn) {
const [[cohortClients], [activeMonths]] = await Promise.all([
conn.execute(`
SELECT id_conta, DATE_FORMAT(MIN(first_op), '%Y-%m') as cohort_month FROM (
SELECT id_conta, MIN(created_at) as first_op FROM br_transaction_to_usa GROUP BY id_conta
UNION ALL
@@ -453,8 +435,8 @@ async function fetchBIStrategic(dataInicio, dataFim) {
WHERE cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
GROUP BY id_conta
) f GROUP BY id_conta
`);
const [activeMonths] = await conn.execute(`
`),
conn.execute(`
SELECT id_conta, active_month FROM (
SELECT id_conta, DATE_FORMAT(created_at, '%Y-%m') as active_month
FROM br_transaction_to_usa GROUP BY id_conta, DATE_FORMAT(created_at, '%Y-%m')
@@ -463,7 +445,8 @@ async function fetchBIStrategic(dataInicio, dataFim) {
FROM pagamento_br WHERE cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
GROUP BY id_conta, DATE_FORMAT(created_at, '%Y-%m')
) m
`);
`)
]);
const clientCohort = {};
cohortClients.forEach(r => { clientCohort[r.id_conta] = r.cohort_month; });
@@ -495,7 +478,7 @@ async function fetchBIStrategic(dataInicio, dataFim) {
});
const cohortKeys = Object.keys(cohortMap).sort().slice(-12);
const cohorts = cohortKeys.map(cm => {
return cohortKeys.map(cm => {
const c = cohortMap[cm];
const maxOff = allMonths.length - allMonths.indexOf(cm);
const retention = [];
@@ -504,9 +487,11 @@ async function fetchBIStrategic(dataInicio, dataFim) {
}
return { month: cm, size: c.size, retention };
});
}
// === 2. REVENUE EXPANSION / CONTRACTION ===
const [currRevenue] = await conn.execute(`
async function _fetchExpansion(conn, dataInicio, dataFim, prevStartStr, prevEndStr) {
const [[currRevenue], [prevRevenue]] = await Promise.all([
conn.execute(`
SELECT id_conta, ROUND(SUM(revenue), 2) as revenue, ROUND(SUM(vol_usd), 2) as vol_usd FROM (
SELECT id_conta, SUM((exchange_rate - ptax) / exchange_rate * amount_usd) as revenue, SUM(amount_usd) as vol_usd
FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
@@ -517,8 +502,8 @@ async function fetchBIStrategic(dataInicio, dataFim) {
AND cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
GROUP BY id_conta
) c GROUP BY id_conta
`, [dataInicio, dataFim, dataInicio, dataFim]);
const [prevRevenue] = await conn.execute(`
`, [dataInicio, dataFim, dataInicio, dataFim]),
conn.execute(`
SELECT id_conta, ROUND(SUM(revenue), 2) as revenue, ROUND(SUM(vol_usd), 2) as vol_usd FROM (
SELECT id_conta, SUM((exchange_rate - ptax) / exchange_rate * amount_usd) as revenue, SUM(amount_usd) as vol_usd
FROM br_transaction_to_usa WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
@@ -529,7 +514,8 @@ async function fetchBIStrategic(dataInicio, dataFim) {
AND cotacao IS NOT NULL AND cotacao > 0 AND (pgto IS NULL OR pgto != 'balance')
GROUP BY id_conta
) p GROUP BY id_conta
`, [prevStartStr, prevEndStr, prevStartStr, prevEndStr]);
`, [prevStartStr, prevEndStr, prevStartStr, prevEndStr])
]);
const currMap = {};
currRevenue.forEach(r => { currMap[r.id_conta] = { revenue: Number(r.revenue), vol_usd: Number(r.vol_usd) }; });
@@ -563,8 +549,10 @@ async function fetchBIStrategic(dataInicio, dataFim) {
}
}
});
return expansion;
}
// === 3. CROSS-SELL ===
async function _fetchCrossSell(conn, dataInicio, dataFim) {
const [crossSellData] = await conn.execute(`
SELECT c.id_conta, t.vol_usd as pay_vol, p.vol_usd as checkout_vol
FROM (
@@ -594,8 +582,10 @@ async function fetchBIStrategic(dataInicio, dataFim) {
else if (pv > 0) { crossSell.pay_only.count++; crossSell.pay_only.vol += pv; }
else if (cv > 0) { crossSell.checkout_only.count++; crossSell.checkout_only.vol += cv; }
});
return crossSell;
}
// === 4. CLIENT MATURITY SEGMENTS ===
async function _fetchMaturity(conn, dataInicio, dataFim, prevStartStr, prevEndStr) {
const [maturityData] = await conn.execute(`
SELECT id_conta,
MIN(first_op) as first_op,
@@ -645,11 +635,7 @@ async function fetchBIStrategic(dataInicio, dataFim) {
maturity.growing.count++; maturity.growing.vol += lv;
}
});
return { cohorts, expansion, crossSell, maturity };
} finally {
conn.release();
}
return maturity;
}
module.exports = {