import db from './db.js'; import { chromium } from 'patchright'; //const browser = await chromium.launch(); let state = []; let code = null; let needCode = false; let running = false; export function getState() { return state; } export function setCode(newCode) { code = newCode; } export function isNeedCode() { return needCode; } export function isRunning() { return running; } export async function pullData(amount = 100) { running = true; try { state = []; code = null; needCode = false; state.push('Starting Browser'); const browser = await chromium.launchPersistentContext('context', { channel: 'chrome', headless: true, viewport: null // do NOT add custom browser headers or userAgent }); const page = await browser.newPage(); state.push('Navigating to United FCU'); await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login'); state.push(`Current URL: ${page.url()}`); if (page.url().includes('interstitial')) { await page.waitForLoadState(); } if (!page.url().includes('dashboard')) { state.push('Logging in to United FCU'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.getByLabel('Login ID').fill('92830'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.getByLabel('Password').fill('Cmtjlt13'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.keyboard.press('Enter'); //await page.waitForLoadState(); //await new Promise((resolve) => setTimeout(resolve, 5 * 5000)); while (page.url().includes('interstitial')) { await new Promise((resolve) => setTimeout(resolve, 1000)); await page.waitForLoadState(); } const url = page.url(); state.push(`Current URL after login: ${url}`); if (url.includes('mfa/targets')) { state.push('MFA required, selecting SMS option'); console.log('MFA required, please complete the authentication process.'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.getByText('SMS: (XXX) XXX-4029').click(); await page.waitForURL('**/mfa/entertarget'); await page.getByPlaceholder('Secure Access Code'); //await page.keyboard.press('Tab'); state.push('Waiting for code input'); needCode = true; for (let i = 0; i < 5 * 60; i++) { await new Promise((resolve) => setTimeout(resolve, 1000)); if (code != null) break; } if (code == null) { needCode = false; state.push('Code not provided within 5 minutes'); throw new Error('Code not provided within 5 minutes'); } state.push(`Got code: ${code}`); await page.getByPlaceholder('Secure Access Code').fill(code); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.keyboard.press('Enter'); await page.getByText('Access Code Accepted.').waitFor(); await page.keyboard.press('Tab'); await page.keyboard.press('Tab'); await page.keyboard.press('Enter'); await page.waitForURL('**/dashboard'); await page.screenshot({ path: 'united-login.png' }); state.push('Logged in successfully'); } } state.push('Fetching q2token'); const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value; console.log('q2token:', q2token); page.setExtraHTTPHeaders({ q2token: q2token }); const accountsToPull = [ { id: '65700', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '497016', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '1417342', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '83851', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` } ]; const accountsURL = 'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts'; await page.goto(accountsURL); const accountsJsonResponse = await page.evaluate(() => { return JSON.parse(document.querySelector('pre').textContent); }); for (const account of accountsJsonResponse.data) { console.log(`Account ID: ${account.id}, Name: ${account.name}`); if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) { const balance = account.id == '1417342' || account.id == '83851' ? '-' + account.extended?.balance1 : account.extended?.balance1; await db`INSERT INTO account (id, balance, balance_date) VALUES (${account.id}, ${balance}, ${account.dataAsOfDate}) ON CONFLICT (id) DO UPDATE SET balance = EXCLUDED.balance, balance_date = EXCLUDED.balance_date`; console.log(`Account ID: ${account.id}, Balance: ${balance}`); } } state.push('Fetching transactions for accounts'); for (const account of accountsToPull) { state.push(`Fetching transactions for account ID: ${account.id}`); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.goto(account.url); const jsonResponse = await page.evaluate(() => { return JSON.parse(document.querySelector('pre').textContent); }); if (!jsonResponse || !jsonResponse?.data?.transactions) { throw new Error(`No data found for account ID: ${account.id}`); } const transactions = jsonResponse.data.transactions; if (transactions.length === 0) { state.push(`No transactions found for account ID: ${account.id}`); continue; } state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`); const cardRegEx = /\d{4}$/; // Check if any pending need to be updated state.push(`Checking for pending transactions for account ID: ${account.id}`); let currentPend = await db`SELECT id, amount, description, date from transaction where account_id = ${account.id} AND pending=true`; for (const pend of currentPend) { const found = transactions.find((t) => t.transactionId === pend.id); if (!found) { const updated = transactions.find( (t) => t.amount == pend.amount && new Date(t.postedDate) == pend.date ); if (updated) { state.push( `I think I found an updated transaction: ${updated.statementDescription} ${updated.amount} for ${pend.description} ${pend.amount}` ); await db`UPDATE budget_transaction SET transaction_id = ${updated.transactionId} WHERE transaction_id = ${pend.id}`; } else { state.push(`Orphaning no longer pending budget transaction with no new parent`); await db`UPDATE budget_transaction SET transaction_id = null WHERE transaction_id = ${pend.id}`; } state.push(`Removing pending transaction: ${pend.id}`); await db`DELETE FROM transaction WHERE id = ${pend.id}`; } } for (const transaction of transactions) { const amount = Number(transaction.amount); const date = new Date(transaction.postedDate); const payee = transaction.description || ''; const statementDescription = transaction.statementDescription; const card = cardRegEx.test(statementDescription) ? statementDescription.match(cardRegEx)[0] : null; const pending = transaction.extended?.allTransactionType == 1 ? true : false; const accountId = transaction.accountId; const id = pending ? `${transaction.postedDate}:${transaction.amount}` : transaction.hostTranNumber; await db`INSERT INTO transaction ( id, account_id, amount, description, date, payee, statement_description, card, pending) VALUES ( ${id}, ${accountId}, ${amount}, ${statementDescription}, ${date}, ${payee}, ${statementDescription}, (SELECT id from cards where card_number=${card}), ${pending}) ON CONFLICT (id) DO UPDATE SET amount = EXCLUDED.amount, description = EXCLUDED.description, card = EXCLUDED.card, date = EXCLUDED.date, payee = EXCLUDED.payee, statement_description = EXCLUDED.statement_description, pending = EXCLUDED.pending`; } } state.push('Orphaning transactions'); const orphaned = await db`SELECT bt.id as id FROM budget_transaction bt LEFT OUTER JOIN transaction t ON bt.transaction_id = t.id WHERE t.id IS NULL;`; for (const orphan of orphaned) { state.push(`Orphaning transaction: ${orphan.id}`); await db`UPDATE budget_transaction set transaction_id = null where id = ${orphan.id}`; } state.push('Done'); await browser.close(); } catch (error) { console.error('Error in pullData:', error); state.push(`Error: ${error.message}`); } running = false; }