import puppeteer from 'puppeteer-extra'; import StealthPlugin from 'puppeteer-extra-plugin-stealth'; import db from './db.js'; import { json } from '@sveltejs/kit'; puppeteer.use(StealthPlugin()); let state = []; let code = null; let needCode = false; let running = false; export function getState() { return state; } export function setCode(newCode) { code = newCode; } export function isNeedCode() { return needCode; } export function isRunning() { return running; } export async function pullData(amount = 100) { running = true; try { state = []; code = null; needCode = false; state.push('Starting Browser'); const browser = await puppeteer.launch({ args: ['--no-sandbox'] //headless: false //defaultViewport: null, //args: ['--disable-blink-features=PrettyPrintJSONDocument'] }); const page = await browser.newPage(); state.push('Loading Cookies'); const cookiesdb = await db`SELECT name, value, domain, path, expires, size, httpOnly, secure, session, priority, sameParty, sourceScheme, sourcePort FROM cookies`; cookiesdb.forEach((cookie) => { cookie.expires = cookie.expires ? Number(cookie.expires) : undefined; cookie.size = cookie.size ? Number(cookie.size) : undefined; cookie.sourcePort = cookie.sourcePort ? Number(cookie.sourcePort) : undefined; state.push('Loading cookie: ' + cookie.name); browser.setCookie(cookie); }); state.push('Navigating to United FCU'); // Navigate the page to a URL. await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login'); if (page.url().includes('interstitial')) { await page.waitForNavigation(); state.push('Already logged in, navigating to dashboard'); } if (!page.url().includes('dashboard')) { state.push('Logging in to United FCU'); // Type into search box using accessible input name. await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.locator('aria/Login ID').fill('92830'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.locator('aria/Password').fill('Cmtjlt13'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.keyboard.press('Enter'); await page.waitForNavigation(); const url = page.url(); console.log('Current URL:', url); if (url.includes('mfa/targets')) { state.push('MFA required, selecting SMS option'); console.log('MFA required, please complete the authentication process.'); await page.locator('aria/SMS: (XXX) XXX-4029').click(); await page.waitForNavigation(); //need to do some stuff ehre await page.keyboard.press('Tab'); state.push('Waiting for code input'); needCode = true; for (let i = 0; i < 5 * 60; i++) { await new Promise((resolve) => setTimeout(resolve, 1000)); if (code != null) break; } if (code == null) { needCode = false; state.push('Code not provided within 5 minutes'); throw new Error('Code not provided within 5 minutes'); } state.push(`Got code: ${code}`); await page.keyboard.type(code); code = null; needCode = false; await page.keyboard.press('Enter'); await page.locator('aria/Register Device').click(); await page.waitForNavigation(); } state.push('Saving cookies'); let cookies = await browser.cookies(); cookies.forEach(async (cookie) => { state.push('Saving cookie: ' + cookie.name); // Insert or update the cookie in the database await db`DELETE FROM cookies WHERE name = ${cookie.name}`; await db`INSERT INTO cookies (name, value, domain, path, expires, size, httpOnly, secure, session, priority, sameParty, sourceScheme, sourcePort) VALUES (${cookie.name}, ${cookie.value}, ${cookie.domain}, ${cookie.path}, ${cookie.expires}, ${cookie.size}, ${cookie.httpOnly}, ${cookie.secure}, ${cookie.session}, ${cookie.priority}, ${cookie.sameParty}, ${cookie.sourceScheme}, ${cookie.sourcePort})`; }); } state.push('Fetching q2token'); const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value; console.log('q2token:', q2token); page.setExtraHTTPHeaders({ q2token: q2token }); const accountsToPull = [ { id: '65700', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '497016', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '1417342', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` }, { id: '83851', url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd` } ]; const accountsURL = 'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts'; await page.goto(accountsURL); const accountsJsonResponse = await page.evaluate(() => { return JSON.parse(document.querySelector('pre').textContent); }); for (const account of accountsJsonResponse.data) { console.log(`Account ID: ${account.id}, Name: ${account.name}`); if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) { const balance = account.id == '1417342' || account.id == '83851' ? '-' + account.extended?.balance1 : account.extended?.balance1; await db`INSERT INTO account (id, balance, balance_date) VALUES (${account.id}, ${balance}, ${account.dataAsOfDate}) ON CONFLICT (id) DO UPDATE SET balance = EXCLUDED.balance, balance_date = EXCLUDED.balance_date`; console.log(`Account ID: ${account.id}, Balance: ${balance}`); } } state.push('Fetching transactions for accounts'); for (const account of accountsToPull) { state.push(`Fetching transactions for account ID: ${account.id}`); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.goto(account.url); const jsonResponse = await page.evaluate(() => { return JSON.parse(document.querySelector('pre').textContent); }); if (!jsonResponse || !jsonResponse?.data?.transactions) { throw new Error(`No data found for account ID: ${account.id}`); } const transactions = jsonResponse.data.transactions; if (transactions.length === 0) { state.push(`No transactions found for account ID: ${account.id}`); continue; } state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`); const cardRegEx = /\d{4}$/; // Check if any pending need to be updated state.push(`Checking for pending transactions for account ID: ${account.id}`); let currentPend = await db`SELECT id, amount, description, date from transaction where account_id = ${account.id} AND pending=true`; for (const pend of currentPend) { const found = transactions.find((t) => t.transactionId === pend.id); if (!found) { const updated = transactions.find( (t) => t.amount == pend.amount && new Date(t.postedDate) == pend.date ); if (updated) { state.push( `I think I found an updated transaction: ${updated.statementDescription} ${updated.amount} for ${pend.description} ${pend.amount}` ); await db`UPDATE budget_transaction SET transaction_id = ${updated.transactionId} WHERE transaction_id = ${pend.id}`; } else { state.push(`Orphaning no longer pending budget transaction with no new parent`); await db`UPDATE budget_transaction SET transaction_id = null WHERE transaction_id = ${pend.id}`; } state.push(`Removing pending transaction: ${pend.id}`); await db`DELETE FROM transaction WHERE id = ${pend.id}`; } } for (const transaction of transactions) { const amount = Number(transaction.amount); const date = new Date(transaction.postedDate); const payee = transaction.description || ''; const statementDescription = transaction.statementDescription; const card = cardRegEx.test(statementDescription) ? statementDescription.match(cardRegEx)[0] : null; const pending = transaction.extended?.allTransactionType == 1 ? true : false; const accountId = transaction.accountId; const id = pending ? `${transaction.postedDate}:${transaction.amount}` : transaction.hostTranNumber; await db`INSERT INTO transaction ( id, account_id, amount, description, date, payee, statement_description, card, pending) VALUES ( ${id}, ${accountId}, ${amount}, ${statementDescription}, ${date}, ${payee}, ${statementDescription}, (SELECT id from cards where card_number=${card}), ${pending}) ON CONFLICT (id) DO UPDATE SET amount = EXCLUDED.amount, description = EXCLUDED.description, card = EXCLUDED.card, date = EXCLUDED.date, payee = EXCLUDED.payee, statement_description = EXCLUDED.statement_description, pending = EXCLUDED.pending`; } } state.push('Orphaning transactions'); const orphaned = await db`SELECT bt.id as id FROM budget_transaction bt LEFT OUTER JOIN transaction t ON bt.transaction_id = t.id WHERE t.id IS NULL;`; for (const orphan of orphaned) { state.push(`Orphaning transaction: ${orphan.id}`); await db`UPDATE budget_transaction set transaction_id = null where id = ${orphan.id}`; } } catch (error) { console.error('Error in pullData:', error); state.push(`Error: ${error.message}`); } running = false; }