275 lines
9.8 KiB
JavaScript
275 lines
9.8 KiB
JavaScript
import db from './db.js';
|
|
import { chromium } from 'patchright';
|
|
|
|
//const browser = await chromium.launch();
|
|
let state = [];
|
|
let code = null;
|
|
let needCode = false;
|
|
let running = false;
|
|
|
|
export function getState() {
|
|
return state;
|
|
}
|
|
export function setCode(newCode) {
|
|
code = newCode;
|
|
}
|
|
|
|
export function isNeedCode() {
|
|
return needCode;
|
|
}
|
|
|
|
export function isRunning() {
|
|
return running;
|
|
}
|
|
|
|
const browser = await chromium.launchPersistentContext('context', {
|
|
channel: 'chrome',
|
|
headless: false,
|
|
viewport: null
|
|
// do NOT add custom browser headers or userAgent
|
|
});
|
|
|
|
export async function pullData(amount = 100) {
|
|
running = true;
|
|
state = [];
|
|
code = null;
|
|
needCode = false;
|
|
state.push('Starting Browser');
|
|
|
|
|
|
const page = await browser.newPage();
|
|
try {
|
|
|
|
state.push('Navigating to United FCU');
|
|
await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login');
|
|
state.push(`Current URL: ${page.url()}`);
|
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
|
|
if (page.url().includes('interstitial')) {
|
|
await page.waitForLoadState();
|
|
}
|
|
while (page.url().includes('interstitial')) {
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
await page.waitForLoadState();
|
|
}
|
|
|
|
if (!page.url().includes('dashboard')) {
|
|
state.push('Logging in to United FCU');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.getByLabel('Login ID').fill('92830');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.getByLabel('Password').fill('Cmtjlt13');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.keyboard.press('Enter');
|
|
|
|
//await page.waitForLoadState();
|
|
//await new Promise((resolve) => setTimeout(resolve, 5 * 5000));
|
|
|
|
while (page.url().includes('interstitial')) {
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
await page.waitForLoadState();
|
|
}
|
|
|
|
const url = page.url();
|
|
state.push(`Current URL after login: ${url}`);
|
|
|
|
if (url.includes('mfa/targets')) {
|
|
state.push('MFA required, selecting SMS option');
|
|
console.log('MFA required, please complete the authentication process.');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.getByText('SMS: (XXX) XXX-4029').click();
|
|
await page.waitForURL('**/mfa/entertarget');
|
|
await page.getByPlaceholder('Secure Access Code');
|
|
//await page.keyboard.press('Tab');
|
|
state.push('Waiting for code input');
|
|
needCode = true;
|
|
for (let i = 0; i < 5 * 60; i++) {
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
if (code != null) break;
|
|
}
|
|
needCode = false;
|
|
if (code == null) {
|
|
state.push('Code not provided within 5 minutes');
|
|
throw new Error('Code not provided within 5 minutes');
|
|
}
|
|
state.push(`Got code: ${code}`);
|
|
await page.getByPlaceholder('Secure Access Code').fill(code);
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.keyboard.press('Enter');
|
|
|
|
await page.getByText('Access Code Accepted.').waitFor();
|
|
await page.keyboard.press('Tab');
|
|
await page.keyboard.press('Tab');
|
|
await page.keyboard.press('Enter');
|
|
await page.waitForURL('**/dashboard');
|
|
await page.screenshot({ path: 'united-login.png' });
|
|
state.push('Logged in successfully');
|
|
}
|
|
}
|
|
|
|
state.push('Fetching q2token');
|
|
const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value;
|
|
|
|
console.log('q2token:', q2token);
|
|
|
|
page.setExtraHTTPHeaders({
|
|
q2token: q2token
|
|
});
|
|
|
|
const accountsToPull = [
|
|
{
|
|
id: '65700',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '497016',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '1417342',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '83851',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
}
|
|
];
|
|
|
|
const accountsURL =
|
|
'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts';
|
|
await page.goto(accountsURL);
|
|
|
|
const accountsJsonResponse = await page.evaluate(() => {
|
|
return JSON.parse(document.querySelector('pre').textContent);
|
|
});
|
|
|
|
for (const account of accountsJsonResponse.data) {
|
|
console.log(`Account ID: ${account.id}, Name: ${account.name}`);
|
|
if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) {
|
|
const balance =
|
|
account.id == '1417342' || account.id == '83851'
|
|
? '-' + account.extended?.balance1
|
|
: account.extended?.balance1;
|
|
await db`INSERT INTO account (id, balance, balance_date)
|
|
VALUES (${account.id}, ${balance}, ${account.dataAsOfDate})
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
balance = EXCLUDED.balance,
|
|
balance_date = EXCLUDED.balance_date`;
|
|
console.log(`Account ID: ${account.id}, Balance: ${balance}`);
|
|
}
|
|
}
|
|
|
|
state.push('Fetching transactions for accounts');
|
|
for (const account of accountsToPull) {
|
|
state.push(`Fetching transactions for account ID: ${account.id}`);
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
|
|
await page.goto(account.url);
|
|
|
|
const jsonResponse = await page.evaluate(() => {
|
|
return JSON.parse(document.querySelector('pre').textContent);
|
|
});
|
|
|
|
if (!jsonResponse || !jsonResponse?.data?.transactions) {
|
|
throw new Error(`No data found for account ID: ${account.id}`);
|
|
}
|
|
|
|
const transactions = jsonResponse.data.transactions;
|
|
if (transactions.length === 0) {
|
|
state.push(`No transactions found for account ID: ${account.id}`);
|
|
continue;
|
|
}
|
|
state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`);
|
|
|
|
const cardRegEx = /\d{4}$/;
|
|
|
|
// Check if any pending need to be updated
|
|
|
|
state.push(`Checking for pending transactions for account ID: ${account.id}`);
|
|
let currentPend =
|
|
await db`SELECT id, amount, description, date from transaction where account_id = ${account.id} AND pending=true`;
|
|
for (const pend of currentPend) {
|
|
const found = transactions.find((t) => `${t.description}:${t.amount}` === pend.id && t.transactionType == "Memo");
|
|
|
|
if (found && found.transactionType != "Memo")
|
|
{state.push(
|
|
`I think I found an updated transaction: ${found.statementDescription} ${found.amount} for ${pend.description} ${pend.amount}`
|
|
);
|
|
await db`UPDATE budget_transaction SET transaction_id = ${found.transactionId} WHERE transaction_id = ${pend.id}`;
|
|
await db`DELETE FROM transaction WHERE id = ${pend.id}`;
|
|
} else if (!found)
|
|
{
|
|
state.push(`Orphaning no longer pending budget transaction with no new parent`);
|
|
|
|
await db`UPDATE budget_transaction SET transaction_id = null, notes = notes || ${pend.description} WHERE transaction_id = ${pend.id}`;
|
|
state.push(`Removing pending transaction: ${pend.id}`);
|
|
await db`DELETE FROM transaction WHERE id = ${pend.id}`;
|
|
}
|
|
}
|
|
|
|
for (const transaction of transactions) {
|
|
const amount = Number(transaction.amount);
|
|
const date = new Date(transaction.postedDate);
|
|
const payee = transaction.description || '';
|
|
const statementDescription = transaction.statementDescription;
|
|
const card = cardRegEx.test(statementDescription)
|
|
? statementDescription.match(cardRegEx)[0]
|
|
: null;
|
|
const pending = transaction.extended?.allTransactionType == 1 ? true : false;
|
|
const accountId = transaction.accountId;
|
|
|
|
const id = pending
|
|
? `${transaction.description}:${transaction.amount}`
|
|
: transaction.hostTranNumber;
|
|
|
|
await db`INSERT INTO transaction (
|
|
id,
|
|
account_id,
|
|
amount,
|
|
description,
|
|
date,
|
|
payee,
|
|
statement_description,
|
|
card,
|
|
pending)
|
|
VALUES (
|
|
${id},
|
|
${accountId},
|
|
${amount},
|
|
${statementDescription},
|
|
${date},
|
|
${payee},
|
|
${statementDescription},
|
|
(SELECT id from cards where card_number=${card}),
|
|
${pending})
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
amount = EXCLUDED.amount,
|
|
description = EXCLUDED.description,
|
|
card = EXCLUDED.card,
|
|
date = EXCLUDED.date,
|
|
payee = EXCLUDED.payee,
|
|
statement_description = EXCLUDED.statement_description,
|
|
pending = EXCLUDED.pending`;
|
|
}
|
|
}
|
|
/*
|
|
state.push('Orphaning transactions');
|
|
|
|
const orphaned = await db`SELECT bt.id as id
|
|
FROM budget_transaction bt
|
|
LEFT OUTER JOIN transaction t ON bt.transaction_id = t.id
|
|
WHERE t.id IS NULL;`;
|
|
for (const orphan of orphaned) {
|
|
state.push(`Orphaning transaction: ${orphan.id}`);
|
|
await db`UPDATE budget_transaction set transaction_id = null where id = ${orphan.id}`;
|
|
}
|
|
*/
|
|
state.push('Done');
|
|
} catch (error) {
|
|
console.error('Error in pullData:', error);
|
|
state.push(`Error: ${error.message}`);
|
|
}
|
|
page.close();
|
|
running = false;
|
|
}
|