301 lines
10 KiB
JavaScript
301 lines
10 KiB
JavaScript
import puppeteer from 'puppeteer-extra';
|
|
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
|
import db from './db.js';
|
|
import { json } from '@sveltejs/kit';
|
|
|
|
puppeteer.use(StealthPlugin());
|
|
let state = [];
|
|
let code = null;
|
|
let needCode = false;
|
|
let running = false;
|
|
|
|
export function getState() {
|
|
return state;
|
|
}
|
|
export function setCode(newCode) {
|
|
code = newCode;
|
|
}
|
|
|
|
export function isNeedCode() {
|
|
return needCode;
|
|
}
|
|
|
|
export function isRunning() {
|
|
return running;
|
|
}
|
|
|
|
export async function pullData(amount = 100) {
|
|
running = true;
|
|
try {
|
|
state = [];
|
|
code = null;
|
|
needCode = false;
|
|
state.push('Starting Browser');
|
|
const browser = await puppeteer.launch({
|
|
args: ['--no-sandbox']
|
|
//headless: false
|
|
//defaultViewport: null,
|
|
//args: ['--disable-blink-features=PrettyPrintJSONDocument']
|
|
});
|
|
|
|
const page = await browser.newPage();
|
|
|
|
state.push('Loading Cookies');
|
|
|
|
const cookiesdb = await db`SELECT
|
|
name,
|
|
value,
|
|
domain,
|
|
path,
|
|
expires,
|
|
size,
|
|
httpOnly,
|
|
secure,
|
|
session,
|
|
priority,
|
|
sameParty,
|
|
sourceScheme,
|
|
sourcePort
|
|
FROM cookies`;
|
|
|
|
cookiesdb.forEach((cookie) => {
|
|
cookie.expires = cookie.expires ? Number(cookie.expires) : undefined;
|
|
cookie.size = cookie.size ? Number(cookie.size) : undefined;
|
|
cookie.sourcePort = cookie.sourcePort ? Number(cookie.sourcePort) : undefined;
|
|
state.push('Loading cookie: ' + cookie.name);
|
|
browser.setCookie(cookie);
|
|
});
|
|
|
|
state.push('Navigating to United FCU');
|
|
|
|
// Navigate the page to a URL.
|
|
await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login');
|
|
|
|
if (page.url().includes('interstitial')) {
|
|
await page.waitForNavigation();
|
|
state.push('Already logged in, navigating to dashboard');
|
|
}
|
|
|
|
if (!page.url().includes('dashboard')) {
|
|
state.push('Logging in to United FCU');
|
|
// Type into search box using accessible input name.
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.locator('aria/Login ID').fill('92830');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
|
|
await page.locator('aria/Password').fill('Cmtjlt13');
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
await page.keyboard.press('Enter');
|
|
|
|
await page.waitForNavigation();
|
|
const url = page.url();
|
|
console.log('Current URL:', url);
|
|
|
|
if (url.includes('mfa/targets')) {
|
|
state.push('MFA required, selecting SMS option');
|
|
console.log('MFA required, please complete the authentication process.');
|
|
await page.locator('aria/SMS: (XXX) XXX-4029').click();
|
|
await page.waitForNavigation();
|
|
//need to do some stuff ehre
|
|
await page.keyboard.press('Tab');
|
|
state.push('Waiting for code input');
|
|
needCode = true;
|
|
for (let i = 0; i < 5 * 60; i++) {
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
if (code != null) break;
|
|
}
|
|
if (code == null) {
|
|
needCode = false;
|
|
state.push('Code not provided within 5 minutes');
|
|
throw new Error('Code not provided within 5 minutes');
|
|
}
|
|
state.push(`Got code: ${code}`);
|
|
|
|
await page.keyboard.type(code);
|
|
code = null;
|
|
needCode = false;
|
|
await page.keyboard.press('Enter');
|
|
await page.locator('aria/Register Device').click();
|
|
await page.waitForNavigation();
|
|
}
|
|
|
|
state.push('Saving cookies');
|
|
let cookies = await browser.cookies();
|
|
|
|
cookies.forEach(async (cookie) => {
|
|
state.push('Saving cookie: ' + cookie.name);
|
|
|
|
// Insert or update the cookie in the database
|
|
await db`DELETE FROM cookies WHERE name = ${cookie.name}`;
|
|
|
|
await db`INSERT INTO cookies (name, value, domain, path, expires, size, httpOnly, secure, session, priority, sameParty, sourceScheme, sourcePort)
|
|
VALUES (${cookie.name}, ${cookie.value}, ${cookie.domain}, ${cookie.path}, ${cookie.expires}, ${cookie.size}, ${cookie.httpOnly}, ${cookie.secure}, ${cookie.session}, ${cookie.priority}, ${cookie.sameParty}, ${cookie.sourceScheme}, ${cookie.sourcePort})`;
|
|
});
|
|
}
|
|
|
|
state.push('Fetching q2token');
|
|
const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value;
|
|
|
|
console.log('q2token:', q2token);
|
|
|
|
page.setExtraHTTPHeaders({
|
|
q2token: q2token
|
|
});
|
|
|
|
const accountsToPull = [
|
|
{
|
|
id: '65700',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '497016',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '1417342',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
},
|
|
{
|
|
id: '83851',
|
|
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
|
}
|
|
];
|
|
|
|
const accountsURL =
|
|
'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts';
|
|
await page.goto(accountsURL);
|
|
|
|
const accountsJsonResponse = await page.evaluate(() => {
|
|
return JSON.parse(document.querySelector('pre').textContent);
|
|
});
|
|
|
|
for (const account of accountsJsonResponse.data) {
|
|
console.log(`Account ID: ${account.id}, Name: ${account.name}`);
|
|
if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) {
|
|
const balance =
|
|
account.id == '1417342' || account.id == '83851'
|
|
? '-' + account.extended?.balance1
|
|
: account.extended?.balance1;
|
|
await db`INSERT INTO account (id, balance, balance_date)
|
|
VALUES (${account.id}, ${balance}, ${account.dataAsOfDate})
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
balance = EXCLUDED.balance,
|
|
balance_date = EXCLUDED.balance_date`;
|
|
console.log(`Account ID: ${account.id}, Balance: ${balance}`);
|
|
}
|
|
}
|
|
|
|
state.push('Fetching transactions for accounts');
|
|
for (const account of accountsToPull) {
|
|
state.push(`Fetching transactions for account ID: ${account.id}`);
|
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
|
|
|
await page.goto(account.url);
|
|
|
|
const jsonResponse = await page.evaluate(() => {
|
|
return JSON.parse(document.querySelector('pre').textContent);
|
|
});
|
|
|
|
if (!jsonResponse || !jsonResponse?.data?.transactions) {
|
|
throw new Error(`No data found for account ID: ${account.id}`);
|
|
}
|
|
|
|
const transactions = jsonResponse.data.transactions;
|
|
if (transactions.length === 0) {
|
|
state.push(`No transactions found for account ID: ${account.id}`);
|
|
continue;
|
|
}
|
|
state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`);
|
|
|
|
const cardRegEx = /\d{4}$/;
|
|
|
|
// Check if any pending need to be updated
|
|
|
|
state.push(`Checking for pending transactions for account ID: ${account.id}`);
|
|
let currentPend =
|
|
await db`SELECT id, amount, description, date from transaction where account_id = ${account.id} AND pending=true`;
|
|
for (const pend of currentPend) {
|
|
const found = transactions.find((t) => t.transactionId === pend.id);
|
|
if (!found) {
|
|
const updated = transactions.find(
|
|
(t) => t.amount == pend.amount && new Date(t.postedDate) == pend.date
|
|
);
|
|
if (updated) {
|
|
state.push(
|
|
`I think I found an updated transaction: ${updated.statementDescription} ${updated.amount} for ${pend.description} ${pend.amount}`
|
|
);
|
|
|
|
await db`UPDATE budget_transaction SET transaction_id = ${updated.transactionId} WHERE transaction_id = ${pend.id}`;
|
|
} else {
|
|
state.push(`Orphaning no longer pending budget transaction with no new parent`);
|
|
await db`UPDATE budget_transaction SET transaction_id = null WHERE transaction_id = ${pend.id}`;
|
|
}
|
|
state.push(`Removing pending transaction: ${pend.id}`);
|
|
await db`DELETE FROM transaction WHERE id = ${pend.id}`;
|
|
}
|
|
}
|
|
|
|
for (const transaction of transactions) {
|
|
const amount = Number(transaction.amount);
|
|
const date = new Date(transaction.postedDate);
|
|
const payee = transaction.description || '';
|
|
const statementDescription = transaction.statementDescription;
|
|
const card = cardRegEx.test(statementDescription)
|
|
? statementDescription.match(cardRegEx)[0]
|
|
: null;
|
|
const pending = transaction.extended?.allTransactionType == 1 ? true : false;
|
|
const accountId = transaction.accountId;
|
|
|
|
const id = pending
|
|
? `${transaction.postedDate}:${transaction.amount}`
|
|
: transaction.hostTranNumber;
|
|
|
|
await db`INSERT INTO transaction (
|
|
id,
|
|
account_id,
|
|
amount,
|
|
description,
|
|
date,
|
|
payee,
|
|
statement_description,
|
|
card,
|
|
pending)
|
|
VALUES (
|
|
${id},
|
|
${accountId},
|
|
${amount},
|
|
${statementDescription},
|
|
${date},
|
|
${payee},
|
|
${statementDescription},
|
|
(SELECT id from cards where card_number=${card}),
|
|
${pending})
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
amount = EXCLUDED.amount,
|
|
description = EXCLUDED.description,
|
|
card = EXCLUDED.card,
|
|
date = EXCLUDED.date,
|
|
payee = EXCLUDED.payee,
|
|
statement_description = EXCLUDED.statement_description,
|
|
pending = EXCLUDED.pending`;
|
|
}
|
|
}
|
|
|
|
state.push('Orphaning transactions');
|
|
|
|
const orphaned = await db`SELECT bt.id as id
|
|
FROM budget_transaction bt
|
|
LEFT OUTER JOIN transaction t ON bt.transaction_id = t.id
|
|
WHERE t.id IS NULL;`;
|
|
for (const orphan of orphaned) {
|
|
state.push(`Orphaning transaction: ${orphan.id}`);
|
|
await db`UPDATE budget_transaction set transaction_id = null where id = ${orphan.id}`;
|
|
}
|
|
} catch (error) {
|
|
console.error('Error in pullData:', error);
|
|
state.push(`Error: ${error.message}`);
|
|
}
|
|
|
|
running = false;
|
|
}
|