Moved to new united api
This commit is contained in:
139
src/lib/db.js
139
src/lib/db.js
@ -84,10 +84,11 @@ export async function getBudgetTransactions(id) {
|
||||
let transactions = await db`
|
||||
select
|
||||
transaction.id as id,
|
||||
transaction.posted as posted,
|
||||
transaction.amount as amount,
|
||||
transaction.account_id as account_id,
|
||||
transaction.description as description,
|
||||
transaction.pending as pending,
|
||||
transaction.pending as pending,
|
||||
transaction.amount as amount,
|
||||
transaction.date as date,
|
||||
budget_transaction.notes as notes,
|
||||
transaction.payee as payee,
|
||||
budget_transaction.amount as budget_amount,
|
||||
@ -98,10 +99,6 @@ export async function getBudgetTransactions(id) {
|
||||
order by transaction.posted desc
|
||||
`;
|
||||
console.log(`Fetched ${transactions.length} transactions for budget ${id}`);
|
||||
transactions = transactions.map((t) => ({
|
||||
...t,
|
||||
date: new Date(t.posted * 1000)
|
||||
}));
|
||||
// transactions = Result [{ id: 1, posted: 1633036800, amount: 50.00, description: "Grocery Store", pending: false, notes: "Weekly groceries" }, ...]
|
||||
return { transactions };
|
||||
} catch {
|
||||
@ -207,20 +204,13 @@ export async function getDeletedBudgets() {
|
||||
export async function getAccount(id) {
|
||||
const account = await db`
|
||||
select
|
||||
account.id as id,
|
||||
account.name as name,
|
||||
account.balance as balance,
|
||||
account.available_balance as available_balance,
|
||||
account.balance_date as balance_date,
|
||||
account.in_total as in_total,
|
||||
account.hide as hide,
|
||||
org.id as org_id,
|
||||
org.name as org_name,
|
||||
org.domain as org_domain,
|
||||
org.sfin_url as org_sfin_url,
|
||||
org.url as org_url
|
||||
from account
|
||||
left join org on account.org_id = org.id
|
||||
account.id as id,
|
||||
account.name as name,
|
||||
account.balance as balance,
|
||||
account.in_total as in_total,
|
||||
account.balance_date as balance_date,
|
||||
account.hide as hide
|
||||
FROM account
|
||||
where account.id = ${id}
|
||||
`;
|
||||
if (!account || account.length === 0) {
|
||||
@ -272,20 +262,19 @@ export async function getTransactions(accountId) {
|
||||
let transactions = await db`
|
||||
select
|
||||
transaction.id as id,
|
||||
transaction.posted as posted,
|
||||
transaction.account_id as account_id,
|
||||
transaction.amount as amount,
|
||||
transaction.description as description,
|
||||
transaction.pending as pending,
|
||||
transaction.notes as notes,
|
||||
transaction.payee as payee
|
||||
transaction.payee as payee,
|
||||
transaction.date as date,
|
||||
transaction.statement_description as statement_description
|
||||
from transaction
|
||||
where account_id = ${accountId}
|
||||
order by posted desc
|
||||
order by date desc
|
||||
`;
|
||||
transactions = transactions.map((t) => ({
|
||||
...t,
|
||||
date: new Date(t.posted * 1000)
|
||||
}));
|
||||
|
||||
return transactions;
|
||||
}
|
||||
|
||||
@ -298,100 +287,6 @@ export async function setTransactionNote(transactionId, note) {
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function updateAccounts(data) {
|
||||
try {
|
||||
console.log('Updating accounts with data:', data);
|
||||
for (const account of data.accounts) {
|
||||
// Upsert Org
|
||||
console.log(`Upserting org for account: ${account.id}`, account.org);
|
||||
await db`
|
||||
insert into org (id, domain, name, sfin_url, url)
|
||||
values (${account.org.id}, ${account.org.domain ?? null}, ${account.org.name ?? null}, ${account.org.sfin_url ?? null}, ${account.org.url ?? null})
|
||||
on conflict (id) do update set
|
||||
domain = excluded.domain,
|
||||
name = excluded.name,
|
||||
sfin_url = excluded.sfin_url,
|
||||
url = excluded.url
|
||||
`;
|
||||
console.log(`Upserting account: ${account.id} (${account.name})`);
|
||||
// Upsert Account
|
||||
await db`
|
||||
insert into account (id, org_id, name, currency, balance, available_balance, balance_date)
|
||||
values (
|
||||
${account.id},
|
||||
${account.org.id},
|
||||
${account.name ?? null},
|
||||
${account.currency ?? null},
|
||||
${account.balance ?? null},
|
||||
${account.available_balance ?? null},
|
||||
${account.balance_date ?? null}
|
||||
)
|
||||
on conflict (id) do update set
|
||||
org_id = excluded.org_id,
|
||||
name = excluded.name,
|
||||
currency = excluded.currency,
|
||||
balance = excluded.balance,
|
||||
available_balance = excluded.available_balance,
|
||||
balance_date = excluded.balance_date
|
||||
`;
|
||||
|
||||
// Upsert Transactions
|
||||
if (account.transactions && account.transactions.length > 0) {
|
||||
for (const txn of account.transactions) {
|
||||
let extraId = null;
|
||||
console.log(`Upserting transaction: ${txn.id} for account: ${account.id}`);
|
||||
if (txn.extra) {
|
||||
// Upsert TransactionExtra (insert only, update not needed for category)
|
||||
const extraResult = await db`
|
||||
insert into transaction_extra (category)
|
||||
values (${txn.extra.category ?? null})
|
||||
on conflict (category) do nothing
|
||||
returning id
|
||||
`;
|
||||
if (extraResult.length > 0) {
|
||||
extraId = extraResult[0].id;
|
||||
} else {
|
||||
// If already exists, fetch id
|
||||
const existing = await db`
|
||||
select id from transaction_extra where category = ${txn.extra.category ?? null}
|
||||
`;
|
||||
if (existing.length > 0) {
|
||||
extraId = existing[0].id;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`Preparing to upsert transaction: ${txn.id} with data:`, txn);
|
||||
await db`
|
||||
insert into transaction (id, account_id, posted, amount, description, pending, transacted_at, payee)
|
||||
values (
|
||||
${txn.id},
|
||||
${account.id},
|
||||
${txn.posted},
|
||||
${txn.amount ?? null},
|
||||
${txn.description ?? null},
|
||||
${txn.pending ?? false},
|
||||
${txn.transacted_at ?? 0},
|
||||
${txn.payee ?? null}
|
||||
)
|
||||
on conflict (id) do update set
|
||||
account_id = excluded.account_id,
|
||||
posted = excluded.posted,
|
||||
amount = excluded.amount,
|
||||
description = excluded.description,
|
||||
pending = excluded.pending,
|
||||
transacted_at = excluded.transacted_at,
|
||||
payee = excluded.payee
|
||||
`;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('updateAccounts error:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getRules(data) {
|
||||
try {
|
||||
const rules = await db`
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
const url =
|
||||
'https://19443E0E8171E175EC5DA0C69B35DD50197F234B9A74C00D27FD606121257ECF:DAA3702E2100CFFD3B544251E6D755E86B1EDDFBFCC7F6FA9CE77AB3677E60DE@beta-bridge.simplefin.org/simplefin';
|
||||
|
||||
export async function fetchAccounts(startDate) {
|
||||
const { username, password, origin, pathname } = new URL(url);
|
||||
const apiUrl = `${origin}${pathname}/accounts?start-date=${startDate}`;
|
||||
const headers = {};
|
||||
|
||||
console.log(`Fetching accounts from: ${apiUrl}`);
|
||||
|
||||
if (username && password) {
|
||||
headers['Authorization'] = 'Basic ' + btoa(`${username}:${password}`);
|
||||
}
|
||||
|
||||
const response = await fetch(apiUrl, { headers });
|
||||
return await response.json();
|
||||
}
|
||||
257
src/lib/united.js
Normal file
257
src/lib/united.js
Normal file
@ -0,0 +1,257 @@
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import db from './db.js';
|
||||
import { json } from '@sveltejs/kit';
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
let state = [];
|
||||
let code = null;
|
||||
let needCode = false;
|
||||
let running = false;
|
||||
|
||||
export function getState() {
|
||||
return state;
|
||||
}
|
||||
export function setCode(newCode) {
|
||||
code = newCode;
|
||||
}
|
||||
|
||||
export function isNeedCode() {
|
||||
return needCode;
|
||||
}
|
||||
|
||||
export function isRunning() {
|
||||
return running;
|
||||
}
|
||||
|
||||
export async function pullData(amount = 100) {
|
||||
running = true;
|
||||
try {
|
||||
state = [];
|
||||
code = null;
|
||||
needCode = false;
|
||||
state.push('Starting Browser');
|
||||
const browser = await puppeteer.launch({
|
||||
//headless: false
|
||||
//defaultViewport: null,
|
||||
//args: ['--disable-blink-features=PrettyPrintJSONDocument']
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
state.push('Loading Cookies');
|
||||
|
||||
const cookiesdb = await db`SELECT
|
||||
name,
|
||||
value,
|
||||
domain,
|
||||
path,
|
||||
expires,
|
||||
size,
|
||||
httpOnly,
|
||||
secure,
|
||||
session,
|
||||
priority,
|
||||
sameParty,
|
||||
sourceScheme,
|
||||
sourcePort
|
||||
FROM cookies`;
|
||||
|
||||
cookiesdb.forEach((cookie) => {
|
||||
cookie.expires = cookie.expires ? Number(cookie.expires) : undefined;
|
||||
cookie.size = cookie.size ? Number(cookie.size) : undefined;
|
||||
cookie.sourcePort = cookie.sourcePort ? Number(cookie.sourcePort) : undefined;
|
||||
state.push('Loading cookie: ' + cookie.name);
|
||||
browser.setCookie(cookie);
|
||||
});
|
||||
|
||||
state.push('Navigating to United FCU');
|
||||
|
||||
// Navigate the page to a URL.
|
||||
await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login');
|
||||
|
||||
if (page.url().includes('interstitial')) {
|
||||
await page.waitForNavigation();
|
||||
state.push('Already logged in, navigating to dashboard');
|
||||
}
|
||||
|
||||
if (!page.url().includes('dashboard')) {
|
||||
state.push('Logging in to United FCU');
|
||||
// Type into search box using accessible input name.
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
||||
await page.locator('aria/Login ID').fill('92830');
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
||||
|
||||
await page.locator('aria/Password').fill('Cmtjlt13');
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
||||
await page.keyboard.press('Enter');
|
||||
|
||||
await page.waitForNavigation();
|
||||
const url = page.url();
|
||||
console.log('Current URL:', url);
|
||||
|
||||
if (url.includes('mfa/targets')) {
|
||||
state.push('MFA required, selecting SMS option');
|
||||
console.log('MFA required, please complete the authentication process.');
|
||||
await page.locator('aria/SMS: (XXX) XXX-4029').click();
|
||||
await page.waitForNavigation();
|
||||
//need to do some stuff ehre
|
||||
await page.keyboard.press('Tab');
|
||||
state.push('Waiting for code input');
|
||||
needCode = true;
|
||||
for (let i = 0; i < 5 * 60; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
if (code != null) break;
|
||||
}
|
||||
if (code == null) {
|
||||
needCode = false;
|
||||
state.push('Code not provided within 5 minutes');
|
||||
throw new Error('Code not provided within 5 minutes');
|
||||
}
|
||||
state.push(`Got code: ${code}`);
|
||||
|
||||
await page.keyboard.type(code);
|
||||
code = null;
|
||||
needCode = false;
|
||||
await page.keyboard.press('Enter');
|
||||
await page.locator('aria/Register Device').click();
|
||||
await page.waitForNavigation();
|
||||
}
|
||||
|
||||
state.push('Saving cookies');
|
||||
let cookies = await browser.cookies();
|
||||
|
||||
cookies.forEach(async (cookie) => {
|
||||
state.push('Saving cookie: ' + cookie.name);
|
||||
|
||||
// Insert or update the cookie in the database
|
||||
await db`DELETE FROM cookies WHERE name = ${cookie.name}`;
|
||||
|
||||
await db`INSERT INTO cookies (name, value, domain, path, expires, size, httpOnly, secure, session, priority, sameParty, sourceScheme, sourcePort)
|
||||
VALUES (${cookie.name}, ${cookie.value}, ${cookie.domain}, ${cookie.path}, ${cookie.expires}, ${cookie.size}, ${cookie.httpOnly}, ${cookie.secure}, ${cookie.session}, ${cookie.priority}, ${cookie.sameParty}, ${cookie.sourceScheme}, ${cookie.sourcePort})`;
|
||||
});
|
||||
}
|
||||
|
||||
state.push('Fetching q2token');
|
||||
const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value;
|
||||
|
||||
console.log('q2token:', q2token);
|
||||
|
||||
page.setExtraHTTPHeaders({
|
||||
q2token: q2token
|
||||
});
|
||||
|
||||
const accountsToPull = [
|
||||
{
|
||||
id: '65700',
|
||||
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
||||
},
|
||||
{
|
||||
id: '497016',
|
||||
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
||||
},
|
||||
{
|
||||
id: '1417342',
|
||||
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
||||
},
|
||||
{
|
||||
id: '83851',
|
||||
url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
|
||||
}
|
||||
];
|
||||
|
||||
const accountsURL =
|
||||
'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts';
|
||||
await page.goto(accountsURL);
|
||||
|
||||
const accountsJsonResponse = await page.evaluate(() => {
|
||||
return JSON.parse(document.querySelector('pre').textContent);
|
||||
});
|
||||
|
||||
for (const account of accountsJsonResponse.data) {
|
||||
console.log(`Account ID: ${account.id}, Name: ${account.name}`);
|
||||
if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) {
|
||||
const balance =
|
||||
account.id == '1417342' || account.id == '83851'
|
||||
? '-' + account.extended?.balance1
|
||||
: account.extended?.balance1;
|
||||
await db`INSERT INTO account (id, balance, balance_date)
|
||||
VALUES (${account.id}, ${balance}, ${account.dataAsOfDate})
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
balance = EXCLUDED.balance,
|
||||
balance_date = EXCLUDED.balance_date`;
|
||||
console.log(`Account ID: ${account.id}, Balance: ${balance}`);
|
||||
}
|
||||
}
|
||||
|
||||
state.push('Fetching transactions for accounts');
|
||||
for (const account of accountsToPull) {
|
||||
state.push(`Fetching transactions for account ID: ${account.id}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
|
||||
|
||||
await page.goto(account.url);
|
||||
|
||||
const jsonResponse = await page.evaluate(() => {
|
||||
return JSON.parse(document.querySelector('pre').textContent);
|
||||
});
|
||||
|
||||
if (!jsonResponse || !jsonResponse?.data?.transactions) {
|
||||
throw new Error(`No data found for account ID: ${account.id}`);
|
||||
}
|
||||
|
||||
const transactions = jsonResponse.data.transactions;
|
||||
if (transactions.length === 0) {
|
||||
state.push(`No transactions found for account ID: ${account.id}`);
|
||||
continue;
|
||||
}
|
||||
state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`);
|
||||
|
||||
const cardRegEx = /\d{4}$/;
|
||||
for (const transaction of transactions) {
|
||||
const amount = Number(transaction.amount);
|
||||
const date = new Date(transaction.postedDate);
|
||||
const id = transaction.transactionId;
|
||||
const payee = transaction.description || '';
|
||||
const statementDescription = transaction.statementDescription;
|
||||
const card = cardRegEx.test(statementDescription)
|
||||
? statementDescription.match(cardRegEx)[0]
|
||||
: null;
|
||||
const pending = transaction.extended?.allTransactionType == 1 ? true : false;
|
||||
const accountId = transaction.accountId;
|
||||
await db`INSERT INTO transaction (
|
||||
id,
|
||||
account_id,
|
||||
amount,
|
||||
description,
|
||||
date,
|
||||
payee,
|
||||
statement_description,
|
||||
card,
|
||||
pending)
|
||||
VALUES (
|
||||
${id},
|
||||
${accountId},
|
||||
${amount},
|
||||
${statementDescription},
|
||||
${date},
|
||||
${payee},
|
||||
${statementDescription},
|
||||
(SELECT id from cards where card_number=${card}),
|
||||
${pending})
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
amount = EXCLUDED.amount,
|
||||
description = EXCLUDED.description,
|
||||
card = EXCLUDED.card,
|
||||
date = EXCLUDED.date,
|
||||
payee = EXCLUDED.payee,
|
||||
statement_description = EXCLUDED.statement_description,
|
||||
pending = EXCLUDED.pending`;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error in pullData:', error);
|
||||
state.push(`Error: ${error.message}`);
|
||||
}
|
||||
|
||||
running = false;
|
||||
}
|
||||
Reference in New Issue
Block a user