Moved to new united api

package.json

@@ -29,6 +29,9 @@
 		"@tailwindcss/vite": "^4.1.11",
 		"cron": "^4.3.2",
 		"postgres": "^3.4.7",
+		"puppeteer": "^24.14.0",
+		"puppeteer-extra": "^3.3.6",
+		"puppeteer-extra-plugin-stealth": "^2.11.2",
 		"tailwindcss": "^4.1.11"
 	}
 }

src/hooks.server.js

@@ -1,10 +1,10 @@
 import { CronJob } from 'cron';
-import { fetchAccounts } from '$lib/simplefin';
 
+/*
 const job = new CronJob(
 	'10 0 * * * *', // cronTime
 	async function () {
-		const statDate = Math.floor(
+		const startDate = Math.floor(
 			new Date(new Date().getFullYear(), new Date().getMonth()).getTime() / 1000
 		);
 		const res = await fetchAccounts(startDate);
@@ -15,3 +15,4 @@ const job = new CronJob(
 	true, // start
 	'America/Detroit' // timeZone
 );
+*/

src/lib/db.js

@@ -84,10 +84,11 @@ export async function getBudgetTransactions(id) {
 		let transactions = await db`
         select
             transaction.id as id,
-            transaction.posted as posted,
+            transaction.account_id as account_id,
-            transaction.amount as amount,
             transaction.description as description,
-            transaction.pending as pending, 
+            transaction.pending as pending,
+            transaction.amount as amount,
+            transaction.date as date,
             budget_transaction.notes as notes,
             transaction.payee as payee,
             budget_transaction.amount as budget_amount,
@@ -98,10 +99,6 @@ export async function getBudgetTransactions(id) {
         order by transaction.posted desc
     `;
 		console.log(`Fetched ${transactions.length} transactions for budget ${id}`);
-		transactions = transactions.map((t) => ({
-			...t,
-			date: new Date(t.posted * 1000)
-		}));
 		// transactions = Result [{ id: 1, posted: 1633036800, amount: 50.00, description: "Grocery Store", pending: false, notes: "Weekly groceries" }, ...]
 		return { transactions };
 	} catch {
@@ -207,20 +204,13 @@ export async function getDeletedBudgets() {
 export async function getAccount(id) {
 	const account = await db`
     select
-      account.id as id,
+        account.id as id,
-      account.name as name,
+        account.name as name,
-      account.balance as balance,
+        account.balance as balance,
-      account.available_balance as available_balance,
+        account.in_total as in_total,
-      account.balance_date as balance_date,
+        account.balance_date as balance_date,
-      account.in_total as in_total,
+        account.hide as hide
-      account.hide as hide,
+    FROM account
-      org.id as org_id,
-      org.name as org_name,
-      org.domain as org_domain,
-      org.sfin_url as org_sfin_url,
-      org.url as org_url
-    from account
-    left join org on account.org_id = org.id
     where account.id = ${id}
   `;
 	if (!account || account.length === 0) {
@@ -272,20 +262,19 @@ export async function getTransactions(accountId) {
 	let transactions = await db`
     select
       transaction.id as id,
-      transaction.posted as posted,
+      transaction.account_id as account_id,
       transaction.amount as amount,
       transaction.description as description,
       transaction.pending as pending, 
       transaction.notes as notes,
-      transaction.payee as payee
+      transaction.payee as payee,
+      transaction.date as date,
+      transaction.statement_description as statement_description
     from transaction
     where account_id = ${accountId}
-    order by posted desc
+    order by date desc
   `;
-	transactions = transactions.map((t) => ({
+
-		...t,
-		date: new Date(t.posted * 1000)
-	}));
 	return transactions;
 }
 
@@ -298,100 +287,6 @@ export async function setTransactionNote(transactionId, note) {
 	return result;
 }
 
-export async function updateAccounts(data) {
-	try {
-		console.log('Updating accounts with data:', data);
-		for (const account of data.accounts) {
-			// Upsert Org
-			console.log(`Upserting org for account: ${account.id}`, account.org);
-			await db`
-                insert into org (id, domain, name, sfin_url, url)
-                values (${account.org.id}, ${account.org.domain ?? null}, ${account.org.name ?? null}, ${account.org.sfin_url ?? null}, ${account.org.url ?? null})
-                on conflict (id) do update set
-                    domain = excluded.domain,
-                    name = excluded.name,
-                    sfin_url = excluded.sfin_url,
-                    url = excluded.url
-            `;
-			console.log(`Upserting account: ${account.id} (${account.name})`);
-			// Upsert Account
-			await db`
-                insert into account (id, org_id, name, currency, balance, available_balance, balance_date)
-                values (
-                    ${account.id},
-                    ${account.org.id},
-                    ${account.name ?? null},
-                    ${account.currency ?? null},
-                    ${account.balance ?? null},
-                    ${account.available_balance ?? null},
-                    ${account.balance_date ?? null}                    
-                )
-                on conflict (id) do update set
-                    org_id = excluded.org_id,
-                    name = excluded.name,
-                    currency = excluded.currency,
-                    balance = excluded.balance,
-                    available_balance = excluded.available_balance,
-                    balance_date = excluded.balance_date
-            `;
-
-			// Upsert Transactions
-			if (account.transactions && account.transactions.length > 0) {
-				for (const txn of account.transactions) {
-					let extraId = null;
-					console.log(`Upserting transaction: ${txn.id} for account: ${account.id}`);
-					if (txn.extra) {
-						// Upsert TransactionExtra (insert only, update not needed for category)
-						const extraResult = await db`
-                            insert into transaction_extra (category)
-                            values (${txn.extra.category ?? null})
-                            on conflict (category) do nothing
-                            returning id
-                        `;
-						if (extraResult.length > 0) {
-							extraId = extraResult[0].id;
-						} else {
-							// If already exists, fetch id
-							const existing = await db`
-                                select id from transaction_extra where category = ${txn.extra.category ?? null}
-                            `;
-							if (existing.length > 0) {
-								extraId = existing[0].id;
-							}
-						}
-					}
-					console.log(`Preparing to upsert transaction: ${txn.id} with data:`, txn);
-					await db`
-                        insert into transaction (id, account_id, posted, amount, description, pending, transacted_at, payee)
-                        values (
-                            ${txn.id},
-                            ${account.id},
-                            ${txn.posted},
-                            ${txn.amount ?? null},
-                            ${txn.description ?? null},
-                            ${txn.pending ?? false},
-                            ${txn.transacted_at ?? 0},
-                            ${txn.payee ?? null}
-                        )
-                        on conflict (id) do update set
-                            account_id = excluded.account_id,
-                            posted = excluded.posted,
-                            amount = excluded.amount,
-                            description = excluded.description,
-                            pending = excluded.pending,
-                            transacted_at = excluded.transacted_at,
-                            payee = excluded.payee
-                    `;
-				}
-			}
-		}
-		return true;
-	} catch (error) {
-		console.error('updateAccounts error:', error);
-		return false;
-	}
-}
-
 export async function getRules(data) {
 	try {
 		const rules = await db`

src/lib/simplefin.js

@@ -1,17 +0,0 @@
-const url =
-	'https://19443E0E8171E175EC5DA0C69B35DD50197F234B9A74C00D27FD606121257ECF:DAA3702E2100CFFD3B544251E6D755E86B1EDDFBFCC7F6FA9CE77AB3677E60DE@beta-bridge.simplefin.org/simplefin';
-
-export async function fetchAccounts(startDate) {
-	const { username, password, origin, pathname } = new URL(url);
-	const apiUrl = `${origin}${pathname}/accounts?start-date=${startDate}`;
-	const headers = {};
-
-	console.log(`Fetching accounts from: ${apiUrl}`);
-
-	if (username && password) {
-		headers['Authorization'] = 'Basic ' + btoa(`${username}:${password}`);
-	}
-
-	const response = await fetch(apiUrl, { headers });
-	return await response.json();
-}

src/lib/united.js

@@ -0,0 +1,257 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import db from './db.js';
+import { json } from '@sveltejs/kit';
+
+puppeteer.use(StealthPlugin());
+let state = [];
+let code = null;
+let needCode = false;
+let running = false;
+
+export function getState() {
+	return state;
+}
+export function setCode(newCode) {
+	code = newCode;
+}
+
+export function isNeedCode() {
+	return needCode;
+}
+
+export function isRunning() {
+	return running;
+}
+
+export async function pullData(amount = 100) {
+	running = true;
+	try {
+		state = [];
+		code = null;
+		needCode = false;
+		state.push('Starting Browser');
+		const browser = await puppeteer.launch({
+			//headless: false
+			//defaultViewport: null,
+			//args: ['--disable-blink-features=PrettyPrintJSONDocument']
+		});
+
+		const page = await browser.newPage();
+
+		state.push('Loading Cookies');
+
+		const cookiesdb = await db`SELECT 
+            name, 
+            value, 
+            domain, 
+            path, 
+            expires, 
+            size,
+            httpOnly,
+            secure,
+            session,
+            priority,
+            sameParty,
+            sourceScheme,
+            sourcePort
+        FROM cookies`;
+
+		cookiesdb.forEach((cookie) => {
+			cookie.expires = cookie.expires ? Number(cookie.expires) : undefined;
+			cookie.size = cookie.size ? Number(cookie.size) : undefined;
+			cookie.sourcePort = cookie.sourcePort ? Number(cookie.sourcePort) : undefined;
+			state.push('Loading cookie: ' + cookie.name);
+			browser.setCookie(cookie);
+		});
+
+		state.push('Navigating to United FCU');
+
+		// Navigate the page to a URL.
+		await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login');
+
+		if (page.url().includes('interstitial')) {
+			await page.waitForNavigation();
+			state.push('Already logged in, navigating to dashboard');
+		}
+
+		if (!page.url().includes('dashboard')) {
+			state.push('Logging in to United FCU');
+			// Type into search box using accessible input name.
+			await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
+			await page.locator('aria/Login ID').fill('92830');
+			await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
+
+			await page.locator('aria/Password').fill('Cmtjlt13');
+			await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
+			await page.keyboard.press('Enter');
+
+			await page.waitForNavigation();
+			const url = page.url();
+			console.log('Current URL:', url);
+
+			if (url.includes('mfa/targets')) {
+				state.push('MFA required, selecting SMS option');
+				console.log('MFA required, please complete the authentication process.');
+				await page.locator('aria/SMS: (XXX) XXX-4029').click();
+				await page.waitForNavigation();
+				//need to do some stuff ehre
+				await page.keyboard.press('Tab');
+				state.push('Waiting for code input');
+				needCode = true;
+				for (let i = 0; i < 5 * 60; i++) {
+					await new Promise((resolve) => setTimeout(resolve, 1000));
+					if (code != null) break;
+				}
+				if (code == null) {
+					needCode = false;
+					state.push('Code not provided within 5 minutes');
+					throw new Error('Code not provided within 5 minutes');
+				}
+				state.push(`Got code: ${code}`);
+
+				await page.keyboard.type(code);
+				code = null;
+				needCode = false;
+				await page.keyboard.press('Enter');
+				await page.locator('aria/Register Device').click();
+				await page.waitForNavigation();
+			}
+
+			state.push('Saving cookies');
+			let cookies = await browser.cookies();
+
+			cookies.forEach(async (cookie) => {
+				state.push('Saving cookie: ' + cookie.name);
+
+				// Insert or update the cookie in the database
+				await db`DELETE FROM cookies WHERE name = ${cookie.name}`;
+
+				await db`INSERT INTO cookies (name, value, domain, path, expires, size, httpOnly, secure, session, priority, sameParty, sourceScheme, sourcePort)
+                VALUES (${cookie.name}, ${cookie.value}, ${cookie.domain}, ${cookie.path}, ${cookie.expires}, ${cookie.size}, ${cookie.httpOnly}, ${cookie.secure}, ${cookie.session}, ${cookie.priority}, ${cookie.sameParty}, ${cookie.sourceScheme}, ${cookie.sourcePort})`;
+			});
+		}
+
+		state.push('Fetching q2token');
+		const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value;
+
+		console.log('q2token:', q2token);
+
+		page.setExtraHTTPHeaders({
+			q2token: q2token
+		});
+
+		const accountsToPull = [
+			{
+				id: '65700',
+				url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/65700?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
+			},
+			{
+				id: '497016',
+				url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/497016?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
+			},
+			{
+				id: '1417342',
+				url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountHistory/1417342?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
+			},
+			{
+				id: '83851',
+				url: `https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accountPfm/83851/history?page[number]=1&page[size]=${amount}&sort=postedDate%1Fd`
+			}
+		];
+
+		const accountsURL =
+			'https://online.unitedfcu.com/UnitedFederalCreditUnionOnline/mobilews/accounts';
+		await page.goto(accountsURL);
+
+		const accountsJsonResponse = await page.evaluate(() => {
+			return JSON.parse(document.querySelector('pre').textContent);
+		});
+
+		for (const account of accountsJsonResponse.data) {
+			console.log(`Account ID: ${account.id}, Name: ${account.name}`);
+			if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) {
+				const balance =
+					account.id == '1417342' || account.id == '83851'
+						? '-' + account.extended?.balance1
+						: account.extended?.balance1;
+				await db`INSERT INTO account (id, balance, balance_date)
+                VALUES (${account.id}, ${balance}, ${account.dataAsOfDate})
+                ON CONFLICT (id) DO UPDATE SET
+                    balance = EXCLUDED.balance,
+                    balance_date = EXCLUDED.balance_date`;
+				console.log(`Account ID: ${account.id}, Balance: ${balance}`);
+			}
+		}
+
+		state.push('Fetching transactions for accounts');
+		for (const account of accountsToPull) {
+			state.push(`Fetching transactions for account ID: ${account.id}`);
+			await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
+
+			await page.goto(account.url);
+
+			const jsonResponse = await page.evaluate(() => {
+				return JSON.parse(document.querySelector('pre').textContent);
+			});
+
+			if (!jsonResponse || !jsonResponse?.data?.transactions) {
+				throw new Error(`No data found for account ID: ${account.id}`);
+			}
+
+			const transactions = jsonResponse.data.transactions;
+			if (transactions.length === 0) {
+				state.push(`No transactions found for account ID: ${account.id}`);
+				continue;
+			}
+			state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`);
+
+			const cardRegEx = /\d{4}$/;
+			for (const transaction of transactions) {
+				const amount = Number(transaction.amount);
+				const date = new Date(transaction.postedDate);
+				const id = transaction.transactionId;
+				const payee = transaction.description || '';
+				const statementDescription = transaction.statementDescription;
+				const card = cardRegEx.test(statementDescription)
+					? statementDescription.match(cardRegEx)[0]
+					: null;
+				const pending = transaction.extended?.allTransactionType == 1 ? true : false;
+				const accountId = transaction.accountId;
+				await db`INSERT INTO transaction (
+                            id, 
+                            account_id, 
+                            amount, 
+                            description, 
+                            date, 
+                            payee, 
+                            statement_description, 
+                            card, 
+                            pending)
+                    VALUES (
+                            ${id}, 
+                            ${accountId}, 
+                            ${amount}, 
+                            ${statementDescription}, 
+                            ${date}, 
+                            ${payee}, 
+                            ${statementDescription}, 
+                            (SELECT id from cards where card_number=${card}), 
+                            ${pending})
+                    ON CONFLICT (id) DO UPDATE SET 
+                            amount = EXCLUDED.amount,
+                            description = EXCLUDED.description,
+                            card = EXCLUDED.card,
+                            date = EXCLUDED.date,
+                            payee = EXCLUDED.payee,
+                            statement_description = EXCLUDED.statement_description,
+                            pending = EXCLUDED.pending`;
+			}
+		}
+	} catch (error) {
+		console.error('Error in pullData:', error);
+		state.push(`Error: ${error.message}`);
+	}
+
+	running = false;
+}

src/routes/account/[slug]/+page.server.js

@@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit';
 import { getAccount, getTransactions, getBudgets, getBudgetTransactionsForAccount } from '$lib/db';
 
 /** @type {import('./$types').PageServerLoad} */
-export async function load({ params }) {
+export async function load({ params, depends }) {
 	const slug = params.slug;
 	const transactions = await getTransactions(slug);
 	const account = await getAccount(slug);

src/routes/account/[slug]/+page.svelte

@@ -1,6 +1,7 @@
 <script>
 	import { EditSymbol } from '$lib/editSymbol.svelte';
 	import { settingsSymbol } from '$lib/settingsSymbol.svelte';
+	import { invalidate, invalidateAll } from '$app/navigation';
 	let { data } = $props();
 	let trans = $derived(data.transactions);
 	let budgets = $derived(data.budgets);
@@ -10,6 +11,7 @@
 	let account = $derived(data.account);
 	let hide = $derived(account?.hide || false);
 	let inTotal = $derived(account?.in_total || false);
+	let expanded = $state([]);
 
 	function editNotes(transaction) {
 		my_modal_3.showModal();
@@ -61,7 +63,9 @@
 	</div>
 	<div class="w-64 grow">{account?.balance}</div>
 	<div class="w-14 flex-none text-right">
-		<button class="btn btn-square btn-ghost">{@render settingsSymbol()} </button>
+		<button class="btn btn-square btn-ghost" onclick={() => settings_modal.showModal()}
+			>{@render settingsSymbol()}
+		</button>
 	</div>
 </div>
 
@@ -160,8 +164,8 @@
 			<button
 				class="btn btn-primary"
 				onclick={() => {
-					if (budget_id) {
+					if (currentTransaction.budget_id) {
-						fetch(`/api/budget/${budget_id}/transaction`, {
+						fetch(`/api/budget/${currentTransaction.budget_id}/transaction`, {
 							method: 'POST',
 							headers: {
 								'Content-Type': 'application/json'

src/routes/api/simplefin/update/+server.js

@@ -1,17 +0,0 @@
-import { fetchAccounts } from '$lib/simplefin';
-import { runRules, updateAccounts } from '$lib/db';
-
-export async function POST({ request }) {
-	let body = null;
-	try {
-		let body = await request.json();
-	} catch (error) {}
-	const startDate = body?.startDate
-		? body
-		: Math.floor(new Date(new Date().getFullYear(), new Date().getMonth() - 1).getTime() / 1000);
-
-	const res = await fetchAccounts(startDate);
-	await updateAccounts(res);
-	await runRules();
-	return new Response(`Accounts updated successfully`, { status: 200 });
-}

src/routes/api/united/code/+server.js

@@ -0,0 +1,14 @@
+import { setCode, isNeedCode } from '$lib/united';
+
+export async function POST({ request }) {
+	let body = await request.json();
+	const code = body.code;
+	if (!code) {
+		return new Response(`Code is required`, { status: 400 });
+	}
+	if (!isNeedCode()) {
+		return new Response(`Code not needed`, { status: 400 });
+	}
+	setCode(code);
+	return new Response(`Started`);
+}

src/routes/api/united/status/+server.js

@@ -0,0 +1,13 @@
+import { getState, isRunning, isNeedCode } from '$lib/united';
+
+export function GET({ params }) {
+	const state = getState();
+
+	return new Response(
+		JSON.stringify({
+			state: state,
+			isRunning: isRunning(),
+			isNeedCode: isNeedCode()
+		})
+	);
+}

src/routes/api/united/update/+server.js

@@ -0,0 +1,11 @@
+import { pullData, isRunning } from '$lib/united';
+
+export async function POST({ request }) {
+	let body = await request.json();
+	const count = body.count || 100;
+	if (isRunning()) {
+		return new Response(`Already running`, { status: 400 });
+	}
+	pullData(count);
+	return new Response(`Started`);
+}

src/routes/budget/[slug]/+page.svelte

@@ -26,7 +26,7 @@
 	}
 
 	function edit(transaction) {
-		newData.amount = transaction.amount;
+		newData.amount = transaction.budget_amount;
 		newData.notes = transaction.notes || '';
 		EditBudgetTransactionModal.showModal();
 	}

src/routes/united/+page.svelte

@@ -0,0 +1,78 @@
+<script>
+	import { onMount } from 'svelte';
+
+	let isRunning = $state(false);
+	let needCode = $state(false);
+	let state = $state(false);
+	let code = $state('');
+	let count = $state(100);
+
+	function checkStatus() {
+		fetch('/api/united/status')
+			.then((response) => response.json())
+			.then((data) => {
+				isRunning = data.isRunning;
+				needCode = data.isNeedCode;
+				state = data.state;
+				setTimeout(checkStatus, 1000);
+			})
+			.catch((error) => console.error('Error fetching status:', error));
+	}
+
+	onMount(() => {
+		checkStatus();
+	});
+</script>
+
+{#if isRunning}
+	<span>Fetching Data</span>
+{:else}
+	<input
+		type="text"
+		bind:value={count}
+		placeholder="Number of transactions"
+		class="input input-bordered w-full max-w-xs"
+	/>
+	<button
+		class="btn btn-primary"
+		onclick={() => {
+			fetch('/api/united/update', {
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify({ count })
+			});
+		}}>Fetch Data</button
+	>
+{/if}
+
+<div class="mockup-code w-full">
+	{#each state as line, i}
+		<pre data-prefix={i}><code>{line}</code></pre>
+	{/each}
+</div>
+
+{#if needCode}
+	<div class="alert alert-warning">
+		<span>Need Code</span>
+		<input
+			type="text"
+			bind:value={code}
+			placeholder="Enter code"
+			class="input input-bordered w-full max-w-xs"
+		/>
+		<button
+			class="btn btn-primary"
+			onclick={() => {
+				fetch('/api/united/code', {
+					method: 'POST',
+					headers: { 'Content-Type': 'application/json' },
+					body: JSON.stringify({ code })
+				}).then(() => {
+					code = '';
+				});
+			}}
+		>
+			Send Code
+		</button>
+	</div>
+{/if}