From d9b1a8430dcba7cb26afb876a4da2368fdd4d7d8 Mon Sep 17 00:00:00 2001 From: Casey Timm Date: Sat, 26 Jul 2025 08:34:40 -0400 Subject: [PATCH] Added logging to united --- package-lock.json | 485 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 4 +- src/lib/logger.js | 19 ++ src/lib/united.js | 37 +++- 4 files changed, 536 insertions(+), 9 deletions(-) create mode 100644 src/lib/logger.js diff --git a/package-lock.json b/package-lock.json index 8176700..41f9e5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@auth/sveltekit": "^1.10.0", + "@pauleliet/winston-pg-native": "^4.4.0", "@tailwindcss/vite": "^4.1.11", "cron": "^4.3.2", "echarts": "^5.6.0", @@ -17,7 +18,8 @@ "puppeteer": "^24.14.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", - "tailwindcss": "^4.1.11" + "tailwindcss": "^4.1.11", + "winston": "^3.17.0" }, "devDependencies": { "@sveltejs/adapter-auto": "^6.0.0", @@ -1643,6 +1645,24 @@ "node": ">=6.9.0" } }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.5.tgz", @@ -2110,6 +2130,24 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/@pauleliet/winston-pg-native": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@pauleliet/winston-pg-native/-/winston-pg-native-4.4.0.tgz", + "integrity": "sha512-13WxcAsWPizPo1tD5tEmRIbwh/Anvzj/lWxhFSmvLxi5BCK87nThGWjt71Uu5PZ245ghGqR2B/pQr9G+l4ZKUw==", + "dependencies": { + "abstract-winston-transport": "^0.5.1", + "flatted": "^2.0.0", + "moment": "^2.23.0", + "pg": "^7.8.0", + "pg-native": "^3.0.0", + "sql": "^0.78.0", + "winston": "^3.0.0", + "winston-transport": "^4.3.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -3025,6 +3063,11 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -3069,6 +3112,14 @@ } } }, + "node_modules/abstract-winston-transport": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/abstract-winston-transport/-/abstract-winston-transport-0.5.1.tgz", + "integrity": "sha512-4w7EPMd7K7kzZFS6lKTbMFmXV+nWontTIzpF2Xc0HFns6RdNOLIOgbKiQzVw+46ghIoweZ4miIMFxmgapJO93A==", + "dependencies": { + "diagnostics": "^1.1.0" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -3210,7 +3261,6 @@ "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "dev": true, "license": "MIT" }, "node_modules/async-function": { @@ -3393,6 +3443,14 @@ "node": ">=10.0.0" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -3452,6 +3510,14 @@ "devOptional": true, "license": "MIT" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -3610,6 +3676,15 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -3628,6 +3703,42 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/colornames": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz", + "integrity": "sha512-/pyV40IrsdulWv+wFPmERh9k/mjsPZ64yUMDmWrtj/k1nmgrzzIENWKdaVKyBbvFdQWqkcaRxr+polCo3VMe7A==" + }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -3920,6 +4031,32 @@ "integrity": "sha512-CAoP3lYfwAGQTaAXYvA6JZR0fjGUb7qec1qf4mToyoH2TZgUFeIqYcjh6f9jNuhHfuZiEdH+PONHYrLhRQX6aw==", "license": "BSD-3-Clause" }, + "node_modules/diagnostics": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", + "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "1.0.x", + "kuler": "1.0.x" + } + }, + "node_modules/diagnostics/node_modules/enabled": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz", + "integrity": "sha512-nnzgVSpB35qKrUN8358SjO1bYAmxoThECTWw9s3J0x5G8A9hokKHVDFzBjVpCoSryo6MhN8woVyascN5jheaNA==", + "dependencies": { + "env-variable": "0.0.x" + } + }, + "node_modules/diagnostics/node_modules/kuler": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz", + "integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==", + "dependencies": { + "colornames": "^1.1.1" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -3978,6 +4115,11 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -4009,6 +4151,11 @@ "node": ">=6" } }, + "node_modules/env-variable": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", + "integrity": "sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==" + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -4367,6 +4514,16 @@ } } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -4400,6 +4557,16 @@ "node": ">=10" } }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -5214,7 +5381,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5493,6 +5659,11 @@ "dev": true, "license": "MIT" }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "node_modules/lazy-cache": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", @@ -5512,6 +5683,16 @@ "node": ">=6" } }, + "node_modules/libpq": { + "version": "1.8.15", + "resolved": "https://registry.npmjs.org/libpq/-/libpq-1.8.15.tgz", + "integrity": "sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==", + "hasInstallScript": true, + "dependencies": { + "bindings": "1.5.0", + "nan": "~2.22.2" + } + }, "node_modules/lightningcss": { "version": "1.30.1", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.1.tgz", @@ -5794,6 +5975,22 @@ "dev": true, "license": "MIT" }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -5929,6 +6126,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "engines": { + "node": "*" + } + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -5953,6 +6158,11 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/nan": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", + "integrity": "sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==" + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -6049,6 +6259,14 @@ "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/own-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz", @@ -6099,6 +6317,11 @@ "node": ">= 14" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -6195,6 +6418,90 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pg": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", + "integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.10", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha512-i0NV/CrSkFTaiOQs9AGy3tq0dkSjtTd4d7DfsjeDVZAA4aIHInwfFEmriNYGGJUfZ5x6IAC/QddoUpUJjQAi0w==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-native": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-native/-/pg-native-3.5.2.tgz", + "integrity": "sha512-3oi+KVil86Vngo4H0IlhBaYSJWdcu8t2f1Y4TkQoQi5oZ9bNeYECGqW3oSGx69mjSZYHoC3h+3jYtqzRgndn5A==", + "dependencies": { + "libpq": "^1.8.15", + "pg-types": "2.2.0" + } + }, + "node_modules/pg-packet-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pg-packet-stream/-/pg-packet-stream-1.1.0.tgz", + "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" + }, + "node_modules/pg-pool": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", + "integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==", + "peerDependencies": { + "pg": ">5.0" + } + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pg/node_modules/semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -6278,6 +6585,41 @@ "url": "https://github.com/sponsors/porsager" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/preact": { "version": "10.24.3", "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.3.tgz", @@ -6598,6 +6940,19 @@ "safe-buffer": "^5.1.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -6853,7 +7208,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -6905,6 +7259,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -7092,6 +7454,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/sirv": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.1.tgz", @@ -7106,6 +7481,11 @@ "node": ">=18" } }, + "node_modules/sliced": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-0.0.5.tgz", + "integrity": "sha512-9bYT917D6H3+q8GlQBJmLVz3bc4OeVGfZ2BB12wvLnluTGfG6/8UdOUbKJDW1EEx9SZMDbjnatkau5/XcUeyOw==" + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -7202,12 +7582,45 @@ "dev": true, "license": "MIT" }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", "license": "BSD-3-Clause" }, + "node_modules/sql": { + "version": "0.78.0", + "resolved": "https://registry.npmjs.org/sql/-/sql-0.78.0.tgz", + "integrity": "sha512-+If27aF3UQxmnuF5JgTJKd/X6TiELa1f58Zo5TX9ULbPnqs1fBDODwwbtU0WwM5H3lSiwx2En4m2lB1rCoIhvQ==", + "dependencies": { + "lodash": "4.1.x", + "sliced": "0.0.x" + }, + "engines": { + "node": "*" + } + }, + "node_modules/sql/node_modules/lodash": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.1.0.tgz", + "integrity": "sha512-B9sgtKUlz0xe7lkYb80BcOpwwJJw5iOiz4HkBDzF0+i5nJLiwfBnL08m7bBkCOPBfi+0aqvrJDMdZDfAvs8vYg==" + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "engines": { + "node": "*" + } + }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", @@ -7235,6 +7648,14 @@ "bare-events": "^2.2.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -7538,6 +7959,11 @@ "b4a": "^1.6.4" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "node_modules/tinyglobby": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz", @@ -7573,6 +7999,14 @@ "punycode": "^2.1.0" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -7814,7 +8248,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/vite": { @@ -8048,6 +8481,40 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/winston": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/workbox-background-sync": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/workbox-background-sync/-/workbox-background-sync-7.3.0.tgz", @@ -8421,6 +8888,14 @@ } } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 9ed9469..5a53372 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ }, "dependencies": { "@auth/sveltekit": "^1.10.0", + "@pauleliet/winston-pg-native": "^4.4.0", "@tailwindcss/vite": "^4.1.11", "cron": "^4.3.2", "echarts": "^5.6.0", @@ -34,6 +35,7 @@ "puppeteer": "^24.14.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", - "tailwindcss": "^4.1.11" + "tailwindcss": "^4.1.11", + "winston": "^3.17.0" } } diff --git a/src/lib/logger.js b/src/lib/logger.js new file mode 100644 index 0000000..29a4124 --- /dev/null +++ b/src/lib/logger.js @@ -0,0 +1,19 @@ +const { Logger } = require('winston'); +const Postgres = require('winston-pg-native'); + +const logger = new Logger({ + transports: [ + new Postgres({ + 'postgres://budget:budget@sql.caseytimm.com:5432/budget', + level: 'info', + poolConfig: { + connectionTimeoutMillis: 0, + idleTimeoutMillis: 0, + max: 10 + }, + tableName: 'winston_logs' + }) + ] +}); + +module.exports = logger; \ No newline at end of file diff --git a/src/lib/united.js b/src/lib/united.js index c3182f9..096534d 100644 --- a/src/lib/united.js +++ b/src/lib/united.js @@ -1,5 +1,7 @@ import db from './db.js'; import { chromium } from 'patchright'; +import logger from './logger.js'; +import { log } from 'echarts/types/src/util/log.js'; //const browser = await chromium.launch(); let state = []; @@ -21,6 +23,8 @@ export function isNeedCode() { export function isRunning() { return running; } + +logger.info('Launching browser for United FCU data pull'); const browser = await chromium.launchPersistentContext('context', { channel: 'chrome', headless: false, @@ -33,14 +37,16 @@ export async function pullData(amount = 100) { state = []; code = null; needCode = false; - state.push('Starting Browser'); - + logger.info('Starting data pull for United FCU'); + state.push('Starting data pull for United FCU'); const page = await browser.newPage(); try { + logger.info('Navigating to United FCU'); state.push('Navigating to United FCU'); await page.goto('https://online.unitedfcu.com/unitedfederalcredituniononline/uux.aspx#/login'); + logger.info(`Current URL: ${page.url()}`); state.push(`Current URL: ${page.url()}`); if (page.url().includes('interstitial')) { @@ -48,6 +54,7 @@ export async function pullData(amount = 100) { } if (!page.url().includes('dashboard')) { + logger.info('Logging in to United FCU'); state.push('Logging in to United FCU'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); await page.getByLabel('Login ID').fill('92830'); @@ -65,9 +72,11 @@ export async function pullData(amount = 100) { } const url = page.url(); + logger.info(`Current URL after login: ${url}`); state.push(`Current URL after login: ${url}`); if (url.includes('mfa/targets')) { + logger.info('MFA required, selecting SMS option'); state.push('MFA required, selecting SMS option'); console.log('MFA required, please complete the authentication process.'); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); @@ -75,6 +84,7 @@ export async function pullData(amount = 100) { await page.waitForURL('**/mfa/entertarget'); await page.getByPlaceholder('Secure Access Code'); //await page.keyboard.press('Tab'); + logger.info('Waiting for user to provide code'); state.push('Waiting for code input'); needCode = true; for (let i = 0; i < 5 * 60; i++) { @@ -84,8 +94,10 @@ export async function pullData(amount = 100) { if (code == null) { needCode = false; state.push('Code not provided within 5 minutes'); + logger.error('Code not provided within 5 minutes'); throw new Error('Code not provided within 5 minutes'); } + logger.info(`Got code: ${code}`); state.push(`Got code: ${code}`); await page.getByPlaceholder('Secure Access Code').fill(code); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); @@ -96,14 +108,16 @@ export async function pullData(amount = 100) { await page.keyboard.press('Tab'); await page.keyboard.press('Enter'); await page.waitForURL('**/dashboard'); - await page.screenshot({ path: 'united-login.png' }); + logger.info('Logged in successfully'); state.push('Logged in successfully'); } } + logger.info('Fetching q2token'); state.push('Fetching q2token'); const q2token = (await browser.cookies()).find((cookie) => cookie.name === 'q2token')?.value; + logger.info('q2token:', q2token); console.log('q2token:', q2token); page.setExtraHTTPHeaders({ @@ -138,6 +152,7 @@ export async function pullData(amount = 100) { }); for (const account of accountsJsonResponse.data) { + logger.info(`Account ID: ${account.id}, Name: ${account.name}`); console.log(`Account ID: ${account.id}, Name: ${account.name}`); if (accountsToPull.map((a) => a.id).includes(`${account.id}`)) { const balance = @@ -150,11 +165,14 @@ export async function pullData(amount = 100) { balance = EXCLUDED.balance, balance_date = EXCLUDED.balance_date`; console.log(`Account ID: ${account.id}, Balance: ${balance}`); + logger.info(`Account ID: ${account.id}, Balance: ${balance}`); } } + logger.info('Fetching transactions for accounts'); state.push('Fetching transactions for accounts'); for (const account of accountsToPull) { + logger.info(`Fetching transactions for account ID: ${account.id}`); state.push(`Fetching transactions for account ID: ${account.id}`); await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000)); @@ -170,34 +188,45 @@ export async function pullData(amount = 100) { const transactions = jsonResponse.data.transactions; if (transactions.length === 0) { + logger.info(`No transactions found for account ID: ${account.id}`); state.push(`No transactions found for account ID: ${account.id}`); continue; } + logger.info(`Found ${transactions.length} transactions for account ID: ${account.id}`); state.push(`Found ${transactions.length} transactions for account ID: ${account.id}`); const cardRegEx = /\d{4}$/; // Check if any pending need to be updated + logger.info(`Checking for pending transactions for account ID: ${account.id}`); state.push(`Checking for pending transactions for account ID: ${account.id}`); let currentPend = await db`SELECT id, amount, description, date from transaction where account_id = ${account.id} AND pending=true`; for (const pend of currentPend) { const found = transactions.find((t) => t.transactionId === pend.id); + logger.info(`Checking pending transaction: ${pend.id}, found: ${!!found}`); + logger.info(`Pending transaction amount: ${pend.amount}, date: ${pend.date}`); if (!found) { const updated = transactions.find( (t) => t.amount == pend.amount && new Date(t.postedDate) == pend.date ); + logger.info(`Updated transaction for pending: ${pend.id}, found: ${!!updated}`); + if (updated) { state.push( `I think I found an updated transaction: ${updated.statementDescription} ${updated.amount} for ${pend.description} ${pend.amount}` ); + logger.info(`I think I found an updated transaction: ${updated.statementDescription} ${updated.amount} for ${pend.description} ${pend.amount}`); + await db`UPDATE budget_transaction SET transaction_id = ${updated.transactionId} WHERE transaction_id = ${pend.id}`; } else { + state.push(`No updated transaction found for pending: ${pend.id}`); state.push(`Orphaning no longer pending budget transaction with no new parent`); await db`UPDATE budget_transaction SET transaction_id = null WHERE transaction_id = ${pend.id}`; } + logger.info(`Removing pending transaction: ${pend.id}`); state.push(`Removing pending transaction: ${pend.id}`); await db`DELETE FROM transaction WHERE id = ${pend.id}`; } @@ -249,6 +278,7 @@ export async function pullData(amount = 100) { } } + logger.info('Orphaning transactions'); state.push('Orphaning transactions'); const orphaned = await db`SELECT bt.id as id @@ -256,6 +286,7 @@ export async function pullData(amount = 100) { LEFT OUTER JOIN transaction t ON bt.transaction_id = t.id WHERE t.id IS NULL;`; for (const orphan of orphaned) { + logger.info(`Orphaning transaction: ${orphan.id}`); state.push(`Orphaning transaction: ${orphan.id}`); await db`UPDATE budget_transaction set transaction_id = null where id = ${orphan.id}`; }