From 427636b59bc85d64376a32e48a0b009e10425ae2 Mon Sep 17 00:00:00 2001 From: Nasyarobby Putra Date: Tue, 22 Sep 2026 10:09:25 +0700 Subject: [PATCH] feat(watcher): add Flip transfer receipt provider Match no-reply@flip.id transfer success mail (direct or forwarded), parse stacked Nominal/ID Transaksi fields, and map spendings with source=flip. Co-authored-by: Cursor --- .../src/handlers/parse-matched-message.js | 2 + apps/watcher/src/parsers/flip.js | 74 ++++++++++ apps/watcher/src/pocketbase/map.js | 2 + apps/watcher/src/rules/flip-transfer.js | 6 + apps/watcher/src/rules/index.js | 2 + apps/watcher/test/fixtures.js | 76 ++++++++++ apps/watcher/test/match.test.js | 28 ++++ apps/watcher/test/parse.test.js | 56 ++++++++ docs/development.md | 130 ++++++++++++++++++ 9 files changed, 376 insertions(+) create mode 100644 apps/watcher/src/parsers/flip.js create mode 100644 apps/watcher/src/rules/flip-transfer.js create mode 100644 docs/development.md diff --git a/apps/watcher/src/handlers/parse-matched-message.js b/apps/watcher/src/handlers/parse-matched-message.js index 413cb6f..2eb3034 100644 --- a/apps/watcher/src/handlers/parse-matched-message.js +++ b/apps/watcher/src/handlers/parse-matched-message.js @@ -1,4 +1,5 @@ import { parseBriQris, parseBriTransfer } from "../parsers/bri.js"; +import { parseFlipTransfer } from "../parsers/flip.js"; import { parseGrabReceipt } from "../parsers/grab.js"; import { parseLivinReceipt } from "../parsers/livin.js"; import { parseTokopediaOrder } from "../parsers/tokopedia.js"; @@ -10,6 +11,7 @@ const PARSERS = { "bri.transfer.success": parseBriTransfer, "bri.qris.success": parseBriQris, "tokopedia.order.completed": parseTokopediaOrder, + "flip.transfer.success": parseFlipTransfer, }; /** diff --git a/apps/watcher/src/parsers/flip.js b/apps/watcher/src/parsers/flip.js new file mode 100644 index 0000000..6b52d58 --- /dev/null +++ b/apps/watcher/src/parsers/flip.js @@ -0,0 +1,74 @@ +import { + captureBlockAfterLabel, + parseDayMonthYear, + parseIdrAmount, + parseTimeWib, + toWibIso, +} from "../lib/parse-helpers.js"; + +/** + * First non-empty line after a Flip stacked label (value on the next line). + */ +function valueAfterLabel(body, label, nextLabels = []) { + const lines = captureBlockAfterLabel(body, label, nextLabels); + return lines?.[0]?.trim() || null; +} + +/** + * Parse Flip transfer-success body (plain text, stacked label/value lines). + * Required: Nominal + ID Transaksi. + */ +export function parseFlipTransfer(body) { + if (!body) return null; + + const referenceRaw = valueAfterLabel(body, "ID Transaksi", [ + "Waktu Terkirim", + "Nama Tujuan", + ]); + const reference = referenceRaw + ? referenceRaw.replace(/^#/, "").trim() + : null; + + const waktuRaw = valueAfterLabel(body, "Waktu Terkirim", [ + "Nama Tujuan", + "Bank Tujuan", + ]); + const dateYmd = parseDayMonthYear(waktuRaw); + const timeHms = parseTimeWib(waktuRaw); + const occurredAt = toWibIso(dateYmd, timeHms); + + const amount = parseIdrAmount( + valueAfterLabel(body, "Nominal", [ + "Kalau ada pertanyaan", + "Makasih", + "Salam", + ]), + ); + + if (amount == null || !reference) return null; + + const namaTujuan = valueAfterLabel(body, "Nama Tujuan", [ + "Bank Tujuan", + "Nomor Rekening Tujuan", + ]); + const bankTujuan = valueAfterLabel(body, "Bank Tujuan", [ + "Nomor Rekening Tujuan", + "Nominal", + ]); + const nomorRekening = valueAfterLabel(body, "Nomor Rekening Tujuan", [ + "Nominal", + ]); + + const details = {}; + if (namaTujuan) details.namaTujuan = namaTujuan; + if (bankTujuan) details.bankTujuan = bankTujuan; + if (nomorRekening) details.nomorRekening = nomorRekening; + + return { + amount, + currency: "IDR", + reference, + occurredAt, + details, + }; +} diff --git a/apps/watcher/src/pocketbase/map.js b/apps/watcher/src/pocketbase/map.js index 50d440d..cdcaa27 100644 --- a/apps/watcher/src/pocketbase/map.js +++ b/apps/watcher/src/pocketbase/map.js @@ -24,6 +24,8 @@ export function toSpendingRecord(parsed, rule) { ? `${base} tip` : base; } + } else if (source === "flip") { + description = parsed.details?.namaTujuan || ""; } return { diff --git a/apps/watcher/src/rules/flip-transfer.js b/apps/watcher/src/rules/flip-transfer.js new file mode 100644 index 0000000..3580dad --- /dev/null +++ b/apps/watcher/src/rules/flip-transfer.js @@ -0,0 +1,6 @@ +export const flipTransfer = { + kind: "flip.transfer.success", + source: "flip", + fromAddress: "no-reply@flip.id", + subject: "Transfer ke", +}; diff --git a/apps/watcher/src/rules/index.js b/apps/watcher/src/rules/index.js index 8321612..2648a7b 100644 --- a/apps/watcher/src/rules/index.js +++ b/apps/watcher/src/rules/index.js @@ -1,5 +1,6 @@ import { briQris } from "./bri-qris.js"; import { briTransfer } from "./bri-transfer.js"; +import { flipTransfer } from "./flip-transfer.js"; import { grabEreceipt } from "./grab-ereceipt.js"; import { livinQrPayment } from "./livin-qr-payment.js"; import { tokopediaOrder } from "./tokopedia-order.js"; @@ -10,4 +11,5 @@ export const rules = [ briTransfer, briQris, tokopediaOrder, + flipTransfer, ]; diff --git a/apps/watcher/test/fixtures.js b/apps/watcher/test/fixtures.js index 25c6275..dea8f6f 100644 --- a/apps/watcher/test/fixtures.js +++ b/apps/watcher/test/fixtures.js @@ -411,3 +411,79 @@ Tujuan Pengiriman Roby `; + +export const flipDirectBody = `BUKTI TRANSFER + +Halo, Kak Nasyarobby. + +Transfer ke tujuan berhasil diproses. Ini detail transaksinya: +ID Transaksi +#ST260918145355861M3ZG4 +Waktu Terkirim +18 Sep 2026 14:53 WIB +Nama Tujuan +Rwen Ibrahim +Bank Tujuan +BCA +Nomor Rekening Tujuan +0020215810 +Nominal +Rp40.001 + +Kalau ada pertanyaan, silakan hubungi Tim Flip lewat menu *Bantuan* +(pilih *Chat +Tim Flip*) atau klik link flip.id/bantuan. +------------------------------ + +Makasih udah pakai Flip! + +Salam hangat, +Tim Flip`; + +export const flipForwardedBody = `---------- Forwarded message --------- +From: Flip +Date: Fri, Sep 18, 2026 at 2:54 PM +Subject: Transfer ke Rwen Ibrahim berhasil +To: + + +BUKTI TRANSFER + +Halo, Kak Nasyarobby. + +Transfer ke tujuan berhasil diproses. Ini detail transaksinya: +ID Transaksi +#ST260918145355861M3ZG4 +Waktu Terkirim +18 Sep 2026 14:53 WIB +Nama Tujuan +Rwen Ibrahim +Bank Tujuan +BCA +Nomor Rekening Tujuan +0020215810 +Nominal +Rp40.001 + +Kalau ada pertanyaan, silakan hubungi Tim Flip lewat menu *Bantuan* +(pilih *Chat +Tim Flip*) atau klik link flip.id/bantuan. +------------------------------ + +Makasih udah pakai Flip! + +Salam hangat, +Tim Flip + +Tidak membuat permintaan ini? Hubungi kami melalui aplikasi lewat ke menu +*Bantuan* (pilih *Chat Tim Flip*) atau klik link flip.id/bantuan. + + +Copyright © 2025 PT. Fliptech Lentera Inspirasi Pertiwi. All Rights +Reserved. + + +-- +Regards, +*Nasyarobby NP* +`; diff --git a/apps/watcher/test/match.test.js b/apps/watcher/test/match.test.js index 24adaed..0aff3c0 100644 --- a/apps/watcher/test/match.test.js +++ b/apps/watcher/test/match.test.js @@ -12,6 +12,8 @@ import { briQrisForwardedBody, briTransferDirectBody, briTransferForwardedBody, + flipDirectBody, + flipForwardedBody, grabBody, grabForwardedBody, livinDirectBody, @@ -223,6 +225,32 @@ describe("matchRule", () => { assert.equal(result.pass.rule.kind, "tokopedia.order.completed"); }); + it("matches direct Flip transfer", async () => { + const result = await matchRule({ + messageId: "flip-direct", + log: silentLog, + message: { + from: "Flip ", + subject: "Transfer ke Rwen Ibrahim berhasil", + body: flipDirectBody, + }, + }); + assert.equal(result.pass.rule.kind, "flip.transfer.success"); + }); + + it("matches forwarded Flip transfer via original header", async () => { + const result = await matchRule({ + messageId: "1a0c704164e0cd54", + log: silentLog, + message: { + from: "NSRB ", + subject: "Fwd: Transfer ke Rwen Ibrahim berhasil", + body: flipForwardedBody, + }, + }); + assert.equal(result.pass.rule.kind, "flip.transfer.success"); + }); + it("breaks on unknown from/subject", async () => { const result = await matchRule({ messageId: "x", diff --git a/apps/watcher/test/parse.test.js b/apps/watcher/test/parse.test.js index 3e2eb59..18003e8 100644 --- a/apps/watcher/test/parse.test.js +++ b/apps/watcher/test/parse.test.js @@ -1,12 +1,14 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { parseBriQris, parseBriTransfer } from "../src/parsers/bri.js"; +import { parseFlipTransfer } from "../src/parsers/flip.js"; import { parseGrabReceipt, extractGrabTripTimesFromHtml } from "../src/parsers/grab.js"; import { parseLivinReceipt, splitMerchantLocation } from "../src/parsers/livin.js"; import { parseTokopediaOrder } from "../src/parsers/tokopedia.js"; import { toSpendingRecord, toPocketBaseDate } from "../src/pocketbase/map.js"; import { briQris } from "../src/rules/bri-qris.js"; import { briTransfer } from "../src/rules/bri-transfer.js"; +import { flipTransfer } from "../src/rules/flip-transfer.js"; import { livinQrPayment } from "../src/rules/livin-qr-payment.js"; import { grabEreceipt } from "../src/rules/grab-ereceipt.js"; import { tokopediaOrder } from "../src/rules/tokopedia-order.js"; @@ -15,6 +17,8 @@ import { briQrisForwardedBody, briTransferDirectBody, briTransferForwardedBody, + flipDirectBody, + flipForwardedBody, grabBody, grabFoodForwardedBody, grabForwardedBody, @@ -366,6 +370,58 @@ describe("toSpendingRecord", () => { assert.equal(rec.description, "LuxePad.Creations - Tokopedia"); assert.equal(rec.trx_date, "2026-09-05 00:00:00.000Z"); }); + + it("maps Flip transfer envelope to spendings fields", () => { + const parsed = { + kind: "flip.transfer.success", + messageId: "1a0c704164e0cd54", + amount: 40001, + reference: "ST260918145355861M3ZG4", + occurredAt: "2026-09-18T14:53:00+07:00", + details: { + namaTujuan: "Rwen Ibrahim", + bankTujuan: "BCA", + nomorRekening: "0020215810", + }, + }; + const rec = toSpendingRecord(parsed, flipTransfer); + assert.equal(rec.invoice_id, "ST260918145355861M3ZG4"); + assert.equal(rec.amount, 40001); + assert.equal(rec.source, "flip"); + assert.equal(rec.description, "Rwen Ibrahim"); + assert.equal(rec.trx_date, toPocketBaseDate(parsed.occurredAt)); + assert.deepEqual(rec.parsed, parsed); + }); +}); + +describe("parseFlipTransfer", () => { + it("parses direct stacked-label body", () => { + const r = parseFlipTransfer(flipDirectBody); + assert.ok(r); + assert.equal(r.amount, 40001); + assert.equal(r.reference, "ST260918145355861M3ZG4"); + assert.equal(r.occurredAt, "2026-09-18T14:53:00+07:00"); + assert.equal(r.details.namaTujuan, "Rwen Ibrahim"); + assert.equal(r.details.bankTujuan, "BCA"); + assert.equal(r.details.nomorRekening, "0020215810"); + }); + + it("parses forwarded body", () => { + const r = parseFlipTransfer(flipForwardedBody); + assert.ok(r); + assert.equal(r.amount, 40001); + assert.equal(r.reference, "ST260918145355861M3ZG4"); + assert.equal(r.occurredAt, "2026-09-18T14:53:00+07:00"); + assert.equal(r.details.namaTujuan, "Rwen Ibrahim"); + }); + + it("returns null when Nominal or ID Transaksi missing", () => { + assert.equal(parseFlipTransfer("BUKTI TRANSFER\nNominal\nRp10.000\n"), null); + assert.equal( + parseFlipTransfer("ID Transaksi\n#ABC\nNama Tujuan\nX\n"), + null, + ); + }); }); describe("parseTokopediaOrder", () => { diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..f97d1f9 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,130 @@ +# Development — e-receipt pipeline + +Watcher + parsers that turn matched receipt mail into PocketBase `spendings` rows. + +## Designated mailbox (`WATCHER_MAILBOXES`) + +Receipt **watch + Pub/Sub processing** can be limited to specific connected accounts: + +```bash +WATCHER_MAILBOXES=eleven16th@gmail.com +# or comma-separated: a@x.com,b@y.com +``` + +- **Set:** only those mailboxes get `users.watch` and receipt match/parse/save. +- **Unset / empty:** all connected accounts (previous behavior). +- **CLI** (`npm run gmail`, `process-messages`) is **not** gated — other signed-in accounts remain usable for list/archive/draft. + +Restart the watcher after changing the env. + +## Watcher pipeline + +`POST /pubsub/gmail` → `history.list` (`messageAdded`) → per message: + +1. `logEvent` + `fetchAndLogMessage` — fetch outer mail; collect `emlMessages` if any +2. **Work items:** if `.eml` / `message/rfc822` attachments exist → process **each attachment only** (outer is a wrapper). If none → process the outer message as usual. +3. For each work item (sequential, in-process — no job queue): `matchRule` → `parseMatchedMessage` → `saveSpending` + +Handlers return `{ pass: {...} }` to merge into ctx or `{ break: true }` to stop. **Parse full `message.body`** (and `message.html` when needed), never the 500-char log `bodyPreview`. + +Attached `.eml` mails use nested From/Subject from Gmail’s expanded `message/rfc822` part. When nested HTML/text is not inlined (`body.data` missing, only `attachmentId`), the watcher fetches those attachments (`hydrateRfc822Message`) before parse. Synthetic `message_id` looks like `outerId#partId`; dedupe remains on `invoice_id`. + +## Match rules + +Resolve original sender: + +1. If body has `---------- Forwarded message ---------`, use its From / Subject / Date. +2. Else use Gmail envelope. + +| kind | From address | Subject | +|---|---|---| +| `livin.qr_payment.success` | `noreply.livin@bankmandiri.co.id` | `Pembayaran Berhasil!` | +| `grab.ereceipt.ride` | `no-reply@grab.com` | `Your Grab E-Receipt` | +| `grab.ereceipt.tip` | same From/Subject as ride; detected from tip body copy | tip invoice_id = `{Booking ID}:tip` | +| `grab.ereceipt.food` | same From/Subject; body has GrabFood / `Selamat menikmati makanan` | `Pesanan ID` as invoice_id | +| `bri.transfer.success` | `bankbri@bri.co.id` | `Pemindahan Dana Sesama Rekening BRI` | +| `bri.qris.success` | `bankbri@bri.co.id` | `Pembelian QRIS Berhasil` | +| `tokopedia.order.completed` | `noreply@tokopedia.com` | `Pesanan Selesai` | +| `flip.transfer.success` | `no-reply@flip.id` | `Transfer ke` (recipient name varies) | + +Production path is **direct provider mail**. Forwards still work via the Fwd header. Do **not** match on the forwarder envelope From. + +Rules: `apps/watcher/src/rules/`. Parsers: `apps/watcher/src/parsers/`. + +## Parsing notes + +- **Required for a spending row:** `amount` + `reference` (invoice id). Fail soft (log + break) if missing — never throw (Pub/Sub must ack). +- **Livin:** support clean forwarded lines and jammed HTML-stripped labels (`Tanggal13 Sep 2026`, `Tanggal15 Agu 2026`). Indonesian month names/abbreviations (`Agu`/`Agustus`, `Des`, …) are accepted. When merchant and city are glued on one line, keep the whole string as `merchant`; only split location when it is already a separate line ending in `- ID`. Do not maintain a city-name list. +- **Grab:** plain text often has date-only `Picked up on …`; pickup/dropoff times (`10:47AM`) live in HTML. Watcher keeps `message.html` and merges the first standalone AM/PM time into `occurredAt` (WIB). Tip receipts (`Your tip goes a long way…`, timestamp like `13 Sep 26 10:47 +0700`) become `grab.ereceipt.tip` with `invoice_id` `{bookingId}:tip` so they do not collide with the ride. +- **BRI / BRImo:** transfer (`Pemindahan Dana Sesama Rekening BRI`) — amount from `Nominal`, `invoice_id` from `Nomor Referensi`, `description` from `Nama Tujuan`, datetime from `Tanggal`. QRIS (`Pembelian QRIS Berhasil`) — amount from `Nominal` (fallback `Total Transaksi`), `description` from `Nama Merchant`, datetime from `Tanggal Transaksi`. Other BRI subjects stay out of scope until samples appear. +- **Tokopedia:** `Pesanan Selesai` — one row per `No.Invoice`; amount from `Total belanja` (not fee lines); `description` = `{Toko} - Tokopedia`; line items (1+) in `details.items`; `trx_date` from `Tanggal Terima` (date-only). +- **Flip:** transfer success (`Transfer ke {Name} berhasil`) — amount from `Nominal`, `invoice_id` from `ID Transaksi` (strip leading `#`), `description` from `Nama Tujuan`, datetime from `Waktu Terkirim`. Labels and values are stacked (value on the next line). Direct Flip mail and manual Fwd both match via `resolveOriginal`. +- Open-ended text (names, products, driver, compliments) is best-effort optional `details`. + +## PocketBase + +Env: `POCKETBASE_URL`, `POCKETBASE_USER`, `POCKETBASE_PASSWORD` (a `_superusers` account). Collection: `spendings`. + +- Unique `invoice_id` — duplicate insert → log skip, no throw. +- Leave `category` empty; user fills it in PocketBase admin. +- Store flat columns for querying; `details` for kind leftovers; `parsed` for the full watcher envelope JSON (audit snapshot). Keep `message_id` so the original Gmail message can be re-fetched if needed — do not store raw email body by default. +- Schema ensure runs on watcher boot (`ensureSpendingsSchema`). +- Never commit secrets. + +Parse failures and non-duplicate skips (incomplete record, save errors) POST to ntfy (`NTFY_URL`, default `https://n.0dev.web.id/system`) with `messageId` and message body. Duplicates do not notify. + +## Manual / backfill (forwarded old receipts) + +Gmail Date on a forward is “today”; `trx_date` still comes from the receipt body. Duplicates skip on unique `invoice_id`. Forward-as-attachment (multiple `*.eml`) is expanded the same way as the watcher. + +```bash +npm run list-messages -- +npm run list-messages -- --query 'newer_than:14d has:attachment filename:eml' --max 100 + +npm run process-messages -- --dry-run +npm run process-messages -- --ids OUTER_MESSAGE_ID +npm run process-messages -- --query 'newer_than:2d subject:"Pembayaran Berhasil!"' +npm run process-messages -- --ids MESSAGE_ID_1,MESSAGE_ID_2 +npm run process-messages -- --user eleven16th@gmail.com --max 100 +``` + +## Adding a provider + +1. Rule module under `src/rules/` (kind, fromAddress, subject, source). +2. Parser under `src/parsers/`; register in `parse-matched-message.js`. +3. Fixture + tests under `test/`. +4. Map into spendings in `pocketbase/map.js` (`source`, `description`, `details`, `parsed`). + +## Change Google API permissions (`gmail.modify`) + +Archive, labels, and drafts need `https://www.googleapis.com/auth/gmail.modify` (superset of read + watch). Keep the same OAuth Client ID / secret — do **not** create a new client. + +### Code + +`apps/api/src/config.js` uses `GMAIL_SCOPE = gmail.modify` (plus openid / email / profile). Auth already uses `prompt: consent` and `access_type: offline`. + +### Google Cloud Console (one-time) + +1. Open [Google Cloud Console](https://console.cloud.google.com/) → same project as the OAuth client. +2. **APIs & Services → Library**: confirm **Gmail API** is enabled. Pub/Sub unchanged; `users.watch` still works. +3. **OAuth consent screen** (or **Google Auth Platform → Data Access**): + - Add scope `https://www.googleapis.com/auth/gmail.modify`. + - Remove `gmail.readonly` if listed (modify already includes read). + - Save. +4. **Audience / Test users:** `gmail.modify` is **sensitive**. While **External + Testing**, every mailbox that signs in must be a **Test user**. Skip Google verification unless you publish to Production. +5. Do **not** change Authorized JavaScript origins / redirect URIs unless they are wrong. + +If consent fails or the new permission never appears, the scope is missing from the consent screen. An “unverified app” warning is expected in Testing. + +### Re-consent every connected mailbox + +Stored tokens were issued for `gmail.readonly` and cannot be upgraded in place: + +1. Restart the API (`npm run dev`). +2. Sign in **once per Gmail account** via the web UI (`/auth/google`). +3. On the consent screen, accept Google’s wording for modify (includes send/delete). The app still never sends or deletes. +4. Confirm: `npm run gmail -- accounts`. `403 insufficientPermissions` on archive/label/draft means that mailbox has not re-consented. + +### Unchanged + +Same client secrets, Pub/Sub topic / push URL, and INBOX watch. Web inbox stays read-only; only the CLI mutates.