feat(watcher): add Flip transfer receipt provider

Match no-reply@flip.id transfer success mail (direct or forwarded), parse stacked Nominal/ID Transaksi fields, and map spendings with source=flip.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-09-22 10:09:25 +07:00
co-authored by Cursor
parent f270f531ea
commit 427636b59b
9 changed files with 376 additions and 0 deletions
@@ -1,4 +1,5 @@
import { parseBriQris, parseBriTransfer } from "../parsers/bri.js"; import { parseBriQris, parseBriTransfer } from "../parsers/bri.js";
import { parseFlipTransfer } from "../parsers/flip.js";
import { parseGrabReceipt } from "../parsers/grab.js"; import { parseGrabReceipt } from "../parsers/grab.js";
import { parseLivinReceipt } from "../parsers/livin.js"; import { parseLivinReceipt } from "../parsers/livin.js";
import { parseTokopediaOrder } from "../parsers/tokopedia.js"; import { parseTokopediaOrder } from "../parsers/tokopedia.js";
@@ -10,6 +11,7 @@ const PARSERS = {
"bri.transfer.success": parseBriTransfer, "bri.transfer.success": parseBriTransfer,
"bri.qris.success": parseBriQris, "bri.qris.success": parseBriQris,
"tokopedia.order.completed": parseTokopediaOrder, "tokopedia.order.completed": parseTokopediaOrder,
"flip.transfer.success": parseFlipTransfer,
}; };
/** /**
+74
View File
@@ -0,0 +1,74 @@
import {
captureBlockAfterLabel,
parseDayMonthYear,
parseIdrAmount,
parseTimeWib,
toWibIso,
} from "../lib/parse-helpers.js";
/**
* First non-empty line after a Flip stacked label (value on the next line).
*/
function valueAfterLabel(body, label, nextLabels = []) {
const lines = captureBlockAfterLabel(body, label, nextLabels);
return lines?.[0]?.trim() || null;
}
/**
* Parse Flip transfer-success body (plain text, stacked label/value lines).
* Required: Nominal + ID Transaksi.
*/
export function parseFlipTransfer(body) {
if (!body) return null;
const referenceRaw = valueAfterLabel(body, "ID Transaksi", [
"Waktu Terkirim",
"Nama Tujuan",
]);
const reference = referenceRaw
? referenceRaw.replace(/^#/, "").trim()
: null;
const waktuRaw = valueAfterLabel(body, "Waktu Terkirim", [
"Nama Tujuan",
"Bank Tujuan",
]);
const dateYmd = parseDayMonthYear(waktuRaw);
const timeHms = parseTimeWib(waktuRaw);
const occurredAt = toWibIso(dateYmd, timeHms);
const amount = parseIdrAmount(
valueAfterLabel(body, "Nominal", [
"Kalau ada pertanyaan",
"Makasih",
"Salam",
]),
);
if (amount == null || !reference) return null;
const namaTujuan = valueAfterLabel(body, "Nama Tujuan", [
"Bank Tujuan",
"Nomor Rekening Tujuan",
]);
const bankTujuan = valueAfterLabel(body, "Bank Tujuan", [
"Nomor Rekening Tujuan",
"Nominal",
]);
const nomorRekening = valueAfterLabel(body, "Nomor Rekening Tujuan", [
"Nominal",
]);
const details = {};
if (namaTujuan) details.namaTujuan = namaTujuan;
if (bankTujuan) details.bankTujuan = bankTujuan;
if (nomorRekening) details.nomorRekening = nomorRekening;
return {
amount,
currency: "IDR",
reference,
occurredAt,
details,
};
}
+2
View File
@@ -24,6 +24,8 @@ export function toSpendingRecord(parsed, rule) {
? `${base} tip` ? `${base} tip`
: base; : base;
} }
} else if (source === "flip") {
description = parsed.details?.namaTujuan || "";
} }
return { return {
+6
View File
@@ -0,0 +1,6 @@
export const flipTransfer = {
kind: "flip.transfer.success",
source: "flip",
fromAddress: "no-reply@flip.id",
subject: "Transfer ke",
};
+2
View File
@@ -1,5 +1,6 @@
import { briQris } from "./bri-qris.js"; import { briQris } from "./bri-qris.js";
import { briTransfer } from "./bri-transfer.js"; import { briTransfer } from "./bri-transfer.js";
import { flipTransfer } from "./flip-transfer.js";
import { grabEreceipt } from "./grab-ereceipt.js"; import { grabEreceipt } from "./grab-ereceipt.js";
import { livinQrPayment } from "./livin-qr-payment.js"; import { livinQrPayment } from "./livin-qr-payment.js";
import { tokopediaOrder } from "./tokopedia-order.js"; import { tokopediaOrder } from "./tokopedia-order.js";
@@ -10,4 +11,5 @@ export const rules = [
briTransfer, briTransfer,
briQris, briQris,
tokopediaOrder, tokopediaOrder,
flipTransfer,
]; ];
+76
View File
@@ -411,3 +411,79 @@ Tujuan Pengiriman
Roby Roby
`; `;
export const flipDirectBody = `BUKTI TRANSFER
Halo, Kak Nasyarobby.
Transfer ke tujuan berhasil diproses. Ini detail transaksinya:
ID Transaksi
#ST260918145355861M3ZG4
Waktu Terkirim
18 Sep 2026 14:53 WIB
Nama Tujuan
Rwen Ibrahim
Bank Tujuan
BCA
Nomor Rekening Tujuan
0020215810
Nominal
Rp40.001
Kalau ada pertanyaan, silakan hubungi Tim Flip lewat menu *Bantuan*
(pilih *Chat
Tim Flip*) atau klik link flip.id/bantuan.
------------------------------
Makasih udah pakai Flip!
Salam hangat,
Tim Flip`;
export const flipForwardedBody = `---------- Forwarded message ---------
From: Flip <no-reply@flip.id>
Date: Fri, Sep 18, 2026 at 2:54 PM
Subject: Transfer ke Rwen Ibrahim berhasil
To: <nasyarobby@gmail.com>
BUKTI TRANSFER
Halo, Kak Nasyarobby.
Transfer ke tujuan berhasil diproses. Ini detail transaksinya:
ID Transaksi
#ST260918145355861M3ZG4
Waktu Terkirim
18 Sep 2026 14:53 WIB
Nama Tujuan
Rwen Ibrahim
Bank Tujuan
BCA
Nomor Rekening Tujuan
0020215810
Nominal
Rp40.001
Kalau ada pertanyaan, silakan hubungi Tim Flip lewat menu *Bantuan*
(pilih *Chat
Tim Flip*) atau klik link flip.id/bantuan.
------------------------------
Makasih udah pakai Flip!
Salam hangat,
Tim Flip
Tidak membuat permintaan ini? Hubungi kami melalui aplikasi lewat ke menu
*Bantuan* (pilih *Chat Tim Flip*) atau klik link flip.id/bantuan.
Copyright © 2025 PT. Fliptech Lentera Inspirasi Pertiwi. All Rights
Reserved.
--
Regards,
*Nasyarobby NP*
`;
+28
View File
@@ -12,6 +12,8 @@ import {
briQrisForwardedBody, briQrisForwardedBody,
briTransferDirectBody, briTransferDirectBody,
briTransferForwardedBody, briTransferForwardedBody,
flipDirectBody,
flipForwardedBody,
grabBody, grabBody,
grabForwardedBody, grabForwardedBody,
livinDirectBody, livinDirectBody,
@@ -223,6 +225,32 @@ describe("matchRule", () => {
assert.equal(result.pass.rule.kind, "tokopedia.order.completed"); assert.equal(result.pass.rule.kind, "tokopedia.order.completed");
}); });
it("matches direct Flip transfer", async () => {
const result = await matchRule({
messageId: "flip-direct",
log: silentLog,
message: {
from: "Flip <no-reply@flip.id>",
subject: "Transfer ke Rwen Ibrahim berhasil",
body: flipDirectBody,
},
});
assert.equal(result.pass.rule.kind, "flip.transfer.success");
});
it("matches forwarded Flip transfer via original header", async () => {
const result = await matchRule({
messageId: "1a0c704164e0cd54",
log: silentLog,
message: {
from: "NSRB <nasyarobby@gmail.com>",
subject: "Fwd: Transfer ke Rwen Ibrahim berhasil",
body: flipForwardedBody,
},
});
assert.equal(result.pass.rule.kind, "flip.transfer.success");
});
it("breaks on unknown from/subject", async () => { it("breaks on unknown from/subject", async () => {
const result = await matchRule({ const result = await matchRule({
messageId: "x", messageId: "x",
+56
View File
@@ -1,12 +1,14 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { describe, it } from "node:test"; import { describe, it } from "node:test";
import { parseBriQris, parseBriTransfer } from "../src/parsers/bri.js"; import { parseBriQris, parseBriTransfer } from "../src/parsers/bri.js";
import { parseFlipTransfer } from "../src/parsers/flip.js";
import { parseGrabReceipt, extractGrabTripTimesFromHtml } from "../src/parsers/grab.js"; import { parseGrabReceipt, extractGrabTripTimesFromHtml } from "../src/parsers/grab.js";
import { parseLivinReceipt, splitMerchantLocation } from "../src/parsers/livin.js"; import { parseLivinReceipt, splitMerchantLocation } from "../src/parsers/livin.js";
import { parseTokopediaOrder } from "../src/parsers/tokopedia.js"; import { parseTokopediaOrder } from "../src/parsers/tokopedia.js";
import { toSpendingRecord, toPocketBaseDate } from "../src/pocketbase/map.js"; import { toSpendingRecord, toPocketBaseDate } from "../src/pocketbase/map.js";
import { briQris } from "../src/rules/bri-qris.js"; import { briQris } from "../src/rules/bri-qris.js";
import { briTransfer } from "../src/rules/bri-transfer.js"; import { briTransfer } from "../src/rules/bri-transfer.js";
import { flipTransfer } from "../src/rules/flip-transfer.js";
import { livinQrPayment } from "../src/rules/livin-qr-payment.js"; import { livinQrPayment } from "../src/rules/livin-qr-payment.js";
import { grabEreceipt } from "../src/rules/grab-ereceipt.js"; import { grabEreceipt } from "../src/rules/grab-ereceipt.js";
import { tokopediaOrder } from "../src/rules/tokopedia-order.js"; import { tokopediaOrder } from "../src/rules/tokopedia-order.js";
@@ -15,6 +17,8 @@ import {
briQrisForwardedBody, briQrisForwardedBody,
briTransferDirectBody, briTransferDirectBody,
briTransferForwardedBody, briTransferForwardedBody,
flipDirectBody,
flipForwardedBody,
grabBody, grabBody,
grabFoodForwardedBody, grabFoodForwardedBody,
grabForwardedBody, grabForwardedBody,
@@ -366,6 +370,58 @@ describe("toSpendingRecord", () => {
assert.equal(rec.description, "LuxePad.Creations - Tokopedia"); assert.equal(rec.description, "LuxePad.Creations - Tokopedia");
assert.equal(rec.trx_date, "2026-09-05 00:00:00.000Z"); assert.equal(rec.trx_date, "2026-09-05 00:00:00.000Z");
}); });
it("maps Flip transfer envelope to spendings fields", () => {
const parsed = {
kind: "flip.transfer.success",
messageId: "1a0c704164e0cd54",
amount: 40001,
reference: "ST260918145355861M3ZG4",
occurredAt: "2026-09-18T14:53:00+07:00",
details: {
namaTujuan: "Rwen Ibrahim",
bankTujuan: "BCA",
nomorRekening: "0020215810",
},
};
const rec = toSpendingRecord(parsed, flipTransfer);
assert.equal(rec.invoice_id, "ST260918145355861M3ZG4");
assert.equal(rec.amount, 40001);
assert.equal(rec.source, "flip");
assert.equal(rec.description, "Rwen Ibrahim");
assert.equal(rec.trx_date, toPocketBaseDate(parsed.occurredAt));
assert.deepEqual(rec.parsed, parsed);
});
});
describe("parseFlipTransfer", () => {
it("parses direct stacked-label body", () => {
const r = parseFlipTransfer(flipDirectBody);
assert.ok(r);
assert.equal(r.amount, 40001);
assert.equal(r.reference, "ST260918145355861M3ZG4");
assert.equal(r.occurredAt, "2026-09-18T14:53:00+07:00");
assert.equal(r.details.namaTujuan, "Rwen Ibrahim");
assert.equal(r.details.bankTujuan, "BCA");
assert.equal(r.details.nomorRekening, "0020215810");
});
it("parses forwarded body", () => {
const r = parseFlipTransfer(flipForwardedBody);
assert.ok(r);
assert.equal(r.amount, 40001);
assert.equal(r.reference, "ST260918145355861M3ZG4");
assert.equal(r.occurredAt, "2026-09-18T14:53:00+07:00");
assert.equal(r.details.namaTujuan, "Rwen Ibrahim");
});
it("returns null when Nominal or ID Transaksi missing", () => {
assert.equal(parseFlipTransfer("BUKTI TRANSFER\nNominal\nRp10.000\n"), null);
assert.equal(
parseFlipTransfer("ID Transaksi\n#ABC\nNama Tujuan\nX\n"),
null,
);
});
}); });
describe("parseTokopediaOrder", () => { describe("parseTokopediaOrder", () => {
+130
View File
@@ -0,0 +1,130 @@
# Development — e-receipt pipeline
Watcher + parsers that turn matched receipt mail into PocketBase `spendings` rows.
## Designated mailbox (`WATCHER_MAILBOXES`)
Receipt **watch + Pub/Sub processing** can be limited to specific connected accounts:
```bash
WATCHER_MAILBOXES=eleven16th@gmail.com
# or comma-separated: a@x.com,b@y.com
```
- **Set:** only those mailboxes get `users.watch` and receipt match/parse/save.
- **Unset / empty:** all connected accounts (previous behavior).
- **CLI** (`npm run gmail`, `process-messages`) is **not** gated — other signed-in accounts remain usable for list/archive/draft.
Restart the watcher after changing the env.
## Watcher pipeline
`POST /pubsub/gmail` → `history.list` (`messageAdded`) → per message:
1. `logEvent` + `fetchAndLogMessage` — fetch outer mail; collect `emlMessages` if any
2. **Work items:** if `.eml` / `message/rfc822` attachments exist → process **each attachment only** (outer is a wrapper). If none → process the outer message as usual.
3. For each work item (sequential, in-process — no job queue): `matchRule` → `parseMatchedMessage` → `saveSpending`
Handlers return `{ pass: {...} }` to merge into ctx or `{ break: true }` to stop. **Parse full `message.body`** (and `message.html` when needed), never the 500-char log `bodyPreview`.
Attached `.eml` mails use nested From/Subject from Gmail’s expanded `message/rfc822` part. When nested HTML/text is not inlined (`body.data` missing, only `attachmentId`), the watcher fetches those attachments (`hydrateRfc822Message`) before parse. Synthetic `message_id` looks like `outerId#partId`; dedupe remains on `invoice_id`.
## Match rules
Resolve original sender:
1. If body has `---------- Forwarded message ---------`, use its From / Subject / Date.
2. Else use Gmail envelope.
| kind | From address | Subject |
|---|---|---|
| `livin.qr_payment.success` | `noreply.livin@bankmandiri.co.id` | `Pembayaran Berhasil!` |
| `grab.ereceipt.ride` | `no-reply@grab.com` | `Your Grab E-Receipt` |
| `grab.ereceipt.tip` | same From/Subject as ride; detected from tip body copy | tip invoice_id = `{Booking ID}:tip` |
| `grab.ereceipt.food` | same From/Subject; body has GrabFood / `Selamat menikmati makanan` | `Pesanan ID` as invoice_id |
| `bri.transfer.success` | `bankbri@bri.co.id` | `Pemindahan Dana Sesama Rekening BRI` |
| `bri.qris.success` | `bankbri@bri.co.id` | `Pembelian QRIS Berhasil` |
| `tokopedia.order.completed` | `noreply@tokopedia.com` | `Pesanan Selesai` |
| `flip.transfer.success` | `no-reply@flip.id` | `Transfer ke` (recipient name varies) |
Production path is **direct provider mail**. Forwards still work via the Fwd header. Do **not** match on the forwarder envelope From.
Rules: `apps/watcher/src/rules/`. Parsers: `apps/watcher/src/parsers/`.
## Parsing notes
- **Required for a spending row:** `amount` + `reference` (invoice id). Fail soft (log + break) if missing — never throw (Pub/Sub must ack).
- **Livin:** support clean forwarded lines and jammed HTML-stripped labels (`Tanggal13 Sep 2026`, `Tanggal15 Agu 2026`). Indonesian month names/abbreviations (`Agu`/`Agustus`, `Des`, …) are accepted. When merchant and city are glued on one line, keep the whole string as `merchant`; only split location when it is already a separate line ending in `- ID`. Do not maintain a city-name list.
- **Grab:** plain text often has date-only `Picked up on …`; pickup/dropoff times (`10:47AM`) live in HTML. Watcher keeps `message.html` and merges the first standalone AM/PM time into `occurredAt` (WIB). Tip receipts (`Your tip goes a long way…`, timestamp like `13 Sep 26 10:47 +0700`) become `grab.ereceipt.tip` with `invoice_id` `{bookingId}:tip` so they do not collide with the ride.
- **BRI / BRImo:** transfer (`Pemindahan Dana Sesama Rekening BRI`) — amount from `Nominal`, `invoice_id` from `Nomor Referensi`, `description` from `Nama Tujuan`, datetime from `Tanggal`. QRIS (`Pembelian QRIS Berhasil`) — amount from `Nominal` (fallback `Total Transaksi`), `description` from `Nama Merchant`, datetime from `Tanggal Transaksi`. Other BRI subjects stay out of scope until samples appear.
- **Tokopedia:** `Pesanan Selesai` — one row per `No.Invoice`; amount from `Total belanja` (not fee lines); `description` = `{Toko} - Tokopedia`; line items (1+) in `details.items`; `trx_date` from `Tanggal Terima` (date-only).
- **Flip:** transfer success (`Transfer ke {Name} berhasil`) — amount from `Nominal`, `invoice_id` from `ID Transaksi` (strip leading `#`), `description` from `Nama Tujuan`, datetime from `Waktu Terkirim`. Labels and values are stacked (value on the next line). Direct Flip mail and manual Fwd both match via `resolveOriginal`.
- Open-ended text (names, products, driver, compliments) is best-effort optional `details`.
## PocketBase
Env: `POCKETBASE_URL`, `POCKETBASE_USER`, `POCKETBASE_PASSWORD` (a `_superusers` account). Collection: `spendings`.
- Unique `invoice_id` — duplicate insert → log skip, no throw.
- Leave `category` empty; user fills it in PocketBase admin.
- Store flat columns for querying; `details` for kind leftovers; `parsed` for the full watcher envelope JSON (audit snapshot). Keep `message_id` so the original Gmail message can be re-fetched if needed — do not store raw email body by default.
- Schema ensure runs on watcher boot (`ensureSpendingsSchema`).
- Never commit secrets.
Parse failures and non-duplicate skips (incomplete record, save errors) POST to ntfy (`NTFY_URL`, default `https://n.0dev.web.id/system`) with `messageId` and message body. Duplicates do not notify.
## Manual / backfill (forwarded old receipts)
Gmail Date on a forward is “today”; `trx_date` still comes from the receipt body. Duplicates skip on unique `invoice_id`. Forward-as-attachment (multiple `*.eml`) is expanded the same way as the watcher.
```bash
npm run list-messages --
npm run list-messages -- --query 'newer_than:14d has:attachment filename:eml' --max 100
npm run process-messages -- --dry-run
npm run process-messages -- --ids OUTER_MESSAGE_ID
npm run process-messages -- --query 'newer_than:2d subject:"Pembayaran Berhasil!"'
npm run process-messages -- --ids MESSAGE_ID_1,MESSAGE_ID_2
npm run process-messages -- --user eleven16th@gmail.com --max 100
```
## Adding a provider
1. Rule module under `src/rules/` (kind, fromAddress, subject, source).
2. Parser under `src/parsers/`; register in `parse-matched-message.js`.
3. Fixture + tests under `test/`.
4. Map into spendings in `pocketbase/map.js` (`source`, `description`, `details`, `parsed`).
## Change Google API permissions (`gmail.modify`)
Archive, labels, and drafts need `https://www.googleapis.com/auth/gmail.modify` (superset of read + watch). Keep the same OAuth Client ID / secret — do **not** create a new client.
### Code
`apps/api/src/config.js` uses `GMAIL_SCOPE = gmail.modify` (plus openid / email / profile). Auth already uses `prompt: consent` and `access_type: offline`.
### Google Cloud Console (one-time)
1. Open [Google Cloud Console](https://console.cloud.google.com/) → same project as the OAuth client.
2. **APIs & Services → Library**: confirm **Gmail API** is enabled. Pub/Sub unchanged; `users.watch` still works.
3. **OAuth consent screen** (or **Google Auth Platform → Data Access**):
- Add scope `https://www.googleapis.com/auth/gmail.modify`.
- Remove `gmail.readonly` if listed (modify already includes read).
- Save.
4. **Audience / Test users:** `gmail.modify` is **sensitive**. While **External + Testing**, every mailbox that signs in must be a **Test user**. Skip Google verification unless you publish to Production.
5. Do **not** change Authorized JavaScript origins / redirect URIs unless they are wrong.
If consent fails or the new permission never appears, the scope is missing from the consent screen. An “unverified app” warning is expected in Testing.
### Re-consent every connected mailbox
Stored tokens were issued for `gmail.readonly` and cannot be upgraded in place:
1. Restart the API (`npm run dev`).
2. Sign in **once per Gmail account** via the web UI (`/auth/google`).
3. On the consent screen, accept Google’s wording for modify (includes send/delete). The app still never sends or deletes.
4. Confirm: `npm run gmail -- accounts`. `403 insufficientPermissions` on archive/label/draft means that mailbox has not re-consented.
### Unchanged
Same client secrets, Pub/Sub topic / push URL, and INBOX watch. Web inbox stays read-only; only the CLI mutates.