fix(server): summarize Buffer values in stored and API JSON
Stop dumping every byte as a number in event I/O, workflow test results, and script dry-runs. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,43 @@
|
|||||||
|
const BUFFER_PREVIEW_BYTES = 16;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {unknown} value
|
||||||
|
*/
|
||||||
|
export function isBinary(value) {
|
||||||
|
return (
|
||||||
|
Buffer.isBuffer(value) ||
|
||||||
|
ArrayBuffer.isView(value) ||
|
||||||
|
value instanceof ArrayBuffer
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compact stand-in for JSON (Buffer.toJSON dumps every byte as a number).
|
||||||
|
* @param {Buffer | ArrayBufferView | ArrayBuffer} value
|
||||||
|
*/
|
||||||
|
export function summarizeBinary(value) {
|
||||||
|
const buf = Buffer.isBuffer(value)
|
||||||
|
? value
|
||||||
|
: value instanceof ArrayBuffer
|
||||||
|
? Buffer.from(value)
|
||||||
|
: Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
||||||
|
const take = Math.min(buf.length, BUFFER_PREVIEW_BYTES);
|
||||||
|
return {
|
||||||
|
type: "Buffer",
|
||||||
|
length: buf.length,
|
||||||
|
preview: buf.subarray(0, take).toString("hex"),
|
||||||
|
truncated: buf.length > take,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON.stringify replacer. Must be a real function so `this` is the holder:
|
||||||
|
* Buffer#toJSON already ran on `value`, but `this[key]` is still the Buffer.
|
||||||
|
* @param {string} key
|
||||||
|
* @param {unknown} value
|
||||||
|
*/
|
||||||
|
export function jsonPreviewReplacer(key, value) {
|
||||||
|
const raw = this[key];
|
||||||
|
if (isBinary(raw)) return summarizeBinary(raw);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
import { isBinary, summarizeBinary } from "../../json-preview.js";
|
||||||
import { redactString } from "../../secret-value.js";
|
import { redactString } from "../../secret-value.js";
|
||||||
|
|
||||||
const LEVEL_TO_NUM = {
|
const LEVEL_TO_NUM = {
|
||||||
@@ -64,7 +65,9 @@ export function safeSerialize(value) {
|
|||||||
try {
|
try {
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
redactString(
|
redactString(
|
||||||
JSON.stringify(value, (_key, v) => {
|
JSON.stringify(value, function (key, v) {
|
||||||
|
const raw = this[key];
|
||||||
|
if (isBinary(raw)) return summarizeBinary(raw);
|
||||||
if (typeof v === "bigint") return v.toString();
|
if (typeof v === "bigint") return v.toString();
|
||||||
if (typeof v === "object" && v !== null) {
|
if (typeof v === "object" && v !== null) {
|
||||||
if (seen.has(v)) return "[Circular]";
|
if (seen.has(v)) return "[Circular]";
|
||||||
|
|||||||
@@ -418,7 +418,7 @@ export default function workflowsPluginFactory(registry) {
|
|||||||
return {
|
return {
|
||||||
runId: result.runId,
|
runId: result.runId,
|
||||||
status: result.status,
|
status: result.status,
|
||||||
result: result.result,
|
result: store.toDisplayValue(result.result),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { db } from "./db.js";
|
import { db } from "./db.js";
|
||||||
|
import { jsonPreviewReplacer } from "./json-preview.js";
|
||||||
import { redactString } from "./secret-value.js";
|
import { redactString } from "./secret-value.js";
|
||||||
|
|
||||||
const MAX_JSON_BYTES = 64 * 1024;
|
const MAX_JSON_BYTES = 64 * 1024;
|
||||||
@@ -12,7 +13,7 @@ export function serialize(value) {
|
|||||||
if (value === undefined || value === null) return null;
|
if (value === undefined || value === null) return null;
|
||||||
let json;
|
let json;
|
||||||
try {
|
try {
|
||||||
json = JSON.stringify(value);
|
json = JSON.stringify(value, jsonPreviewReplacer);
|
||||||
} catch {
|
} catch {
|
||||||
json = JSON.stringify({ truncated: true, reason: "unserializable" });
|
json = JSON.stringify({ truncated: true, reason: "unserializable" });
|
||||||
}
|
}
|
||||||
@@ -24,6 +25,20 @@ export function serialize(value) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parsed JSON-safe copy for API / UI (buffers summarized, size-capped).
|
||||||
|
* @param {unknown} value
|
||||||
|
*/
|
||||||
|
export function toDisplayValue(value) {
|
||||||
|
const json = serialize(value);
|
||||||
|
if (json == null) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(json);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {string | null} value
|
* @param {string | null} value
|
||||||
* @returns {unknown}
|
* @returns {unknown}
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import { jsonPreviewReplacer, summarizeBinary } from "../json-preview.js";
|
||||||
|
import { serialize, toDisplayValue } from "../store.js";
|
||||||
|
import { safeSerialize } from "../src/api/dry-run-logger.js";
|
||||||
|
|
||||||
|
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]);
|
||||||
|
const summary = summarizeBinary(png);
|
||||||
|
if (summary.length !== png.length || summary.preview !== "89504e470d0a1a0a010203" || summary.truncated !== false) {
|
||||||
|
throw new Error(`summarizeBinary: ${JSON.stringify(summary)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const long = Buffer.alloc(32, 0xff);
|
||||||
|
const longSummary = summarizeBinary(long);
|
||||||
|
if (longSummary.length !== 32 || longSummary.preview.length !== 32 || longSummary.truncated !== true) {
|
||||||
|
throw new Error(`long summarizeBinary: ${JSON.stringify(longSummary)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const dumped = JSON.stringify({ file: png });
|
||||||
|
if (!dumped.includes('"data":[')) {
|
||||||
|
throw new Error("expected default Buffer JSON to include data array");
|
||||||
|
}
|
||||||
|
|
||||||
|
const previewed = JSON.stringify({ file: png }, jsonPreviewReplacer);
|
||||||
|
if (previewed.includes('"data":[')) {
|
||||||
|
throw new Error(`replacer still dumped bytes: ${previewed}`);
|
||||||
|
}
|
||||||
|
if (!previewed.includes('"preview":"89504e470d0a1a0a010203"')) {
|
||||||
|
throw new Error(`replacer missing hex preview: ${previewed}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stored = serialize({ output: { file: png, filename: "test.png" } });
|
||||||
|
if (stored.includes('"data":[')) {
|
||||||
|
throw new Error(`serialize dumped bytes: ${stored.slice(0, 200)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const display = toDisplayValue({
|
||||||
|
output: { file: png },
|
||||||
|
context: { file: png },
|
||||||
|
});
|
||||||
|
if (display.output.file.length !== png.length || display.context.file.truncated !== false) {
|
||||||
|
throw new Error(`toDisplayValue: ${JSON.stringify(display)}`);
|
||||||
|
}
|
||||||
|
if (Array.isArray(display.output.file.data)) {
|
||||||
|
throw new Error("toDisplayValue should not keep Buffer.data");
|
||||||
|
}
|
||||||
|
|
||||||
|
const dry = safeSerialize({ file: png, n: 1n });
|
||||||
|
if (dry.n !== "1" || Array.isArray(dry.file.data)) {
|
||||||
|
throw new Error(`safeSerialize: ${JSON.stringify(dry)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const typed = safeSerialize({ file: new Uint8Array(png) });
|
||||||
|
if (typed.file.length !== png.length || typed.file.type !== "Buffer") {
|
||||||
|
throw new Error(`Uint8Array: ${JSON.stringify(typed)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("json-preview-smoke: ok");
|
||||||
Reference in New Issue
Block a user