fix(db): side_question checkpoint 入库前剥除 jsonb 不支持的 NUL 转义

HTTP/工具响应含裸 NUL 字节时,json.Marshal 将其编码为 \u0000 转义,
PostgreSQL jsonb 列拒绝并抛 SQLSTATE 22P05,导致 SaveSideSnapshot 的
INSERT 每秒失败数十次刷屏。新增 jsonbClean 在 marshal 后剥除真正的
\u0000 转义(按前置反斜杠奇偶判断,避免误伤字面 \\u0000),并订正
exploration.go 中 "json.Marshal already sanitizes" 的错误注释。

Closes #111, Closes #112
This commit is contained in:
Autumn-27 committed 2026-09-12 03:00:28 -04:00
1 parent 42f0ff97d3
commit 16e87b48c2
3 files changed
+80 -3

No files matched your search

+35 -2
View File
@@ -1,6 +1,7 @@
package db
import (
"bytes"
"database/sql"
"encoding/json"
"errors"
@@ -20,8 +21,9 @@ var ErrIntentStateConflict = errors.New("intent state changed concurrently")
// UTF8 encoding rejects ("invalid byte sequence for encoding UTF8"); without this
// the INSERT fails and the activity record is silently lost. NUL is *valid* UTF-8
// (U+0000) so ToValidUTF8 leaves it in place, yet PostgreSQL text still rejects it
// (SQLSTATE 22021) — so it must be removed separately. JSONB payloads are fine
// (json.Marshal already sanitizes), so only the plain text columns need it.
// (SQLSTATE 22021) — so it must be removed separately. JSONB columns need the
// separate jsonbClean below: json.Marshal encodes a NUL as the escape backslash-u-0000,
// which the text-type json accepts but jsonb rejects (SQLSTATE 22P05).
func utf8Clean(s string) string {
if strings.IndexByte(s, 0) >= 0 {
s = strings.ReplaceAll(s, "\x00", "")
@@ -29,6 +31,37 @@ func utf8Clean(s string) string {
return strings.ToValidUTF8(s, "�")
}
// jsonbClean makes marshaled JSON safe for a PostgreSQL jsonb column. json.Marshal
// faithfully encodes a NUL byte (U+0000) as the 6-byte escape sequence \u0000; the
// json type stores it, but jsonb rejects it with "unsupported Unicode escape
// sequence" (SQLSTATE 22P05). Captured HTTP/tool bytes can carry NULs, so drop the
// escape — this also covers NULs nested inside json.RawMessage fields, which are
// copied verbatim into the output. Only a *real* escape is stripped: a \u0000 is
// genuine when preceded by an even number of backslashes, so an escaped-backslash
// run such as \\u0000 (the literal text u0000) is left intact.
func jsonbClean(b []byte) []byte {
if !bytes.Contains(b, []byte("\\u0000")) {
return b
}
out := make([]byte, 0, len(b))
bs := 0 // consecutive backslashes already emitted before position i
for i := 0; i < len(b); i++ {
if b[i] == '\\' && bs%2 == 0 && i+5 < len(b) &&
b[i+1] == 'u' && b[i+2] == '0' && b[i+3] == '0' && b[i+4] == '0' && b[i+5] == '0' {
i += 5 // skip the whole \u0000
bs = 0
continue
}
if b[i] == '\\' {
bs++
} else {
bs = 0
}
out = append(out, b[i])
}
return out
}
// Node is a typed reasoning node (= old task_nodes). kind ∈ goal|intent|finding|hint.
type Node struct {
FindingID int64 `json:"finding_id,omitempty"` // populated by finding-aware reads; never inferred from ID
+44
View File
@@ -0,0 +1,44 @@
package db
import (
"encoding/json"
"strings"
"testing"
)
func TestJsonbClean(t *testing.T) {
// A marshaled payload carrying a NUL byte (e.g. captured HTTP/tool output).
b, err := json.Marshal(map[string]string{"body": "ab\x00cd"})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), `\u0000`) {
t.Fatalf("precondition: marshaled JSON should contain the NUL escape, got %s", b)
}
cleaned := jsonbClean(b)
if strings.Contains(string(cleaned), `\u0000`) {
t.Fatalf("jsonbClean left a NUL escape jsonb rejects: %s", cleaned)
}
// Result must stay valid JSON with the NUL simply dropped.
var out map[string]string
if err := json.Unmarshal(cleaned, &out); err != nil {
t.Fatalf("cleaned bytes are not valid JSON: %v (%s)", err, cleaned)
}
if out["body"] != "abcd" {
t.Fatalf("expected NUL stripped to \"abcd\", got %q", out["body"])
}
// A NUL escape typed literally in source text (doubled backslash) is preserved.
lit := []byte(`{"body":"\\u0000"}`)
if got := jsonbClean(lit); string(got) != string(lit) {
t.Fatalf("literal \\\\u0000 must be untouched, got %s", got)
}
// No NUL escape → returned unchanged.
plain := []byte(`{"body":"hello"}`)
if got := jsonbClean(plain); string(got) != string(plain) {
t.Fatalf("plain JSON must be untouched, got %s", got)
}
}
+1 -1
View File
@@ -56,7 +56,7 @@ func (d *DB) SaveSideSnapshot(ctx context.Context, s sidequestion.Snapshot) erro
_, err = tx.ExecContext(ctx, `INSERT INTO side_question_sessions(session_key,conversation_id,task_id,exploration_id,intent_id,run_id,version,snapshot)
VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT(session_key) DO UPDATE SET run_id=EXCLUDED.run_id,version=EXCLUDED.version,snapshot=EXCLUDED.snapshot
WHERE (side_question_sessions.run_id,side_question_sessions.version)<(EXCLUDED.run_id,EXCLUDED.version)`,
s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(b))
s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(jsonbClean(b)))
if err != nil {
return err
}