diff --git a/db/exploration.go b/db/exploration.go index 51a22fa..5be7a45 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -1,6 +1,7 @@ package db import ( + "bytes" "database/sql" "encoding/json" "errors" @@ -20,8 +21,9 @@ var ErrIntentStateConflict = errors.New("intent state changed concurrently") // UTF8 encoding rejects ("invalid byte sequence for encoding UTF8"); without this // the INSERT fails and the activity record is silently lost. NUL is *valid* UTF-8 // (U+0000) so ToValidUTF8 leaves it in place, yet PostgreSQL text still rejects it -// (SQLSTATE 22021) — so it must be removed separately. JSONB payloads are fine -// (json.Marshal already sanitizes), so only the plain text columns need it. +// (SQLSTATE 22021) — so it must be removed separately. JSONB columns need the +// separate jsonbClean below: json.Marshal encodes a NUL as the escape backslash-u-0000, +// which the text-type json accepts but jsonb rejects (SQLSTATE 22P05). func utf8Clean(s string) string { if strings.IndexByte(s, 0) >= 0 { s = strings.ReplaceAll(s, "\x00", "") @@ -29,6 +31,37 @@ func utf8Clean(s string) string { return strings.ToValidUTF8(s, "�") } +// jsonbClean makes marshaled JSON safe for a PostgreSQL jsonb column. json.Marshal +// faithfully encodes a NUL byte (U+0000) as the 6-byte escape sequence \u0000; the +// json type stores it, but jsonb rejects it with "unsupported Unicode escape +// sequence" (SQLSTATE 22P05). Captured HTTP/tool bytes can carry NULs, so drop the +// escape — this also covers NULs nested inside json.RawMessage fields, which are +// copied verbatim into the output. Only a *real* escape is stripped: a \u0000 is +// genuine when preceded by an even number of backslashes, so an escaped-backslash +// run such as \\u0000 (the literal text u0000) is left intact. +func jsonbClean(b []byte) []byte { + if !bytes.Contains(b, []byte("\\u0000")) { + return b + } + out := make([]byte, 0, len(b)) + bs := 0 // consecutive backslashes already emitted before position i + for i := 0; i < len(b); i++ { + if b[i] == '\\' && bs%2 == 0 && i+5 < len(b) && + b[i+1] == 'u' && b[i+2] == '0' && b[i+3] == '0' && b[i+4] == '0' && b[i+5] == '0' { + i += 5 // skip the whole \u0000 + bs = 0 + continue + } + if b[i] == '\\' { + bs++ + } else { + bs = 0 + } + out = append(out, b[i]) + } + return out +} + // Node is a typed reasoning node (= old task_nodes). kind ∈ goal|intent|finding|hint. type Node struct { FindingID int64 `json:"finding_id,omitempty"` // populated by finding-aware reads; never inferred from ID diff --git a/db/jsonb_clean_test.go b/db/jsonb_clean_test.go new file mode 100644 index 0000000..51993e8 --- /dev/null +++ b/db/jsonb_clean_test.go @@ -0,0 +1,44 @@ +package db + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestJsonbClean(t *testing.T) { + // A marshaled payload carrying a NUL byte (e.g. captured HTTP/tool output). + b, err := json.Marshal(map[string]string{"body": "ab\x00cd"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `\u0000`) { + t.Fatalf("precondition: marshaled JSON should contain the NUL escape, got %s", b) + } + + cleaned := jsonbClean(b) + if strings.Contains(string(cleaned), `\u0000`) { + t.Fatalf("jsonbClean left a NUL escape jsonb rejects: %s", cleaned) + } + + // Result must stay valid JSON with the NUL simply dropped. + var out map[string]string + if err := json.Unmarshal(cleaned, &out); err != nil { + t.Fatalf("cleaned bytes are not valid JSON: %v (%s)", err, cleaned) + } + if out["body"] != "abcd" { + t.Fatalf("expected NUL stripped to \"abcd\", got %q", out["body"]) + } + + // A NUL escape typed literally in source text (doubled backslash) is preserved. + lit := []byte(`{"body":"\\u0000"}`) + if got := jsonbClean(lit); string(got) != string(lit) { + t.Fatalf("literal \\\\u0000 must be untouched, got %s", got) + } + + // No NUL escape → returned unchanged. + plain := []byte(`{"body":"hello"}`) + if got := jsonbClean(plain); string(got) != string(plain) { + t.Fatalf("plain JSON must be untouched, got %s", got) + } +} diff --git a/db/side_questions.go b/db/side_questions.go index b25816a..957b319 100644 --- a/db/side_questions.go +++ b/db/side_questions.go @@ -56,7 +56,7 @@ func (d *DB) SaveSideSnapshot(ctx context.Context, s sidequestion.Snapshot) erro _, err = tx.ExecContext(ctx, `INSERT INTO side_question_sessions(session_key,conversation_id,task_id,exploration_id,intent_id,run_id,version,snapshot) VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT(session_key) DO UPDATE SET run_id=EXCLUDED.run_id,version=EXCLUDED.version,snapshot=EXCLUDED.snapshot WHERE (side_question_sessions.run_id,side_question_sessions.version)<(EXCLUDED.run_id,EXCLUDED.version)`, - s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(b)) + s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(jsonbClean(b))) if err != nil { return err }