mirror of
https://github.com/RuoJi6/ARTEX.git
synced 2026-10-27 17:03:49 +08:00
fix(db): side_question checkpoint 入库前剥除 jsonb 不支持的 NUL 转义
HTTP/工具响应含裸 NUL 字节时,json.Marshal 将其编码为 \u0000 转义, PostgreSQL jsonb 列拒绝并抛 SQLSTATE 22P05,导致 SaveSideSnapshot 的 INSERT 每秒失败数十次刷屏。新增 jsonbClean 在 marshal 后剥除真正的 \u0000 转义(按前置反斜杠奇偶判断,避免误伤字面 \\u0000),并订正 exploration.go 中 "json.Marshal already sanitizes" 的错误注释。 Closes #111, Closes #112
This commit is contained in:
1 parent
42f0ff97d3
commit
16e87b48c2
3 files changed
+80
-3
No files matched your search
+35
-2
@@ -1,6 +1,7 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -20,8 +21,9 @@ var ErrIntentStateConflict = errors.New("intent state changed concurrently")
|
||||
// UTF8 encoding rejects ("invalid byte sequence for encoding UTF8"); without this
|
||||
// the INSERT fails and the activity record is silently lost. NUL is *valid* UTF-8
|
||||
// (U+0000) so ToValidUTF8 leaves it in place, yet PostgreSQL text still rejects it
|
||||
// (SQLSTATE 22021) — so it must be removed separately. JSONB payloads are fine
|
||||
// (json.Marshal already sanitizes), so only the plain text columns need it.
|
||||
// (SQLSTATE 22021) — so it must be removed separately. JSONB columns need the
|
||||
// separate jsonbClean below: json.Marshal encodes a NUL as the escape backslash-u-0000,
|
||||
// which the text-type json accepts but jsonb rejects (SQLSTATE 22P05).
|
||||
func utf8Clean(s string) string {
|
||||
if strings.IndexByte(s, 0) >= 0 {
|
||||
s = strings.ReplaceAll(s, "\x00", "")
|
||||
@@ -29,6 +31,37 @@ func utf8Clean(s string) string {
|
||||
return strings.ToValidUTF8(s, "�")
|
||||
}
|
||||
|
||||
// jsonbClean makes marshaled JSON safe for a PostgreSQL jsonb column. json.Marshal
|
||||
// faithfully encodes a NUL byte (U+0000) as the 6-byte escape sequence \u0000; the
|
||||
// json type stores it, but jsonb rejects it with "unsupported Unicode escape
|
||||
// sequence" (SQLSTATE 22P05). Captured HTTP/tool bytes can carry NULs, so drop the
|
||||
// escape — this also covers NULs nested inside json.RawMessage fields, which are
|
||||
// copied verbatim into the output. Only a *real* escape is stripped: a \u0000 is
|
||||
// genuine when preceded by an even number of backslashes, so an escaped-backslash
|
||||
// run such as \\u0000 (the literal text u0000) is left intact.
|
||||
func jsonbClean(b []byte) []byte {
|
||||
if !bytes.Contains(b, []byte("\\u0000")) {
|
||||
return b
|
||||
}
|
||||
out := make([]byte, 0, len(b))
|
||||
bs := 0 // consecutive backslashes already emitted before position i
|
||||
for i := 0; i < len(b); i++ {
|
||||
if b[i] == '\\' && bs%2 == 0 && i+5 < len(b) &&
|
||||
b[i+1] == 'u' && b[i+2] == '0' && b[i+3] == '0' && b[i+4] == '0' && b[i+5] == '0' {
|
||||
i += 5 // skip the whole \u0000
|
||||
bs = 0
|
||||
continue
|
||||
}
|
||||
if b[i] == '\\' {
|
||||
bs++
|
||||
} else {
|
||||
bs = 0
|
||||
}
|
||||
out = append(out, b[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Node is a typed reasoning node (= old task_nodes). kind ∈ goal|intent|finding|hint.
|
||||
type Node struct {
|
||||
FindingID int64 `json:"finding_id,omitempty"` // populated by finding-aware reads; never inferred from ID
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJsonbClean(t *testing.T) {
|
||||
// A marshaled payload carrying a NUL byte (e.g. captured HTTP/tool output).
|
||||
b, err := json.Marshal(map[string]string{"body": "ab\x00cd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(b), `\u0000`) {
|
||||
t.Fatalf("precondition: marshaled JSON should contain the NUL escape, got %s", b)
|
||||
}
|
||||
|
||||
cleaned := jsonbClean(b)
|
||||
if strings.Contains(string(cleaned), `\u0000`) {
|
||||
t.Fatalf("jsonbClean left a NUL escape jsonb rejects: %s", cleaned)
|
||||
}
|
||||
|
||||
// Result must stay valid JSON with the NUL simply dropped.
|
||||
var out map[string]string
|
||||
if err := json.Unmarshal(cleaned, &out); err != nil {
|
||||
t.Fatalf("cleaned bytes are not valid JSON: %v (%s)", err, cleaned)
|
||||
}
|
||||
if out["body"] != "abcd" {
|
||||
t.Fatalf("expected NUL stripped to \"abcd\", got %q", out["body"])
|
||||
}
|
||||
|
||||
// A NUL escape typed literally in source text (doubled backslash) is preserved.
|
||||
lit := []byte(`{"body":"\\u0000"}`)
|
||||
if got := jsonbClean(lit); string(got) != string(lit) {
|
||||
t.Fatalf("literal \\\\u0000 must be untouched, got %s", got)
|
||||
}
|
||||
|
||||
// No NUL escape → returned unchanged.
|
||||
plain := []byte(`{"body":"hello"}`)
|
||||
if got := jsonbClean(plain); string(got) != string(plain) {
|
||||
t.Fatalf("plain JSON must be untouched, got %s", got)
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func (d *DB) SaveSideSnapshot(ctx context.Context, s sidequestion.Snapshot) erro
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO side_question_sessions(session_key,conversation_id,task_id,exploration_id,intent_id,run_id,version,snapshot)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT(session_key) DO UPDATE SET run_id=EXCLUDED.run_id,version=EXCLUDED.version,snapshot=EXCLUDED.snapshot
|
||||
WHERE (side_question_sessions.run_id,side_question_sessions.version)<(EXCLUDED.run_id,EXCLUDED.version)`,
|
||||
s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(b))
|
||||
s.Parent.Key(), nullableSideID(s.Parent.ConversationID), nullableSideID(s.Parent.TaskID), nullableSideID(s.Parent.ExplorationID), nullableSideID(s.Parent.IntentID), s.RunID, s.Version, string(jsonbClean(b)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user