fix(skills): 修复技能压缩包上传失败与中文命名被判非法
- 补上 bzip2 / Zstandard 解压器(标准库 + 已有的 klauspost/compress,不引新依赖), 不再对压缩软件非默认档位打出的包抛 "zip: unsupported compression"; Deflate64/LZMA/XZ/PPMd 等真解不了的方式与加密包改为解压前给出中文提示, 点名文件、压缩方式与重新打包的办法 - 技能内文件路径校验从 [A-Za-z0-9-_./] 的 ASCII 白名单改为 Unicode 黑名单: 放行各语言文件名与空格,仍拒绝控制字符、非法 UTF-8、零宽与双向控制字符、 \ % # ? * : " < > | 及 .. / 绝对路径 / 空路径段,防目录穿越行为不变 - 技能名放开非 ASCII 字母数字(中文可直接作技能名),ASCII 仍守 agentskills.io 规范,空格、点与路径分隔符照旧拒绝 - Windows 压缩软件按 GBK 写的文件名兜底解码后再校验;frontmatter 的 name: "中文技能" 去引号 - 新增 8 个测试:zstd 包、GBK 包、不支持的压缩方式、加密包、目录穿越、名称与 路径校验
This commit is contained in:
1 parent
99fb78df1c
commit
338e77c064
5 files changed
+483
-51
No files matched your search
@@ -16,6 +16,14 @@
|
||||
|
||||
- 去掉企业资产范围「最多 256 条规则」的限制:逐个 IP / 域名录入范围的企业很容易撞上这个上限,撞上之后只能拆成多个企业,而范围本身并不因为条数多就更慢。前后端与 demo mock 三处上限一并移除,单条规则仍限 1024 字符,请求正文 2 MiB 的上限保留作兜底(约合四五万条规则)。
|
||||
|
||||
### 技能
|
||||
|
||||
#### 修复的问题
|
||||
|
||||
- 修复上传技能压缩包报「上传失败:zip: unsupported compression」的问题:Go 标准库只内置 Store / Deflate 两种解压器,压缩软件在非默认档位下写出的 bzip2、Zstandard 包一律解不开。现在补上这两种解压器(纯 Go 实现,不引新的外部依赖);Deflate64 / LZMA / XZ / PPMd 等确实解不了的方式,以及加密压缩包,改为在解压前就报出中文提示,直接点名是哪个文件用了哪种压缩方式、该怎么重新打包,不再把底层英文错误甩给用户。
|
||||
- 修复技能内文件名校验把中文文件名判为非法的问题:路径校验原先是 `[A-Za-z0-9-_./]` 的 ASCII 白名单,压缩包里只要有一个中文命名的文件(`参考/说明.md` 之类),整包上传就会以「压缩包含非法路径」失败。改为 Unicode 黑名单:允许各种语言的文件名与空格,仍然拒绝控制字符、非法 UTF-8、零宽与双向控制字符(RLO 文件名伪装)、`\ % # ? * : " < > |` 以及 `..` / 绝对路径 / 空路径段,防目录穿越的行为不变。技能名同样放开——ASCII 仍限小写字母、数字与连字符(agentskills.io 规范),中文等非 ASCII 字母可直接作技能名,但不接受空格、点和路径分隔符。
|
||||
- 修复 Windows 压缩软件打出的中文包解压后文件名乱码或整包被拒的问题:这类 zip 不置 UTF-8 标志位、按 GBK 写文件名,现在按 GBK 兜底解码后再做路径校验。同时 `name: "中文技能"` 这种带引号的 frontmatter 也能正确取到技能名。
|
||||
|
||||
### UI
|
||||
|
||||
#### 新增的功能
|
||||
|
||||
@@ -12,6 +12,7 @@ require (
|
||||
github.com/projectdiscovery/dnsx v1.2.3
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/text v0.40.0
|
||||
modernc.org/sqlite v1.52.0
|
||||
)
|
||||
|
||||
@@ -53,7 +54,6 @@ require (
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.3 // indirect
|
||||
|
||||
+102
-50
@@ -20,28 +20,42 @@ import (
|
||||
"text/template"
|
||||
"text/template/parse"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/norma/skill"
|
||||
)
|
||||
|
||||
// reSkillName is the base character-set pattern; validSkillName enforces
|
||||
// the full agentskills.io spec rules on top of it.
|
||||
var reSkillName = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
|
||||
|
||||
// validSkillName checks all agentskills.io name constraints:
|
||||
// lowercase alphanumeric + hyphens, 1-64 chars, no leading/trailing/consecutive hyphens.
|
||||
// validSkillName checks the agentskills.io name constraints, widened so a skill can
|
||||
// also be named in Chinese (or any other script): ASCII must stay lowercase
|
||||
// alphanumeric + hyphens, while non-ASCII letters/digits are accepted as-is.
|
||||
// 1-64 runes, must start with a letter, no leading/trailing/consecutive hyphens.
|
||||
// The name doubles as a directory name under skillDir, so nothing that could carry a
|
||||
// path (separators, dots, spaces, control characters) is allowed through.
|
||||
func validSkillName(name string) bool {
|
||||
if l := len(name); l == 0 || l > 64 {
|
||||
if name == "" || !utf8.ValidString(name) {
|
||||
return false
|
||||
}
|
||||
if !reSkillName.MatchString(name) {
|
||||
rs := []rune(name)
|
||||
if len(rs) > 64 {
|
||||
return false
|
||||
}
|
||||
if name[len(name)-1] == '-' {
|
||||
isLetter := func(r rune) bool {
|
||||
return (r >= 'a' && r <= 'z') || (r > unicode.MaxASCII && unicode.IsLetter(r))
|
||||
}
|
||||
if !isLetter(rs[0]) || rs[len(rs)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for _, r := range rs {
|
||||
switch {
|
||||
case isLetter(r), r >= '0' && r <= '9', r == '-':
|
||||
case r > unicode.MaxASCII && unicode.IsDigit(r):
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return !strings.Contains(name, "--")
|
||||
}
|
||||
|
||||
@@ -1257,7 +1271,8 @@ func skillNameFromFrontmatter(md []byte) string {
|
||||
break // end of frontmatter
|
||||
}
|
||||
if inFM && strings.HasPrefix(t, "name:") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(t, "name:"))
|
||||
v := strings.TrimSpace(strings.TrimPrefix(t, "name:"))
|
||||
return strings.Trim(v, `"'`) // name: "中文技能" 也认
|
||||
}
|
||||
}
|
||||
return ""
|
||||
@@ -1282,36 +1297,43 @@ func (s *Server) fsUploadSkill(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
|
||||
zr, err := newSkillZipReader(buf)
|
||||
if err != nil {
|
||||
writeErr(w, 400, "无法解析压缩包(需为 zip 格式):"+err.Error())
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
// entries carry UTF-8-decoded names (GBK 包也能读) and exclude archiver junk.
|
||||
entriesAll := skillZipEntries(zr)
|
||||
if err := checkSkillZipMethods(entriesAll); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// locate the shallowest SKILL.md → its directory is the skill root inside the zip.
|
||||
skillMD := ""
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() || path.Base(f.Name) != "SKILL.md" {
|
||||
var skillMD *skillZipEntry
|
||||
for i := range entriesAll {
|
||||
e := &entriesAll[i]
|
||||
if path.Base(e.name) != "SKILL.md" {
|
||||
continue
|
||||
}
|
||||
if skillMD == "" || strings.Count(f.Name, "/") < strings.Count(skillMD, "/") {
|
||||
skillMD = f.Name
|
||||
if skillMD == nil || strings.Count(e.name, "/") < strings.Count(skillMD.name, "/") {
|
||||
skillMD = e
|
||||
}
|
||||
}
|
||||
if skillMD == "" {
|
||||
if skillMD == nil {
|
||||
writeErr(w, 400, "压缩包内未找到 SKILL.md")
|
||||
return
|
||||
}
|
||||
root := path.Dir(skillMD) // "." when SKILL.md is at the zip root
|
||||
root := path.Dir(skillMD.name) // "." when SKILL.md is at the zip root
|
||||
prefix := ""
|
||||
if root != "." {
|
||||
prefix = root + "/"
|
||||
}
|
||||
|
||||
// derive + validate the skill name from the SKILL.md frontmatter.
|
||||
md, err := readZipEntry(zr, skillMD)
|
||||
md, err := readZipEntry(skillMD.f)
|
||||
if err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
writeErr(w, 400, "读取 SKILL.md 失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
name := skillNameFromFrontmatter(md)
|
||||
@@ -1319,10 +1341,12 @@ func (s *Server) fsUploadSkill(w http.ResponseWriter, r *http.Request) {
|
||||
name = path.Base(strings.TrimSuffix(prefix, "/"))
|
||||
}
|
||||
if name == "" {
|
||||
name = strings.TrimSuffix(hdr.Filename, ".zip")
|
||||
base := path.Base(filepath.ToSlash(hdr.Filename))
|
||||
name = strings.TrimSuffix(base, path.Ext(base))
|
||||
}
|
||||
if !validSkillName(name) {
|
||||
writeErr(w, 400, "skill 名称无效(取自 SKILL.md 的 name 字段):"+name)
|
||||
writeErr(w, 400, "skill 名称无效(取自 SKILL.md 的 name 字段):"+name+
|
||||
"(≤64 字符,字母开头,只能用小写字母/数字/连字符或中文等非 ASCII 字母,不能有空格、点、路径分隔符)")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1345,21 +1369,19 @@ func (s *Server) fsUploadSkill(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var total int64
|
||||
entries := 0
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
for _, e := range entriesAll {
|
||||
f := e.f
|
||||
// only files under the skill root; skip anything outside it.
|
||||
if prefix != "" && !strings.HasPrefix(e.name, prefix) {
|
||||
continue
|
||||
}
|
||||
// only files under the skill root; skip archive junk.
|
||||
if prefix != "" && !strings.HasPrefix(f.Name, prefix) {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(f.Name, prefix)
|
||||
if rel == "" || strings.HasPrefix(rel, "__MACOSX/") || path.Base(rel) == ".DS_Store" {
|
||||
rel := strings.TrimPrefix(e.name, prefix)
|
||||
if rel == "" {
|
||||
continue
|
||||
}
|
||||
clean, msg := skillRelPath(rel)
|
||||
if msg != "" {
|
||||
writeErr(w, 400, "压缩包含非法路径 "+f.Name+":"+msg)
|
||||
writeErr(w, 400, "压缩包含非法路径 "+e.name+":"+msg)
|
||||
return
|
||||
}
|
||||
if entries++; entries > maxSkillEntries {
|
||||
@@ -1414,14 +1436,15 @@ func (s *Server) fsUploadSkill(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 201, map[string]any{"name": name, "files": entries})
|
||||
}
|
||||
|
||||
// readZipEntry reads a single named entry's bytes (capped) from an open zip.
|
||||
func readZipEntry(zr *zip.Reader, name string) ([]byte, error) {
|
||||
f, err := zr.Open(name)
|
||||
// readZipEntry reads a single entry's bytes (capped). Takes the *zip.File rather than
|
||||
// a name because zr.Open rejects names that aren't valid UTF-8 (GBK-named archives).
|
||||
func readZipEntry(f *zip.File) ([]byte, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return io.ReadAll(io.LimitReader(f, maxSkillFileBytes))
|
||||
defer rc.Close()
|
||||
return io.ReadAll(io.LimitReader(rc, maxSkillFileBytes))
|
||||
}
|
||||
|
||||
func (s *Server) fsDeleteSkill(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1445,23 +1468,52 @@ func (s *Server) fsDeleteSkill(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, map[string]any{"deleted": name})
|
||||
}
|
||||
|
||||
// reFilePath is a strict ASCII whitelist for relative paths inside a skill directory.
|
||||
// Allowed: A-Z a-z 0-9 hyphen underscore dot forward-slash.
|
||||
// This blocks before any normalization: null bytes, backslash, %, unicode look-alikes,
|
||||
// control characters, and any other byte not in the set.
|
||||
// Go's net/http URL-decodes PathValue once (e.g. %2F→/ %2e→.), so an attacker who
|
||||
// sends %252e%252e gets the literal string "%2e%2e" here — the '%' fails the whitelist.
|
||||
var reFilePath = regexp.MustCompile(`^[A-Za-z0-9\-_./]+$`)
|
||||
// skillPathBlocked lists the ASCII characters a skill-relative path may not contain.
|
||||
// '\' is a Windows separator; '%' keeps double-URL-encoding tricks from surviving
|
||||
// (Go's net/http URL-decodes PathValue once — %2F→/ %2e→. — so an attacker sending
|
||||
// %252e%252e arrives here as the literal "%2e%2e" and is rejected on the '%'); '#'
|
||||
// and '?' would truncate the path when it travels back through a URL; the rest are
|
||||
// reserved on Windows filesystems.
|
||||
const skillPathBlocked = `\%#?*:"<>|`
|
||||
|
||||
// skillPathRune reports whether r may appear in a client-supplied skill path.
|
||||
// It is a blacklist over Unicode rather than an ASCII whitelist so that 中文 (and any
|
||||
// other script) file names work, while everything that makes path validation hard is
|
||||
// still refused: control/format characters, look-alike whitespace, separators.
|
||||
func skillPathRune(r rune) bool {
|
||||
switch {
|
||||
case r < 0x20, r == 0x7f, r == utf8.RuneError:
|
||||
return false // NUL & control characters, invalid UTF-8
|
||||
case strings.ContainsRune(skillPathBlocked, r):
|
||||
return false
|
||||
case unicode.Is(unicode.Cf, r), unicode.Is(unicode.Co, r), unicode.Is(unicode.Cs, r):
|
||||
return false // zero-width joiners, bidi overrides (RLO 文件名伪装), private use
|
||||
case r != ' ' && unicode.IsSpace(r):
|
||||
return false // NBSP / 全角空格 之类:看着是空格,其实不是
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// maxSkillPathLen caps a relative path so a pathological name can't reach the syscall.
|
||||
const maxSkillPathLen = 512
|
||||
|
||||
// skillRelPath validates a relative file path supplied by the client.
|
||||
// Returns the cleaned path and an empty error string on success.
|
||||
// Validation order matters: whitelist first (before Clean) so encoding tricks can't
|
||||
// survive normalization.
|
||||
// Validation order matters: character check first (before Clean) so encoding tricks
|
||||
// can't survive normalization.
|
||||
func skillRelPath(file string) (string, string) {
|
||||
// 1. Whitelist — reject anything outside [A-Za-z0-9-_./] before normalization.
|
||||
// This alone defeats: null bytes, backslash, %, unicode, control chars.
|
||||
if !reFilePath.MatchString(file) {
|
||||
return "", "invalid path: only [A-Za-z0-9-_./] allowed"
|
||||
// 1. Character check — before normalization. Defeats null bytes, backslash,
|
||||
// '%', control characters and unicode look-alikes; CJK names pass through.
|
||||
if file == "" || len(file) > maxSkillPathLen {
|
||||
return "", "invalid path: empty or too long"
|
||||
}
|
||||
if !utf8.ValidString(file) {
|
||||
return "", "invalid path: not valid UTF-8"
|
||||
}
|
||||
for _, r := range file {
|
||||
if !skillPathRune(r) {
|
||||
return "", "invalid path: illegal character " + strconv.QuoteRune(r)
|
||||
}
|
||||
}
|
||||
// 2. Reject ".." explicitly. With the whitelist above, encoding bypass is already
|
||||
// impossible, but we keep this check so the intent is obvious to reviewers.
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
func TestValidSkillName(t *testing.T) {
|
||||
ok := []string{"web-recon", "a", "nuclei2", "中文技能", "端口扫描-x", "日本語スキル"}
|
||||
bad := []string{
|
||||
"", "Web-Recon", "-lead", "trail-", "dou--ble", "has space", "中文 技能",
|
||||
"dot.name", "a/b", `a\b`, "..", ".", "中文/技能", "sk\x00ill", "中文",
|
||||
strings.Repeat("a", 65), "1abc", "中文技能!",
|
||||
}
|
||||
for _, n := range ok {
|
||||
if !validSkillName(n) {
|
||||
t.Errorf("validSkillName(%q) = false, want true", n)
|
||||
}
|
||||
}
|
||||
for _, n := range bad {
|
||||
if validSkillName(n) {
|
||||
t.Errorf("validSkillName(%q) = true, want false", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkillRelPath(t *testing.T) {
|
||||
ok := map[string]string{
|
||||
"SKILL.md": "SKILL.md",
|
||||
"scripts/run.py": "scripts/run.py",
|
||||
"参考/中文说明.md": "参考/中文说明.md",
|
||||
"references/a b.txt": "references/a b.txt",
|
||||
"./SKILL.md": "SKILL.md",
|
||||
"assets/图片-1_v2.png": "assets/图片-1_v2.png",
|
||||
}
|
||||
for in, want := range ok {
|
||||
got, msg := skillRelPath(in)
|
||||
if msg != "" || got != want {
|
||||
t.Errorf("skillRelPath(%q) = (%q, %q), want (%q, \"\")", in, got, msg, want)
|
||||
}
|
||||
}
|
||||
bad := []string{
|
||||
"", "../etc/passwd", "a/../../b", "/abs/path", "a//b", `..\..\x`,
|
||||
"%2e%2e/x", "a\x00b", "中文 名.md", "中文.md", "a#b.md", "a?b.md",
|
||||
"a:b.md", string([]byte{0xd6, 0xd0}) + ".md", // 裸 GBK 字节:非法 UTF-8
|
||||
strings.Repeat("a", maxSkillPathLen+1),
|
||||
}
|
||||
for _, in := range bad {
|
||||
if got, msg := skillRelPath(in); msg == "" {
|
||||
t.Errorf("skillRelPath(%q) = (%q, \"\"), want rejection", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type nopWriteCloser struct{ io.Writer }
|
||||
|
||||
func (nopWriteCloser) Close() error { return nil }
|
||||
|
||||
// zipFile describes one entry for buildZip.
|
||||
type zipFile struct {
|
||||
name string
|
||||
body string
|
||||
method uint16
|
||||
nonUTF8 bool // write the name bytes as-is (GBK 包)
|
||||
}
|
||||
|
||||
func buildZip(t *testing.T, files ...zipFile) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
zw.RegisterCompressor(zipMethodZstd, zstd.ZipCompressor())
|
||||
// Deflate64 没有纯 Go 编码器;这里原样写入,只是为了给条目打上 method 9 的标记 ——
|
||||
// 断言的是「方法不支持时给什么提示」,不会真去解压它。
|
||||
zw.RegisterCompressor(zipMethodDeflate64, func(w io.Writer) (io.WriteCloser, error) {
|
||||
return nopWriteCloser{w}, nil
|
||||
})
|
||||
for _, f := range files {
|
||||
h := &zip.FileHeader{Name: f.name, Method: f.method, NonUTF8: f.nonUTF8}
|
||||
w, err := zw.CreateHeader(h)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateHeader(%q): %v", f.name, err)
|
||||
}
|
||||
if _, err := io.WriteString(w, f.body); err != nil {
|
||||
t.Fatalf("write %q: %v", f.name, err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zip close: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// uploadZip posts raw zip bytes to fsUploadSkill and returns the response.
|
||||
func uploadZip(t *testing.T, skillDir string, filename string, data []byte) (*httptest.ResponseRecorder, map[string]any) {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
fw, err := mw.CreateFormFile("file", filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := fw.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mw.Close()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/skills/upload", &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
rr := httptest.NewRecorder()
|
||||
(&Server{skillDir: skillDir}).fsUploadSkill(rr, req)
|
||||
|
||||
var out map[string]any
|
||||
_ = json.Unmarshal(rr.Body.Bytes(), &out)
|
||||
return rr, out
|
||||
}
|
||||
|
||||
const zhSkillMD = "---\nname: 中文技能\ndescription: 测试\n---\n正文\n"
|
||||
|
||||
// A zstd-compressed archive (WinZip 的可选压缩方式) used to blow up with
|
||||
// "zip: unsupported compression"; it now installs like any Deflate archive.
|
||||
func TestUploadSkillZstdAndChineseNames(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
data := buildZip(t,
|
||||
zipFile{name: "中文技能/SKILL.md", body: zhSkillMD, method: zipMethodZstd},
|
||||
zipFile{name: "中文技能/参考/说明 文档.md", body: "参考", method: zipMethodZstd},
|
||||
zipFile{name: "中文技能/scripts/run.py", body: "print(1)", method: zip.Deflate},
|
||||
)
|
||||
rr, out := uploadZip(t, dir, "中文技能.zip", data)
|
||||
if rr.Code != 201 {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body)
|
||||
}
|
||||
if out["name"] != "中文技能" {
|
||||
t.Fatalf("name = %v, want 中文技能", out["name"])
|
||||
}
|
||||
for _, rel := range []string{"SKILL.md", "参考/说明 文档.md", "scripts/run.py"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "中文技能", rel)); err != nil {
|
||||
t.Errorf("missing extracted file %q: %v", rel, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GBK-named entries (7-Zip / 资源管理器 on Chinese Windows) must be decoded rather
|
||||
// than rejected as invalid UTF-8 paths.
|
||||
func TestUploadSkillGBKNames(t *testing.T) {
|
||||
gbk := func(s string) string {
|
||||
b, err := simplifiedchinese.GBK.NewEncoder().String(s)
|
||||
if err != nil {
|
||||
t.Fatalf("gbk encode %q: %v", s, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
dir := t.TempDir()
|
||||
data := buildZip(t,
|
||||
zipFile{name: gbk("中文技能/SKILL.md"), body: zhSkillMD, method: zip.Deflate, nonUTF8: true},
|
||||
zipFile{name: gbk("中文技能/参考资料.md"), body: "内容", method: zip.Deflate, nonUTF8: true},
|
||||
)
|
||||
rr, out := uploadZip(t, dir, "skill.zip", data)
|
||||
if rr.Code != 201 {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body)
|
||||
}
|
||||
if out["name"] != "中文技能" {
|
||||
t.Fatalf("name = %v, want 中文技能", out["name"])
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "中文技能", "参考资料.md")); err != nil {
|
||||
t.Errorf("GBK-named entry not extracted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An archive we genuinely cannot decode should name the method in Chinese instead of
|
||||
// surfacing "zip: unsupported compression algorithm".
|
||||
func TestUploadSkillUnsupportedMethod(t *testing.T) {
|
||||
data := buildZip(t,
|
||||
zipFile{name: "demo/SKILL.md", body: "---\nname: demo\n---\n", method: zipMethodDeflate64},
|
||||
)
|
||||
rr, out := uploadZip(t, t.TempDir(), "demo.zip", data)
|
||||
if rr.Code != 400 {
|
||||
t.Fatalf("status = %d, want 400 (body %s)", rr.Code, rr.Body)
|
||||
}
|
||||
msg, _ := out["error"].(string)
|
||||
if !strings.Contains(msg, "Deflate64") || !strings.Contains(msg, "不支持的压缩方式") {
|
||||
t.Fatalf("error = %q, want a Chinese message naming Deflate64", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSkillEncrypted(t *testing.T) {
|
||||
data := buildZip(t, zipFile{name: "demo/SKILL.md", body: "---\nname: demo\n---\n", method: zip.Deflate})
|
||||
// flip the "encrypted" general-purpose flag bit in the local file header
|
||||
// (offset 6) and in the central directory copy (offset 8).
|
||||
local := bytes.Index(data, []byte("PK\x03\x04"))
|
||||
central := bytes.Index(data, []byte("PK\x01\x02"))
|
||||
if local < 0 || central < 0 {
|
||||
t.Fatal("could not locate zip headers")
|
||||
}
|
||||
data[local+6] |= 1
|
||||
data[central+8] |= 1
|
||||
|
||||
rr, out := uploadZip(t, t.TempDir(), "demo.zip", data)
|
||||
if rr.Code != 400 {
|
||||
t.Fatalf("status = %d, want 400 (body %s)", rr.Code, rr.Body)
|
||||
}
|
||||
if msg, _ := out["error"].(string); !strings.Contains(msg, "已加密") {
|
||||
t.Fatalf("error = %q, want 加密 hint", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Zip-slip must still be refused now that the path check accepts Unicode.
|
||||
func TestUploadSkillRejectsTraversal(t *testing.T) {
|
||||
data := buildZip(t,
|
||||
zipFile{name: "demo/SKILL.md", body: "---\nname: demo\n---\n", method: zip.Deflate},
|
||||
zipFile{name: "demo/../../evil.sh", body: "rm -rf /", method: zip.Deflate},
|
||||
)
|
||||
dir := t.TempDir()
|
||||
rr, out := uploadZip(t, dir, "demo.zip", data)
|
||||
if rr.Code != 400 {
|
||||
t.Fatalf("status = %d, want 400 (body %s)", rr.Code, rr.Body)
|
||||
}
|
||||
if msg, _ := out["error"].(string); !strings.Contains(msg, "非法路径") {
|
||||
t.Fatalf("error = %q, want 非法路径", msg)
|
||||
}
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 0 {
|
||||
t.Fatalf("upload left files behind: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkillNameFromFrontmatterQuoted(t *testing.T) {
|
||||
got := skillNameFromFrontmatter([]byte("---\nname: \"中文技能\"\ndescription: x\n---\n"))
|
||||
if got != "中文技能" {
|
||||
t.Fatalf("name = %q, want 中文技能", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// Go 的 archive/zip 只内置 Store(0) 和 Deflate(8) 两种解压器,遇到别的方法会返回
|
||||
// "zip: unsupported compression algorithm"。压缩软件在非默认档位下经常写出别的方法
|
||||
// (7-Zip 的 bzip2、WinZip 的 zstd),所以这里把纯 Go 能解的两种补上;真的解不了的
|
||||
// (Deflate64 / LZMA / XZ / PPMd / 加密包) 在解压前就报出中文提示,而不是把底层
|
||||
// 错误原样甩给用户。
|
||||
const (
|
||||
zipMethodStore = 0
|
||||
zipMethodDeflate = 8
|
||||
zipMethodDeflate64 = 9
|
||||
zipMethodBzip2 = 12
|
||||
zipMethodLZMA = 14
|
||||
zipMethodZstdPKW = 20 // PKWARE 早期给 zstd 分配的编号
|
||||
zipMethodZstd = 93
|
||||
zipMethodXZ = 95
|
||||
zipMethodJPEG = 96
|
||||
zipMethodWavPack = 97
|
||||
zipMethodPPMd = 98
|
||||
zipMethodAES = 99
|
||||
)
|
||||
|
||||
var zipMethodNames = map[uint16]string{
|
||||
zipMethodStore: "Store",
|
||||
zipMethodDeflate: "Deflate",
|
||||
zipMethodDeflate64: "Deflate64",
|
||||
zipMethodBzip2: "bzip2",
|
||||
zipMethodLZMA: "LZMA",
|
||||
zipMethodZstdPKW: "Zstandard",
|
||||
zipMethodZstd: "Zstandard",
|
||||
zipMethodXZ: "XZ",
|
||||
zipMethodJPEG: "JPEG",
|
||||
zipMethodWavPack: "WavPack",
|
||||
zipMethodPPMd: "PPMd",
|
||||
zipMethodAES: "AES 加密",
|
||||
}
|
||||
|
||||
func zipMethodName(m uint16) string {
|
||||
if n, ok := zipMethodNames[m]; ok {
|
||||
return n
|
||||
}
|
||||
return "未知"
|
||||
}
|
||||
|
||||
// newSkillZipReader parses an uploaded archive and registers the extra decompressors
|
||||
// we can support beyond the stdlib's Store/Deflate.
|
||||
func newSkillZipReader(buf []byte) (*zip.Reader, error) {
|
||||
zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("无法解析压缩包(需为 zip 格式):%w", err)
|
||||
}
|
||||
zr.RegisterDecompressor(zipMethodBzip2, func(r io.Reader) io.ReadCloser {
|
||||
return io.NopCloser(bzip2.NewReader(r))
|
||||
})
|
||||
zdec := zstd.ZipDecompressor(zstd.WithDecoderConcurrency(1))
|
||||
zr.RegisterDecompressor(zipMethodZstd, zdec)
|
||||
zr.RegisterDecompressor(zipMethodZstdPKW, zdec)
|
||||
return zr, nil
|
||||
}
|
||||
|
||||
// skillZipEntry pairs a zip entry with its decoded (UTF-8) name — f.Name may hold
|
||||
// raw GBK bytes, see zipEntryName.
|
||||
type skillZipEntry struct {
|
||||
f *zip.File
|
||||
name string
|
||||
}
|
||||
|
||||
// skillZipEntries lists the archive's real files (no directory entries, no archiver
|
||||
// junk) with their names decoded to UTF-8.
|
||||
func skillZipEntries(zr *zip.Reader) []skillZipEntry {
|
||||
out := make([]skillZipEntry, 0, len(zr.File))
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
name := zipEntryName(f)
|
||||
if strings.HasPrefix(name, "__MACOSX/") || strings.Contains(name, "/__MACOSX/") ||
|
||||
path.Base(name) == ".DS_Store" {
|
||||
continue // macOS 打包残留
|
||||
}
|
||||
out = append(out, skillZipEntry{f: f, name: name})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// zipEntryName returns the entry path as UTF-8. Windows 上的 7-Zip / WinRAR / 资源管理器
|
||||
// 在不置 UTF-8 标志位时会把中文文件名按 GBK 写进 zip,Go 原样保留这些字节,于是名字
|
||||
// 既不是合法 UTF-8 也过不了路径校验 —— 这里按 GBK 兜底解码。
|
||||
func zipEntryName(f *zip.File) string {
|
||||
if utf8.ValidString(f.Name) {
|
||||
return f.Name
|
||||
}
|
||||
if dec, err := simplifiedchinese.GBK.NewDecoder().String(f.Name); err == nil && utf8.ValidString(dec) {
|
||||
return dec
|
||||
}
|
||||
return f.Name
|
||||
}
|
||||
|
||||
// checkSkillZipMethods rejects archives we cannot extract, naming the offending
|
||||
// entry and method instead of letting f.Open() fail with an opaque English error.
|
||||
func checkSkillZipMethods(entries []skillZipEntry) error {
|
||||
for _, e := range entries {
|
||||
if e.f.Flags&0x1 != 0 || e.f.Method == zipMethodAES {
|
||||
return fmt.Errorf("压缩包已加密(%s),请上传未加密的 zip", e.name)
|
||||
}
|
||||
switch e.f.Method {
|
||||
case zipMethodStore, zipMethodDeflate, zipMethodBzip2, zipMethodZstd, zipMethodZstdPKW:
|
||||
default:
|
||||
return fmt.Errorf("压缩包使用了不支持的压缩方式 %s(method %d):%s。"+
|
||||
"请改用「存储」或「Deflate」重新打包(7-Zip/WinRAR 的压缩方式选 Deflate,"+
|
||||
"或直接用系统自带的“压缩/发送到压缩文件夹”、命令行 zip -r)",
|
||||
zipMethodName(e.f.Method), e.f.Method, e.name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user