mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-18 06:00:54 +08:00
Merge pull request #252 from renaesop/codex/upload-500mib
fix: support 512 MiB task and chat attachments
This commit is contained in:
@@ -148,6 +148,7 @@ jobs:
|
||||
- run: npx tsc --noEmit
|
||||
- run: npm run test:polling
|
||||
- run: npm run test:replay
|
||||
- run: npm run test:upload
|
||||
- run: npm run build:static
|
||||
|
||||
python:
|
||||
|
||||
@@ -476,6 +476,10 @@ server {
|
||||
# ssl_certificate / ssl_certificate_key ...
|
||||
|
||||
location / {
|
||||
# 512 MiB combined file limit plus 1 MiB of multipart metadata.
|
||||
client_max_body_size 513m;
|
||||
proxy_request_buffering off;
|
||||
|
||||
proxy_pass http://127.0.0.1:8787;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
+24
-4
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -12,8 +13,13 @@ import (
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// maxChatUpload caps a single chat-attachment upload request (memory + spill).
|
||||
const maxChatUpload = 128 << 20 // 128 MiB
|
||||
const (
|
||||
// maxChatUpload caps the combined file data in one attachment upload.
|
||||
maxChatUpload = 512 << 20 // 512 MiB
|
||||
// Allow multipart headers and boundaries in addition to the file data.
|
||||
chatUploadMultipartOverhead = 1 << 20 // 1 MiB
|
||||
chatUploadLimitMessage = "files exceed the 512 MiB upload limit"
|
||||
)
|
||||
|
||||
// safeChatID guards the {id} path segment against traversal — task ids are numeric,
|
||||
// session ids are alnum/_/- ; anything with "/" or ".." is rejected.
|
||||
@@ -79,16 +85,30 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, 500, "failed to create directory: "+err.Error())
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxChatUpload)
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxChatUpload+chatUploadMultipartOverhead)
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
writeErr(w, 400, "failed to parse the upload or size limit exceeded: "+err.Error())
|
||||
var sizeErr *http.MaxBytesError
|
||||
if errors.As(err, &sizeErr) {
|
||||
writeErr(w, http.StatusRequestEntityTooLarge, chatUploadLimitMessage)
|
||||
return
|
||||
}
|
||||
writeErr(w, 400, "failed to parse the upload: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer r.MultipartForm.RemoveAll()
|
||||
files := r.MultipartForm.File["file"]
|
||||
if len(files) == 0 {
|
||||
writeErr(w, 400, "missing upload file (form field file)")
|
||||
return
|
||||
}
|
||||
var totalSize int64
|
||||
for _, hdr := range files {
|
||||
totalSize += hdr.Size
|
||||
}
|
||||
if totalSize > maxChatUpload {
|
||||
writeErr(w, http.StatusRequestEntityTooLarge, chatUploadLimitMessage)
|
||||
return
|
||||
}
|
||||
out := make([]chatAttachment, 0, len(files))
|
||||
for _, hdr := range files {
|
||||
name := filepath.Base(hdr.Filename) // strip any path component
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type uploadLimitZeroReader struct{}
|
||||
|
||||
func (uploadLimitZeroReader) Read(p []byte) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestChatUpload512MiBBoundary(t *testing.T) {
|
||||
if maxChatUpload != 512<<20 {
|
||||
t.Fatal("attachment limit must be 512 MiB")
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
sizes []int64
|
||||
status int
|
||||
}{
|
||||
{"above previous limit", []int64{129 << 20}, http.StatusOK},
|
||||
{"exactly 512 MiB", []int64{512 << 20}, http.StatusOK},
|
||||
{"single file over limit", []int64{(512 << 20) + 1}, http.StatusRequestEntityTooLarge},
|
||||
{"combined files exactly at limit", []int64{256 << 20, 256 << 20}, http.StatusOK},
|
||||
{"combined files over limit", []int64{256 << 20, (256 << 20) + 1}, http.StatusRequestEntityTooLarge},
|
||||
{"request body over limit", []int64{(513 << 20) + 1}, http.StatusRequestEntityTooLarge},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
spillDir := t.TempDir()
|
||||
t.Setenv("TMPDIR", spillDir)
|
||||
body, writer := io.Pipe()
|
||||
form := multipart.NewWriter(writer)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
for _, size := range tc.sizes {
|
||||
part, err := form.CreateFormFile("file", "boundary.bin")
|
||||
if err == nil {
|
||||
_, err = io.CopyN(part, uploadLimitZeroReader{}, size)
|
||||
}
|
||||
if err != nil {
|
||||
writer.CloseWithError(err)
|
||||
done <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
err := form.Close()
|
||||
writer.CloseWithError(err)
|
||||
done <- err
|
||||
}()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/chat/upload?scope=staging&id=boundary-test", body)
|
||||
req.Header.Set("Content-Type", form.FormDataContentType())
|
||||
rec := httptest.NewRecorder()
|
||||
srv := &Server{m: &Manager{dir: workDir}}
|
||||
srv.chatUpload(rec, req)
|
||||
body.Close()
|
||||
writeErr := <-done
|
||||
if rec.Code != tc.status {
|
||||
t.Fatalf("status %d, want %d: %s", rec.Code, tc.status, rec.Body.String())
|
||||
}
|
||||
if tc.status == http.StatusOK {
|
||||
if writeErr != nil {
|
||||
t.Fatal(writeErr)
|
||||
}
|
||||
var reply struct {
|
||||
Attachments []chatAttachment `json:"attachments"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reply.Attachments) != len(tc.sizes) {
|
||||
t.Fatalf("got %d attachments", len(reply.Attachments))
|
||||
}
|
||||
for i, attachment := range reply.Attachments {
|
||||
info, err := os.Stat(attachment.Abs)
|
||||
if err != nil || attachment.Size != tc.sizes[i] || info.Size() != tc.sizes[i] {
|
||||
t.Fatalf("attachment size mismatch: %+v, %v", attachment, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var reply struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reply.Error != "files exceed the 512 MiB upload limit" {
|
||||
t.Fatalf("unexpected size-limit error: %q", reply.Error)
|
||||
}
|
||||
files, err := os.ReadDir(filepath.Join(workDir, "drafts", "boundary-test", "uploads"))
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatalf("oversized upload saved files: %v, %v", files, err)
|
||||
}
|
||||
}
|
||||
spillFiles, err := os.ReadDir(spillDir)
|
||||
if err != nil || len(spillFiles) != 0 {
|
||||
t.Fatalf("multipart files were not cleaned up: %v, %v", spillFiles, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -14,7 +14,8 @@
|
||||
"prepare": "husky",
|
||||
"generate:presets": "ts-node -P tsconfig.scripts.json src/scripts/generate-theme-presets.ts",
|
||||
"test:polling": "node --test scripts/polling.test.mjs",
|
||||
"test:replay": "node --test scripts/replay.test.mjs"
|
||||
"test:replay": "node --test scripts/replay.test.mjs",
|
||||
"test:upload": "node --test scripts/upload.test.mjs"
|
||||
},
|
||||
"lint-staged": {
|
||||
"*.{js,ts,jsx,tsx}": [
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import vm from "node:vm";
|
||||
import ts from "typescript";
|
||||
|
||||
const web = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const output = ts.transpileModule(fs.readFileSync(path.join(web, "src/lib/api.ts"), "utf8"), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
const MiB = 1024 * 1024;
|
||||
|
||||
function setup(mock = false) {
|
||||
const requests = [];
|
||||
const forms = [];
|
||||
const exports = {};
|
||||
vm.runInNewContext(output, {
|
||||
exports,
|
||||
require(name) {
|
||||
if (name === "@/lib/mock/enabled") return { MOCK: mock };
|
||||
if (name === "@/lib/mock/handler") return { mockHandle: () => assert.fail("unexpected mock request") };
|
||||
throw new Error("Unexpected import: " + name);
|
||||
},
|
||||
FormData: class {
|
||||
fields = [];
|
||||
constructor() {
|
||||
forms.push(this);
|
||||
}
|
||||
append(name, value) {
|
||||
this.fields.push([name, value]);
|
||||
}
|
||||
},
|
||||
fetch: async (url, init) => {
|
||||
requests.push({ url, init });
|
||||
return { ok: true, json: async () => ({ attachments: [] }) };
|
||||
},
|
||||
});
|
||||
return { api: exports.api, requests, forms };
|
||||
}
|
||||
|
||||
for (const scope of ["task", "session", "staging"]) {
|
||||
test(scope + ": rejects oversized single and combined files before FormData or fetch", async () => {
|
||||
const s = setup();
|
||||
for (const sizes of [[512 * MiB + 1], [256 * MiB, 256 * MiB + 1]]) {
|
||||
const files = sizes.map((size, i) => ({ name: i + ".apk", size }));
|
||||
await assert.rejects(s.api.chatUpload(scope, "42", files), /512 MiB.*combined size/);
|
||||
}
|
||||
assert.equal(s.forms.length, 0);
|
||||
assert.equal(s.requests.length, 0);
|
||||
});
|
||||
|
||||
test(scope + ": accepts ordinary files and exactly 512 MiB, including combined files", async () => {
|
||||
for (const sizes of [[129 * MiB], [512 * MiB], [256 * MiB, 256 * MiB]]) {
|
||||
const s = setup();
|
||||
const files = sizes.map((size, i) => ({ name: i + ".apk", size }));
|
||||
const result = await s.api.chatUpload(scope, "42", files);
|
||||
assert.equal(result.attachments.length, 0);
|
||||
assert.equal(s.requests.length, 1);
|
||||
assert.equal(s.requests[0].url, "/api/chat/upload?scope=" + scope + "&id=42");
|
||||
assert.equal(s.requests[0].init.method, "POST");
|
||||
assert.equal(s.requests[0].init.body, s.forms[0]);
|
||||
assert.deepEqual(s.forms[0].fields, files.map((file) => ["file", file]));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("mock uploads enforce the same combined limit", async () => {
|
||||
const s = setup(true);
|
||||
await assert.rejects(
|
||||
s.api.chatUpload("staging", "42", [{ name: "app.apk", size: 512 * MiB + 1 }]),
|
||||
/512 MiB/,
|
||||
);
|
||||
const result = await s.api.chatUpload("staging", "42", [{ name: "app.apk", size: 512 * MiB }]);
|
||||
assert.equal(result.attachments[0].size, 512 * MiB);
|
||||
assert.equal(s.requests.length, 0);
|
||||
});
|
||||
@@ -110,6 +110,9 @@ import type {
|
||||
WorkspaceListing,
|
||||
} from "@/lib/types";
|
||||
|
||||
// Combined file data per attachment upload; matches maxChatUpload in server/chatupload.go.
|
||||
const MAX_CHAT_UPLOAD_BYTES = 512 * 1024 * 1024;
|
||||
|
||||
function getToken(): string | null {
|
||||
if (typeof window === "undefined") return null;
|
||||
return localStorage.getItem("artex_token");
|
||||
@@ -935,6 +938,10 @@ export const api = {
|
||||
chatStatus: (taskId: string) => get<{ running: boolean }>(`/tasks/${taskId}/chat/status`),
|
||||
// Option 1, file upload: lands in the session/task working directory uploads/, returning a relative path the agent can Read.
|
||||
chatUpload: async (scope: "task" | "session" | "staging", id: string, files: File[]) => {
|
||||
const totalSize = files.reduce((total, file) => total + file.size, 0);
|
||||
if (totalSize > MAX_CHAT_UPLOAD_BYTES) {
|
||||
throw new Error("Files exceed the 512 MiB upload limit (combined size).");
|
||||
}
|
||||
if (MOCK)
|
||||
return {
|
||||
attachments: files.map((f) => ({
|
||||
|
||||
Reference in New Issue
Block a user