Merge pull request #252 from renaesop/codex/upload-500mib

fix: support 512 MiB task and chat attachments
This commit is contained in:
BaaRai
2026-11-17 16:22:43 +01:00
committed by GitHub
7 changed files with 227 additions and 5 deletions
+1
View File
@@ -148,6 +148,7 @@ jobs:
- run: npx tsc --noEmit
- run: npm run test:polling
- run: npm run test:replay
- run: npm run test:upload
- run: npm run build:static
python:
+4
View File
@@ -476,6 +476,10 @@ server {
# ssl_certificate / ssl_certificate_key ...
location / {
# 512 MiB combined file limit plus 1 MiB of multipart metadata.
client_max_body_size 513m;
proxy_request_buffering off;
proxy_pass http://127.0.0.1:8787;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
+24 -4
View File
@@ -2,6 +2,7 @@ package server
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
@@ -12,8 +13,13 @@ import (
"github.com/Autumn-27/artex/db"
)
// maxChatUpload caps a single chat-attachment upload request (memory + spill).
const maxChatUpload = 128 << 20 // 128 MiB
const (
// maxChatUpload caps the combined file data in one attachment upload.
maxChatUpload = 512 << 20 // 512 MiB
// Allow multipart headers and boundaries in addition to the file data.
chatUploadMultipartOverhead = 1 << 20 // 1 MiB
chatUploadLimitMessage = "files exceed the 512 MiB upload limit"
)
// safeChatID guards the {id} path segment against traversal — task ids are numeric,
// session ids are alnum/_/- ; anything with "/" or ".." is rejected.
@@ -79,16 +85,30 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
writeErr(w, 500, "failed to create directory: "+err.Error())
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxChatUpload)
r.Body = http.MaxBytesReader(w, r.Body, maxChatUpload+chatUploadMultipartOverhead)
if err := r.ParseMultipartForm(32 << 20); err != nil {
writeErr(w, 400, "failed to parse the upload or size limit exceeded: "+err.Error())
var sizeErr *http.MaxBytesError
if errors.As(err, &sizeErr) {
writeErr(w, http.StatusRequestEntityTooLarge, chatUploadLimitMessage)
return
}
writeErr(w, 400, "failed to parse the upload: "+err.Error())
return
}
defer r.MultipartForm.RemoveAll()
files := r.MultipartForm.File["file"]
if len(files) == 0 {
writeErr(w, 400, "missing upload file (form field file)")
return
}
var totalSize int64
for _, hdr := range files {
totalSize += hdr.Size
}
if totalSize > maxChatUpload {
writeErr(w, http.StatusRequestEntityTooLarge, chatUploadLimitMessage)
return
}
out := make([]chatAttachment, 0, len(files))
for _, hdr := range files {
name := filepath.Base(hdr.Filename) // strip any path component
+111
View File
@@ -0,0 +1,111 @@
package server
import (
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
type uploadLimitZeroReader struct{}
func (uploadLimitZeroReader) Read(p []byte) (int, error) {
clear(p)
return len(p), nil
}
func TestChatUpload512MiBBoundary(t *testing.T) {
if maxChatUpload != 512<<20 {
t.Fatal("attachment limit must be 512 MiB")
}
cases := []struct {
name string
sizes []int64
status int
}{
{"above previous limit", []int64{129 << 20}, http.StatusOK},
{"exactly 512 MiB", []int64{512 << 20}, http.StatusOK},
{"single file over limit", []int64{(512 << 20) + 1}, http.StatusRequestEntityTooLarge},
{"combined files exactly at limit", []int64{256 << 20, 256 << 20}, http.StatusOK},
{"combined files over limit", []int64{256 << 20, (256 << 20) + 1}, http.StatusRequestEntityTooLarge},
{"request body over limit", []int64{(513 << 20) + 1}, http.StatusRequestEntityTooLarge},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
workDir := t.TempDir()
spillDir := t.TempDir()
t.Setenv("TMPDIR", spillDir)
body, writer := io.Pipe()
form := multipart.NewWriter(writer)
done := make(chan error, 1)
go func() {
for _, size := range tc.sizes {
part, err := form.CreateFormFile("file", "boundary.bin")
if err == nil {
_, err = io.CopyN(part, uploadLimitZeroReader{}, size)
}
if err != nil {
writer.CloseWithError(err)
done <- err
return
}
}
err := form.Close()
writer.CloseWithError(err)
done <- err
}()
req := httptest.NewRequest(http.MethodPost, "/api/chat/upload?scope=staging&id=boundary-test", body)
req.Header.Set("Content-Type", form.FormDataContentType())
rec := httptest.NewRecorder()
srv := &Server{m: &Manager{dir: workDir}}
srv.chatUpload(rec, req)
body.Close()
writeErr := <-done
if rec.Code != tc.status {
t.Fatalf("status %d, want %d: %s", rec.Code, tc.status, rec.Body.String())
}
if tc.status == http.StatusOK {
if writeErr != nil {
t.Fatal(writeErr)
}
var reply struct {
Attachments []chatAttachment `json:"attachments"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
t.Fatal(err)
}
if len(reply.Attachments) != len(tc.sizes) {
t.Fatalf("got %d attachments", len(reply.Attachments))
}
for i, attachment := range reply.Attachments {
info, err := os.Stat(attachment.Abs)
if err != nil || attachment.Size != tc.sizes[i] || info.Size() != tc.sizes[i] {
t.Fatalf("attachment size mismatch: %+v, %v", attachment, err)
}
}
} else {
var reply struct {
Error string `json:"error"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
t.Fatal(err)
}
if reply.Error != "files exceed the 512 MiB upload limit" {
t.Fatalf("unexpected size-limit error: %q", reply.Error)
}
files, err := os.ReadDir(filepath.Join(workDir, "drafts", "boundary-test", "uploads"))
if err != nil || len(files) != 0 {
t.Fatalf("oversized upload saved files: %v, %v", files, err)
}
}
spillFiles, err := os.ReadDir(spillDir)
if err != nil || len(spillFiles) != 0 {
t.Fatalf("multipart files were not cleaned up: %v, %v", spillFiles, err)
}
})
}
}
+2 -1
View File
@@ -14,7 +14,8 @@
"prepare": "husky",
"generate:presets": "ts-node -P tsconfig.scripts.json src/scripts/generate-theme-presets.ts",
"test:polling": "node --test scripts/polling.test.mjs",
"test:replay": "node --test scripts/replay.test.mjs"
"test:replay": "node --test scripts/replay.test.mjs",
"test:upload": "node --test scripts/upload.test.mjs"
},
"lint-staged": {
"*.{js,ts,jsx,tsx}": [
+78
View File
@@ -0,0 +1,78 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { test } from "node:test";
import { fileURLToPath } from "node:url";
import vm from "node:vm";
import ts from "typescript";
const web = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const output = ts.transpileModule(fs.readFileSync(path.join(web, "src/lib/api.ts"), "utf8"), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
}).outputText;
const MiB = 1024 * 1024;
function setup(mock = false) {
const requests = [];
const forms = [];
const exports = {};
vm.runInNewContext(output, {
exports,
require(name) {
if (name === "@/lib/mock/enabled") return { MOCK: mock };
if (name === "@/lib/mock/handler") return { mockHandle: () => assert.fail("unexpected mock request") };
throw new Error("Unexpected import: " + name);
},
FormData: class {
fields = [];
constructor() {
forms.push(this);
}
append(name, value) {
this.fields.push([name, value]);
}
},
fetch: async (url, init) => {
requests.push({ url, init });
return { ok: true, json: async () => ({ attachments: [] }) };
},
});
return { api: exports.api, requests, forms };
}
for (const scope of ["task", "session", "staging"]) {
test(scope + ": rejects oversized single and combined files before FormData or fetch", async () => {
const s = setup();
for (const sizes of [[512 * MiB + 1], [256 * MiB, 256 * MiB + 1]]) {
const files = sizes.map((size, i) => ({ name: i + ".apk", size }));
await assert.rejects(s.api.chatUpload(scope, "42", files), /512 MiB.*combined size/);
}
assert.equal(s.forms.length, 0);
assert.equal(s.requests.length, 0);
});
test(scope + ": accepts ordinary files and exactly 512 MiB, including combined files", async () => {
for (const sizes of [[129 * MiB], [512 * MiB], [256 * MiB, 256 * MiB]]) {
const s = setup();
const files = sizes.map((size, i) => ({ name: i + ".apk", size }));
const result = await s.api.chatUpload(scope, "42", files);
assert.equal(result.attachments.length, 0);
assert.equal(s.requests.length, 1);
assert.equal(s.requests[0].url, "/api/chat/upload?scope=" + scope + "&id=42");
assert.equal(s.requests[0].init.method, "POST");
assert.equal(s.requests[0].init.body, s.forms[0]);
assert.deepEqual(s.forms[0].fields, files.map((file) => ["file", file]));
}
});
}
test("mock uploads enforce the same combined limit", async () => {
const s = setup(true);
await assert.rejects(
s.api.chatUpload("staging", "42", [{ name: "app.apk", size: 512 * MiB + 1 }]),
/512 MiB/,
);
const result = await s.api.chatUpload("staging", "42", [{ name: "app.apk", size: 512 * MiB }]);
assert.equal(result.attachments[0].size, 512 * MiB);
assert.equal(s.requests.length, 0);
});
+7
View File
@@ -110,6 +110,9 @@ import type {
WorkspaceListing,
} from "@/lib/types";
// Combined file data per attachment upload; matches maxChatUpload in server/chatupload.go.
const MAX_CHAT_UPLOAD_BYTES = 512 * 1024 * 1024;
function getToken(): string | null {
if (typeof window === "undefined") return null;
return localStorage.getItem("artex_token");
@@ -935,6 +938,10 @@ export const api = {
chatStatus: (taskId: string) => get<{ running: boolean }>(`/tasks/${taskId}/chat/status`),
// Option 1, file upload: lands in the session/task working directory uploads/, returning a relative path the agent can Read.
chatUpload: async (scope: "task" | "session" | "staging", id: string, files: File[]) => {
const totalSize = files.reduce((total, file) => total + file.size, 0);
if (totalSize > MAX_CHAT_UPLOAD_BYTES) {
throw new Error("Files exceed the 512 MiB upload limit (combined size).");
}
if (MOCK)
return {
attachments: files.map((f) => ({