mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 22:50:55 +08:00
feat(server): add PostgreSQL readiness probe
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot App
parent
65045efae6
commit
2aa89ab786
+2
-2
@@ -283,11 +283,11 @@ func extractToken(r *http.Request) string {
|
||||
}
|
||||
|
||||
// requireAuth wraps h with JWT validation.
|
||||
// /api/auth/* and /api/health are exempt.
|
||||
// /api/auth/*, /api/health and /api/ready are exempt.
|
||||
func (s *Server) requireAuth(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := r.URL.Path
|
||||
if strings.HasPrefix(p, "/api/auth/") || p == "/api/health" {
|
||||
if strings.HasPrefix(p, "/api/auth/") || p == "/api/health" || p == "/api/ready" {
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
readyCacheTTL = time.Second
|
||||
readyPingTimeout = time.Second
|
||||
)
|
||||
|
||||
var errNoPostgres = errors.New("PostgreSQL is not configured")
|
||||
|
||||
type readyCache struct {
|
||||
mu sync.Mutex
|
||||
checked time.Time
|
||||
ready bool
|
||||
pending chan struct{}
|
||||
}
|
||||
|
||||
func (c *readyCache) check(ctx context.Context, ping func(context.Context) error) bool {
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
if !c.checked.IsZero() && time.Since(c.checked) < readyCacheTTL {
|
||||
ready := c.ready
|
||||
c.mu.Unlock()
|
||||
return ready
|
||||
}
|
||||
if done := c.pending; done != nil {
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-done:
|
||||
c.mu.Lock()
|
||||
ready := c.ready
|
||||
c.mu.Unlock()
|
||||
return ready
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
done := make(chan struct{})
|
||||
c.pending = done
|
||||
c.mu.Unlock()
|
||||
|
||||
pingCtx, cancel := context.WithTimeout(context.Background(), readyPingTimeout)
|
||||
err := ping(pingCtx)
|
||||
cancel()
|
||||
|
||||
c.mu.Lock()
|
||||
c.ready = err == nil
|
||||
c.checked = time.Now()
|
||||
c.pending = nil
|
||||
close(done)
|
||||
ready := c.ready
|
||||
c.mu.Unlock()
|
||||
return ready
|
||||
}
|
||||
|
||||
// ready reports PostgreSQL reachability without exposing connection details.
|
||||
// Its cached ping is bounded to one second and never runs application work.
|
||||
func (s *Server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
ready := s.readyPG.check(r.Context(), func(ctx context.Context) error {
|
||||
if s.m == nil || s.m.pg == nil {
|
||||
return errNoPostgres
|
||||
}
|
||||
return s.m.pg.PingContext(ctx)
|
||||
})
|
||||
status := http.StatusServiceUnavailable
|
||||
if ready {
|
||||
status = http.StatusOK
|
||||
}
|
||||
writeJSON(w, status, map[string]bool{"ready": ready})
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
type readyTestConnector struct {
|
||||
ping func(context.Context) error
|
||||
}
|
||||
|
||||
func (c readyTestConnector) Connect(context.Context) (driver.Conn, error) {
|
||||
return readyTestConn{ping: c.ping}, nil
|
||||
}
|
||||
|
||||
func (c readyTestConnector) Driver() driver.Driver { return readyTestDriver{} }
|
||||
|
||||
type readyTestDriver struct{}
|
||||
|
||||
func (readyTestDriver) Open(string) (driver.Conn, error) {
|
||||
return nil, errors.New("ready test driver must be opened through its connector")
|
||||
}
|
||||
|
||||
type readyTestConn struct {
|
||||
ping func(context.Context) error
|
||||
}
|
||||
|
||||
func (readyTestConn) Prepare(string) (driver.Stmt, error) {
|
||||
return nil, errors.New("not supported")
|
||||
}
|
||||
func (readyTestConn) Close() error { return nil }
|
||||
func (readyTestConn) Begin() (driver.Tx, error) { return nil, errors.New("not supported") }
|
||||
func (c readyTestConn) Ping(ctx context.Context) error {
|
||||
return c.ping(ctx)
|
||||
}
|
||||
|
||||
func newReadyServer(t *testing.T, ping func(context.Context) error) (*Server, *sql.DB) {
|
||||
t.Helper()
|
||||
sqlDB := sql.OpenDB(readyTestConnector{ping: ping})
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
return &Server{m: &Manager{pg: &db.DB{DB: sqlDB}}}, sqlDB
|
||||
}
|
||||
|
||||
func readyRequest(h http.Handler, method, path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(method, path, nil))
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertReadyResponse(t *testing.T, rec *httptest.ResponseRecorder, status int, ready bool) {
|
||||
t.Helper()
|
||||
if rec.Code != status {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, status, rec.Body.String())
|
||||
}
|
||||
var body map[string]bool
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode body %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
if len(body) != 1 || body["ready"] != ready {
|
||||
t.Fatalf("body = %s, want only ready:%t", rec.Body.String(), ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyRouteAvailabilityAndCompatibility(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
s, _ := newReadyServer(t, func(context.Context) error {
|
||||
calls.Add(1)
|
||||
return nil
|
||||
})
|
||||
h := s.Handler()
|
||||
|
||||
assertReadyResponse(t, readyRequest(h, http.MethodGet, "/api/ready"), http.StatusOK, true)
|
||||
if got := readyRequest(h, http.MethodGet, "/api/health"); got.Code != http.StatusOK || strings.TrimSpace(got.Body.String()) != `{"ok":true}` {
|
||||
t.Fatalf("unauthenticated health = %d %s, want 200 {\"ok\":true}", got.Code, got.Body.String())
|
||||
}
|
||||
if got := readyRequest(h, http.MethodGet, "/api/stats"); got.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthenticated stats = %d, want %d", got.Code, http.StatusUnauthorized)
|
||||
}
|
||||
if got := calls.Load(); got != 1 {
|
||||
t.Fatalf("database ping count = %d after readiness and liveness requests, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyOutageRecoveryAndErrorRedaction(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
var available atomic.Bool
|
||||
s, _ := newReadyServer(t, func(context.Context) error {
|
||||
calls.Add(1)
|
||||
if !available.Load() {
|
||||
return errors.New("secret PostgreSQL connection details")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
h := s.Handler()
|
||||
|
||||
first := readyRequest(h, http.MethodGet, "/api/ready")
|
||||
assertReadyResponse(t, first, http.StatusServiceUnavailable, false)
|
||||
if strings.Contains(first.Body.String(), "secret") {
|
||||
t.Fatalf("readiness response exposed database error: %s", first.Body.String())
|
||||
}
|
||||
|
||||
available.Store(true)
|
||||
s.readyPG.mu.Lock()
|
||||
s.readyPG.checked = time.Now().Add(-2 * readyCacheTTL)
|
||||
s.readyPG.mu.Unlock()
|
||||
assertReadyResponse(t, readyRequest(h, http.MethodGet, "/api/ready"), http.StatusOK, true)
|
||||
if got := calls.Load(); got != 2 {
|
||||
t.Fatalf("database ping count = %d, want outage and recovery checks", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyClosedDatabaseReturnsUnavailable(t *testing.T) {
|
||||
s, sqlDB := newReadyServer(t, func(context.Context) error { return nil })
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertReadyResponse(t, readyRequest(s.Handler(), http.MethodGet, "/api/ready"), http.StatusServiceUnavailable, false)
|
||||
}
|
||||
|
||||
func TestReadyPingTimeout(t *testing.T) {
|
||||
var cache readyCache
|
||||
started := make(chan struct{})
|
||||
start := time.Now()
|
||||
ready := cache.check(context.Background(), func(ctx context.Context) error {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
})
|
||||
if ready {
|
||||
t.Fatal("timed-out ping reported ready")
|
||||
}
|
||||
<-started
|
||||
if elapsed := time.Since(start); elapsed < readyPingTimeout || elapsed > 2*readyPingTimeout {
|
||||
t.Fatalf("ping timeout elapsed %s, want about %s", elapsed, readyPingTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
type readyWaitContext struct {
|
||||
context.Context
|
||||
waiting chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (c *readyWaitContext) Done() <-chan struct{} {
|
||||
c.once.Do(func() { close(c.waiting) })
|
||||
return c.Context.Done()
|
||||
}
|
||||
|
||||
func TestReadyCanceledWaiterDoesNotCancelSharedPing(t *testing.T) {
|
||||
var cache readyCache
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
pingResult := make(chan bool, 1)
|
||||
go func() {
|
||||
pingResult <- cache.check(context.Background(), func(context.Context) error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
<-started
|
||||
|
||||
waitCtx, cancel := context.WithCancel(context.Background())
|
||||
waitingCtx := &readyWaitContext{Context: waitCtx, waiting: make(chan struct{})}
|
||||
waiterResult := make(chan bool, 1)
|
||||
go func() { waiterResult <- cache.check(waitingCtx, func(context.Context) error { return nil }) }()
|
||||
<-waitingCtx.waiting
|
||||
cancel()
|
||||
select {
|
||||
case ready := <-waiterResult:
|
||||
if ready {
|
||||
t.Fatal("canceled waiter reported ready")
|
||||
}
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("canceled waiter did not return promptly")
|
||||
}
|
||||
|
||||
close(release)
|
||||
if ready := <-pingResult; !ready {
|
||||
t.Fatal("canceling a waiter canceled the shared ping")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyConcurrentRequestsShareOnePing(t *testing.T) {
|
||||
var cache readyCache
|
||||
var calls atomic.Int32
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
ping := func(context.Context) error {
|
||||
calls.Add(1)
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
|
||||
const callers = 20
|
||||
results := make([]bool, callers)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[0] = cache.check(context.Background(), ping)
|
||||
}()
|
||||
<-started
|
||||
|
||||
for i := 1; i < callers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
results[i] = cache.check(context.Background(), ping)
|
||||
}(i)
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if got := calls.Load(); got != 1 {
|
||||
t.Fatalf("database ping count = %d for %d callers, want 1", got, callers)
|
||||
}
|
||||
for i, ready := range results {
|
||||
if !ready {
|
||||
t.Errorf("caller %d reported not ready", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,6 +74,8 @@ type Server struct {
|
||||
// free slots off the same snapshot and over-promote past the limit.
|
||||
concMu sync.Mutex
|
||||
|
||||
readyPG readyCache
|
||||
|
||||
cfgMu sync.Mutex
|
||||
chatAgent *agent.ChatAgent // conversational runner for the chat page; nil w/o LLM
|
||||
// llmProv is the fully-decorated global provider (recorder + failover chain)
|
||||
@@ -745,6 +747,7 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /api/auth/change-password", s.authChangePassword)
|
||||
|
||||
mux.HandleFunc("GET /api/health", s.health)
|
||||
mux.HandleFunc("GET /api/ready", s.ready)
|
||||
mux.HandleFunc("GET /api/stats", s.stats)
|
||||
mux.HandleFunc("GET /api/logs", s.getLogs)
|
||||
mux.HandleFunc("GET /api/logs/history", s.getLogsHistory)
|
||||
|
||||
Reference in New Issue
Block a user