feat: store layer with indexes, optimistic concurrency, GridFS, credential crypto (phase 2)
- mongo-driver v2 connection with bounded startup retry + /readyz gate - idempotent creation of all §4.9 indexes - UpdateVersioned: version-filtered updates, conflict vs not-found errors - AES-256-GCM Seal/Open for ticketing credentials, base64(nonce|ct) - GridFS file store: MIME sniffing, MAX_UPLOAD_MB cap, sha256 metadata - HMAC-signed short-lived file URL tokens (§5.1, 1h TTL) - integration tests against compose Mongo via docker-compose.test.yml overlay Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -36,3 +36,24 @@ Spec-silent choices, recorded as required by the build instructions.
|
|||||||
is not an error (containers receive env via compose `env_file`).
|
is not an error (containers receive env via compose `env_file`).
|
||||||
- **Makefile `seed`/`backup`/`restore` are failing stubs until Phase 12** so
|
- **Makefile `seed`/`backup`/`restore` are failing stubs until Phase 12** so
|
||||||
the targets exist but cannot be mistaken for working.
|
the targets exist but cannot be mistaken for working.
|
||||||
|
|
||||||
|
## Phase 2
|
||||||
|
|
||||||
|
- **Integration tests reach Mongo via `docker-compose.test.yml`**, an explicit
|
||||||
|
test-only overlay publishing Mongo on `127.0.0.1:27017`. The normal compose
|
||||||
|
file still never publishes Mongo (§12); the spec's integration-test
|
||||||
|
requirement (§2.1) needs host access, and a loopback-only opt-in overlay is
|
||||||
|
the smallest hole. Each run uses a `bountyboard_test_<ulid>` database and
|
||||||
|
drops it on cleanup.
|
||||||
|
- **`UpdateVersioned` rejects updates that touch `version`** and merges
|
||||||
|
`updatedAt`/`$inc version` into the caller's operators; it distinguishes
|
||||||
|
`ErrNotFound` from `ErrVersionConflict` with a follow-up existence check.
|
||||||
|
- **File MIME type = sniffed (`http.DetectContentType`) unless inconclusive**
|
||||||
|
(`application/octet-stream`), in which case the client-declared type is
|
||||||
|
kept. A confident sniff overrides a lying declaration.
|
||||||
|
- **sha256 of uploads is recorded post-upload** via an update on `fs.files`
|
||||||
|
metadata (GridFS metadata must be supplied before streaming; the hash is
|
||||||
|
only known after).
|
||||||
|
- **Signed file tokens** are `base64url(exp || hmac-sha256(fileID|exp))` with
|
||||||
|
a key derived as `sha256("bountyboard/file-url/v1" + SESSION_SECRET)` so
|
||||||
|
file tokens can never collide with other uses of the session secret.
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ run:
|
|||||||
test:
|
test:
|
||||||
$(GO) test ./...
|
$(GO) test ./...
|
||||||
|
|
||||||
|
# Requires Mongo reachable on 127.0.0.1:27017 (or MONGO_TEST_URI):
|
||||||
|
# docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo
|
||||||
test-integration:
|
test-integration:
|
||||||
$(GO) test -tags=integration ./...
|
$(GO) test -tags=integration ./...
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
# Progress
|
# Progress
|
||||||
|
|
||||||
- Phase 1 (scaffold): repo layout, Makefile, .env loader + validated config, slog JSON, ULID pkg, trusted-proxy middleware, /healthz /readyz /metricsz, Dockerfile + compose (mongo+app, healthchecks, localhost-bound 8787), graceful shutdown — build/vet/tests green, smoke PASS.
|
- Phase 1 (scaffold): repo layout, Makefile, .env loader + validated config, slog JSON, ULID pkg, trusted-proxy middleware, /healthz /readyz /metricsz, Dockerfile + compose (mongo+app, healthchecks, localhost-bound 8787), graceful shutdown — build/vet/tests green, smoke PASS.
|
||||||
|
- Phase 2 (store layer): mongo-driver v2 connection with startup retry, §4.9 indexes idempotent, optimistic-concurrency UpdateVersioned (ErrVersionConflict/ErrNotFound), AES-256-GCM credential crypto, HMAC-signed file URL tokens (1h TTL), GridFS store with MIME sniffing + size cap + sha256, /readyz mongo check (verified 503 on stopped Mongo) — unit + integration tests green.
|
||||||
|
|
||||||
|
|||||||
+23
-3
@@ -8,10 +8,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
"bountyboard/internal/config"
|
"bountyboard/internal/config"
|
||||||
httpx "bountyboard/internal/http"
|
httpx "bountyboard/internal/http"
|
||||||
"bountyboard/internal/metrics"
|
"bountyboard/internal/metrics"
|
||||||
|
"bountyboard/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -33,10 +35,28 @@ func run() error {
|
|||||||
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||||
slog.SetDefault(log)
|
slog.SetDefault(log)
|
||||||
|
|
||||||
reg := metrics.NewRegistry()
|
|
||||||
srv := httpx.New(cfg, log, reg)
|
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
|
st, err := store.Connect(ctx, cfg, log)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := st.Close(closeCtx); err != nil {
|
||||||
|
log.Error("close mongo", "err", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
reg := metrics.NewRegistry()
|
||||||
|
srv := httpx.New(cfg, log, reg)
|
||||||
|
srv.AddReadinessCheck(httpx.ReadinessCheck{
|
||||||
|
Name: "mongo",
|
||||||
|
Required: true,
|
||||||
|
Probe: st.Ping,
|
||||||
|
})
|
||||||
|
|
||||||
return srv.Run(ctx)
|
return srv.Run(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# Test-only overlay: publishes Mongo on loopback so host-run integration
|
||||||
|
# tests can reach it. Never used in normal deployment (§12 keeps Mongo
|
||||||
|
# unpublished):
|
||||||
|
# docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo
|
||||||
|
services:
|
||||||
|
mongo:
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:27017:27017"
|
||||||
@@ -1,3 +1,16 @@
|
|||||||
module bountyboard
|
module bountyboard
|
||||||
|
|
||||||
go 1.26
|
go 1.26
|
||||||
|
|
||||||
|
require go.mongodb.org/mongo-driver/v2 v2.6.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/klauspost/compress v1.17.6 // indirect
|
||||||
|
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||||
|
github.com/xdg-go/scram v1.2.0 // indirect
|
||||||
|
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||||
|
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||||
|
golang.org/x/crypto v0.33.0 // indirect
|
||||||
|
golang.org/x/sync v0.11.0 // indirect
|
||||||
|
golang.org/x/text v0.22.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
|
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
|
||||||
|
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||||
|
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||||
|
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||||
|
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||||
|
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||||
|
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||||
|
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||||
|
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||||
|
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||||
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
|
go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8=
|
||||||
|
go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
|
||||||
|
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||||
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
|
||||||
|
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
|
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||||
|
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
|
||||||
|
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
// Package crypto provides AES-256-GCM sealing for customer ticketing
|
||||||
|
// credentials stored in Mongo (§4.2): ciphertext format base64(nonce|sealed).
|
||||||
|
package crypto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/aes"
|
||||||
|
"crypto/cipher"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
const keyLen = 32
|
||||||
|
|
||||||
|
// Seal encrypts plaintext with AES-256-GCM and returns
|
||||||
|
// base64(nonce|ciphertext) for storage.
|
||||||
|
func Seal(key, plaintext []byte) (string, error) {
|
||||||
|
gcm, err := newGCM(key)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
nonce := make([]byte, gcm.NonceSize())
|
||||||
|
if _, err := rand.Read(nonce); err != nil {
|
||||||
|
return "", fmt.Errorf("nonce: %w", err)
|
||||||
|
}
|
||||||
|
sealed := gcm.Seal(nonce, nonce, plaintext, nil)
|
||||||
|
return base64.StdEncoding.EncodeToString(sealed), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open decrypts a value produced by Seal.
|
||||||
|
func Open(key []byte, encoded string) ([]byte, error) {
|
||||||
|
gcm, err := newGCM(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(encoded)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode ciphertext: %w", err)
|
||||||
|
}
|
||||||
|
if len(raw) < gcm.NonceSize() {
|
||||||
|
return nil, errors.New("ciphertext shorter than nonce")
|
||||||
|
}
|
||||||
|
plaintext, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decrypt: %w", err)
|
||||||
|
}
|
||||||
|
return plaintext, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGCM(key []byte) (cipher.AEAD, error) {
|
||||||
|
if len(key) != keyLen {
|
||||||
|
return nil, fmt.Errorf("key must be %d bytes, got %d", keyLen, len(key))
|
||||||
|
}
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("aes: %w", err)
|
||||||
|
}
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gcm: %w", err)
|
||||||
|
}
|
||||||
|
return gcm, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package crypto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func testKey(t *testing.T) []byte {
|
||||||
|
t.Helper()
|
||||||
|
key := make([]byte, 32)
|
||||||
|
if _, err := rand.Read(key); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSealOpenRoundTrip(t *testing.T) {
|
||||||
|
key := testKey(t)
|
||||||
|
tests := [][]byte{
|
||||||
|
[]byte(`{"email":"a@b.c","apiToken":"secret-token"}`),
|
||||||
|
[]byte(""),
|
||||||
|
[]byte(strings.Repeat("x", 100000)),
|
||||||
|
{0x00, 0xFF, 0x10},
|
||||||
|
}
|
||||||
|
for _, plaintext := range tests {
|
||||||
|
sealed, err := Seal(key, plaintext)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Seal: %v", err)
|
||||||
|
}
|
||||||
|
got, err := Open(key, sealed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Open: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, plaintext) {
|
||||||
|
t.Errorf("round trip mismatch for %d-byte input", len(plaintext))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSealIsNonDeterministic(t *testing.T) {
|
||||||
|
key := testKey(t)
|
||||||
|
a, _ := Seal(key, []byte("same"))
|
||||||
|
b, _ := Seal(key, []byte("same"))
|
||||||
|
if a == b {
|
||||||
|
t.Fatal("two seals of the same plaintext must differ (random nonce)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRejectsTampering(t *testing.T) {
|
||||||
|
key := testKey(t)
|
||||||
|
sealed, err := Seal(key, []byte("credentials"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := Open(testKey(t), sealed); err == nil {
|
||||||
|
t.Error("wrong key must fail")
|
||||||
|
}
|
||||||
|
if _, err := Open(key, "!!not-base64!!"); err == nil {
|
||||||
|
t.Error("invalid base64 must fail")
|
||||||
|
}
|
||||||
|
if _, err := Open(key, "AAAA"); err == nil {
|
||||||
|
t.Error("too-short ciphertext must fail")
|
||||||
|
}
|
||||||
|
corrupted := []byte(sealed)
|
||||||
|
mid := len(corrupted) / 2
|
||||||
|
if corrupted[mid] == 'A' {
|
||||||
|
corrupted[mid] = 'B'
|
||||||
|
} else {
|
||||||
|
corrupted[mid] = 'A'
|
||||||
|
}
|
||||||
|
if _, err := Open(key, string(corrupted)); err == nil {
|
||||||
|
t.Error("corrupted ciphertext must fail authentication")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKeyLengthEnforced(t *testing.T) {
|
||||||
|
if _, err := Seal(make([]byte, 16), []byte("x")); err == nil {
|
||||||
|
t.Error("16-byte key must be rejected")
|
||||||
|
}
|
||||||
|
if _, err := Open(make([]byte, 31), "AAAA"); err == nil {
|
||||||
|
t.Error("31-byte key must be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// Package files implements upload storage (GridFS) and the short-lived
|
||||||
|
// HMAC-signed file URLs from §5.1: /files/{id}?st=<token> lets external
|
||||||
|
// services (atomizer, work performer) fetch attachments without a session.
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/binary"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultTokenTTL is the §5.1 signed-URL lifetime.
|
||||||
|
const DefaultTokenTTL = time.Hour
|
||||||
|
|
||||||
|
// signingKey derives a dedicated key from the session secret so file tokens
|
||||||
|
// can never be confused with anything else signed by the same secret.
|
||||||
|
func signingKey(secret []byte) []byte {
|
||||||
|
h := sha256.New()
|
||||||
|
h.Write([]byte("bountyboard/file-url/v1"))
|
||||||
|
h.Write(secret)
|
||||||
|
return h.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mac(key []byte, fileID string, exp int64) []byte {
|
||||||
|
m := hmac.New(sha256.New, key)
|
||||||
|
fmt.Fprintf(m, "%s|%d", fileID, exp)
|
||||||
|
return m.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignToken returns the st token granting access to fileID until expiry.
|
||||||
|
// Format: base64url(bigendian-int64-exp || hmac-sha256(fileID|exp)).
|
||||||
|
func SignToken(secret []byte, fileID string, expiresAt time.Time) string {
|
||||||
|
exp := expiresAt.Unix()
|
||||||
|
key := signingKey(secret)
|
||||||
|
buf := make([]byte, 8, 8+sha256.Size)
|
||||||
|
binary.BigEndian.PutUint64(buf, uint64(exp))
|
||||||
|
buf = append(buf, mac(key, fileID, exp)...)
|
||||||
|
return base64.RawURLEncoding.EncodeToString(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifyToken reports whether token grants access to fileID at time now.
|
||||||
|
func VerifyToken(secret []byte, fileID, token string, now time.Time) bool {
|
||||||
|
raw, err := base64.RawURLEncoding.DecodeString(token)
|
||||||
|
if err != nil || len(raw) != 8+sha256.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
exp := int64(binary.BigEndian.Uint64(raw[:8]))
|
||||||
|
if now.Unix() > exp {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
expected := mac(signingKey(secret), fileID, exp)
|
||||||
|
return hmac.Equal(raw[8:], expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignedURL builds the absolute fetch URL for a stored file.
|
||||||
|
func SignedURL(appBaseURL string, secret []byte, fileID string, expiresAt time.Time) string {
|
||||||
|
return fmt.Sprintf("%s/files/%s?st=%s", appBaseURL, fileID, SignToken(secret, fileID, expiresAt))
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSignVerifyRoundTrip(t *testing.T) {
|
||||||
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
now := time.Now()
|
||||||
|
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
|
||||||
|
|
||||||
|
if !VerifyToken(secret, "01JFILE", token, now) {
|
||||||
|
t.Fatal("fresh token must verify")
|
||||||
|
}
|
||||||
|
if !VerifyToken(secret, "01JFILE", token, now.Add(59*time.Minute)) {
|
||||||
|
t.Fatal("token must verify just before expiry")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyRejections(t *testing.T) {
|
||||||
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
now := time.Now()
|
||||||
|
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fileID string
|
||||||
|
token string
|
||||||
|
at time.Time
|
||||||
|
}{
|
||||||
|
{"expired", "01JFILE", token, now.Add(2 * time.Hour)},
|
||||||
|
{"wrong file", "01JOTHER", token, now},
|
||||||
|
{"garbage token", "01JFILE", "not-a-token", now},
|
||||||
|
{"empty token", "01JFILE", "", now},
|
||||||
|
{"truncated token", "01JFILE", token[:len(token)-10], now},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if VerifyToken(secret, tt.fileID, tt.token, tt.at) {
|
||||||
|
t.Error("must not verify")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if VerifyToken([]byte("another-secret-another-secret-xx"), "01JFILE", token, now) {
|
||||||
|
t.Error("different secret must not verify")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignedURLShape(t *testing.T) {
|
||||||
|
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||||
|
u := SignedURL("http://localhost:8787", secret, "01JFILE", time.Now().Add(time.Hour))
|
||||||
|
if !strings.HasPrefix(u, "http://localhost:8787/files/01JFILE?st=") {
|
||||||
|
t.Fatalf("unexpected URL %q", u)
|
||||||
|
}
|
||||||
|
token := u[strings.Index(u, "?st=")+4:]
|
||||||
|
if !VerifyToken(secret, "01JFILE", token, time.Now()) {
|
||||||
|
t.Fatal("token embedded in URL must verify")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||||
|
|
||||||
|
"bountyboard/internal/ulid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrTooLarge is returned when an upload exceeds MAX_UPLOAD_MB.
|
||||||
|
var ErrTooLarge = errors.New("file exceeds maximum upload size")
|
||||||
|
|
||||||
|
// ErrNotFound is returned when no stored file has the given id.
|
||||||
|
var ErrNotFound = errors.New("file not found")
|
||||||
|
|
||||||
|
// Valid metadata scopes (§4.7); access control at the HTTP layer keys off
|
||||||
|
// these.
|
||||||
|
const (
|
||||||
|
ScopeAvatar = "avatar"
|
||||||
|
ScopeChat = "chat"
|
||||||
|
ScopeTask = "task"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Meta struct {
|
||||||
|
ID string
|
||||||
|
Name string
|
||||||
|
OwnerID string
|
||||||
|
Scope string
|
||||||
|
MimeType string
|
||||||
|
Size int64
|
||||||
|
SHA256 string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store wraps the GridFS bucket with MIME sniffing, a size cap, and sha256
|
||||||
|
// recording.
|
||||||
|
type Store struct {
|
||||||
|
bucket *mongo.GridFSBucket
|
||||||
|
maxBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStore(db *mongo.Database, maxUploadMB int) *Store {
|
||||||
|
return &Store{
|
||||||
|
bucket: db.GridFSBucket(),
|
||||||
|
maxBytes: int64(maxUploadMB) * 1024 * 1024,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save streams r into GridFS. The content type is sniffed from the first
|
||||||
|
// 512 bytes; declaredMime is used only when sniffing is inconclusive.
|
||||||
|
func (s *Store) Save(ctx context.Context, scope, ownerID, name, declaredMime string, r io.Reader) (*Meta, error) {
|
||||||
|
head := make([]byte, 512)
|
||||||
|
n, err := io.ReadFull(r, head)
|
||||||
|
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||||
|
return nil, fmt.Errorf("read upload head: %w", err)
|
||||||
|
}
|
||||||
|
head = head[:n]
|
||||||
|
|
||||||
|
mimeType := http.DetectContentType(head)
|
||||||
|
if mimeType == "application/octet-stream" && declaredMime != "" {
|
||||||
|
mimeType = declaredMime
|
||||||
|
}
|
||||||
|
|
||||||
|
id := ulid.New()
|
||||||
|
hasher := sha256.New()
|
||||||
|
// Allow one byte over the cap so an oversize upload is detectable.
|
||||||
|
body := &countingReader{r: io.TeeReader(
|
||||||
|
io.LimitReader(io.MultiReader(bytes.NewReader(head), r), s.maxBytes+1),
|
||||||
|
hasher)}
|
||||||
|
err = s.bucket.UploadFromStreamWithID(ctx, id, name, body,
|
||||||
|
options.GridFSUpload().SetMetadata(bson.M{
|
||||||
|
"ownerId": ownerID,
|
||||||
|
"scope": scope,
|
||||||
|
"mimeType": mimeType,
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gridfs upload: %w", err)
|
||||||
|
}
|
||||||
|
size := body.n
|
||||||
|
if size > s.maxBytes {
|
||||||
|
if delErr := s.bucket.Delete(ctx, id); delErr != nil {
|
||||||
|
return nil, fmt.Errorf("delete oversize upload: %w", delErr)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w (limit %d bytes)", ErrTooLarge, s.maxBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
sum := hex.EncodeToString(hasher.Sum(nil))
|
||||||
|
_, err = s.bucket.GetFilesCollection().UpdateOne(ctx,
|
||||||
|
bson.M{"_id": id},
|
||||||
|
bson.M{"$set": bson.M{"metadata.sha256": sum}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("record sha256: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Meta{
|
||||||
|
ID: id, Name: name, OwnerID: ownerID, Scope: scope,
|
||||||
|
MimeType: mimeType, Size: size, SHA256: sum,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open returns a reader over the stored file plus its metadata. The caller
|
||||||
|
// must Close the stream.
|
||||||
|
func (s *Store) Open(ctx context.Context, id string) (io.ReadCloser, *Meta, error) {
|
||||||
|
ds, err := s.bucket.OpenDownloadStream(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, mongo.ErrFileNotFound) {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", id, ErrNotFound)
|
||||||
|
}
|
||||||
|
return nil, nil, fmt.Errorf("gridfs open %s: %w", id, err)
|
||||||
|
}
|
||||||
|
f := ds.GetFile()
|
||||||
|
var md struct {
|
||||||
|
OwnerID string `bson:"ownerId"`
|
||||||
|
Scope string `bson:"scope"`
|
||||||
|
MimeType string `bson:"mimeType"`
|
||||||
|
SHA256 string `bson:"sha256"`
|
||||||
|
}
|
||||||
|
if f.Metadata != nil {
|
||||||
|
if err := bson.Unmarshal(f.Metadata, &md); err != nil {
|
||||||
|
ds.Close()
|
||||||
|
return nil, nil, fmt.Errorf("decode file metadata %s: %w", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ds, &Meta{
|
||||||
|
ID: id, Name: f.Name, OwnerID: md.OwnerID, Scope: md.Scope,
|
||||||
|
MimeType: md.MimeType, Size: f.Length, SHA256: md.SHA256,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes the file and its chunks.
|
||||||
|
func (s *Store) Delete(ctx context.Context, id string) error {
|
||||||
|
if err := s.bucket.Delete(ctx, id); err != nil {
|
||||||
|
if errors.Is(err, mongo.ErrFileNotFound) {
|
||||||
|
return fmt.Errorf("%s: %w", id, ErrNotFound)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("gridfs delete %s: %w", id, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// countingReader tracks how many bytes the upload actually consumed, since
|
||||||
|
// the driver's upload helper does not report a size.
|
||||||
|
type countingReader struct {
|
||||||
|
r io.Reader
|
||||||
|
n int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := c.r.Read(p)
|
||||||
|
c.n += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignedURLFor is a convenience for handing attachments to external services.
|
||||||
|
func (s *Store) SignedURLFor(appBaseURL string, secret []byte, fileID string) string {
|
||||||
|
return SignedURL(appBaseURL, secret, fileID, time.Now().Add(DefaultTokenTTL))
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"bountyboard/internal/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSaveOpenRoundTrip(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
s := NewStore(db, 25)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
content := []byte("%PDF-1.4 fake pdf content for sniffing")
|
||||||
|
meta, err := s.Save(ctx, ScopeTask, "01JOWNER", "spec.pdf", "", bytes.NewReader(content))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Save: %v", err)
|
||||||
|
}
|
||||||
|
if meta.MimeType != "application/pdf" {
|
||||||
|
t.Errorf("sniffed mime = %q, want application/pdf", meta.MimeType)
|
||||||
|
}
|
||||||
|
if meta.Size != int64(len(content)) {
|
||||||
|
t.Errorf("size = %d, want %d", meta.Size, len(content))
|
||||||
|
}
|
||||||
|
wantSum := sha256.Sum256(content)
|
||||||
|
if meta.SHA256 != hex.EncodeToString(wantSum[:]) {
|
||||||
|
t.Errorf("sha256 mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
rc, got, err := s.Open(ctx, meta.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Open: %v", err)
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
data, err := io.ReadAll(rc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data, content) {
|
||||||
|
t.Error("downloaded content differs")
|
||||||
|
}
|
||||||
|
if got.OwnerID != "01JOWNER" || got.Scope != ScopeTask || got.Name != "spec.pdf" {
|
||||||
|
t.Errorf("metadata round trip: %+v", got)
|
||||||
|
}
|
||||||
|
if got.SHA256 != meta.SHA256 || got.MimeType != meta.MimeType {
|
||||||
|
t.Errorf("stored metadata mismatch: %+v vs %+v", got, meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeclaredMimeUsedWhenSniffInconclusive(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
s := NewStore(db, 25)
|
||||||
|
|
||||||
|
meta, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "data.bin",
|
||||||
|
"application/x-custom", bytes.NewReader([]byte{0x01, 0x02, 0x03}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if meta.MimeType != "application/x-custom" {
|
||||||
|
t.Errorf("mime = %q, want declared application/x-custom", meta.MimeType)
|
||||||
|
}
|
||||||
|
|
||||||
|
// But a confident sniff overrides a lying declaration.
|
||||||
|
meta2, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "fake.txt",
|
||||||
|
"text/plain", strings.NewReader("\x89PNG\r\n\x1a\n123456789"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if meta2.MimeType != "image/png" {
|
||||||
|
t.Errorf("mime = %q, want sniffed image/png", meta2.MimeType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSizeLimitEnforced(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
s := NewStore(db, 1) // 1 MiB cap
|
||||||
|
|
||||||
|
big := io.LimitReader(neverEnding('x'), 1<<20+1)
|
||||||
|
_, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "big.bin", "", big)
|
||||||
|
if !errors.Is(err, ErrTooLarge) {
|
||||||
|
t.Fatalf("want ErrTooLarge, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No orphaned file documents may remain.
|
||||||
|
n, err := s.bucket.GetFilesCollection().CountDocuments(context.Background(), struct{}{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 0 {
|
||||||
|
t.Errorf("oversize upload left %d file docs behind", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly at the limit is fine.
|
||||||
|
ok := io.LimitReader(neverEnding('x'), 1<<20)
|
||||||
|
if _, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "exact.bin", "", ok); err != nil {
|
||||||
|
t.Fatalf("upload at exactly the limit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenMissingFile(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
s := NewStore(db, 25)
|
||||||
|
if _, _, err := s.Open(context.Background(), "01JNOPE"); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDelete(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
s := NewStore(db, 25)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
meta, err := s.Save(ctx, ScopeAvatar, "01JOWNER", "a.txt", "", strings.NewReader("hello"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.Delete(ctx, meta.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := s.Open(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound after delete, got %v", err)
|
||||||
|
}
|
||||||
|
if err := s.Delete(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("double delete: want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// neverEnding is an infinite reader of one repeated byte.
|
||||||
|
type neverEnding byte
|
||||||
|
|
||||||
|
func (b neverEnding) Read(p []byte) (int, error) {
|
||||||
|
for i := range p {
|
||||||
|
p[i] = byte(b)
|
||||||
|
}
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EnsureIndexes creates every index from spec §4.9. CreateMany is idempotent:
|
||||||
|
// existing identical indexes are no-ops, so this runs safely at every startup.
|
||||||
|
func EnsureIndexes(ctx context.Context, db *mongo.Database) error {
|
||||||
|
plan := map[string][]mongo.IndexModel{
|
||||||
|
"users": {
|
||||||
|
{Keys: bson.D{{Key: "email", Value: 1}}, Options: options.Index().SetUnique(true)},
|
||||||
|
{Keys: bson.D{{Key: "auth.oidc.issuer", Value: 1}, {Key: "auth.oidc.subject", Value: 1}},
|
||||||
|
Options: options.Index().SetUnique(true).SetSparse(true)},
|
||||||
|
},
|
||||||
|
"customers": {
|
||||||
|
{Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index().SetUnique(true)},
|
||||||
|
{Keys: bson.D{{Key: "consultantIds", Value: 1}}},
|
||||||
|
},
|
||||||
|
"pools": {
|
||||||
|
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "developerId", Value: 1}},
|
||||||
|
Options: options.Index().SetUnique(true)},
|
||||||
|
{Keys: bson.D{{Key: "developerId", Value: 1}}},
|
||||||
|
},
|
||||||
|
"tasks": {
|
||||||
|
{Keys: bson.D{{Key: "customerId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "assignee.userId", Value: 1}, {Key: "status", Value: 1}}},
|
||||||
|
{Keys: bson.D{{Key: "rootId", Value: 1}}},
|
||||||
|
{Keys: bson.D{{Key: "parentId", Value: 1}}},
|
||||||
|
{Keys: bson.D{{Key: "external.system", Value: 1}, {Key: "external.key", Value: 1}, {Key: "customerId", Value: 1}},
|
||||||
|
Options: options.Index().SetUnique(true).SetSparse(true)},
|
||||||
|
{Keys: bson.D{{Key: "title", Value: "text"}, {Key: "description", Value: "text"}}},
|
||||||
|
},
|
||||||
|
"bountyAwards": {
|
||||||
|
{Keys: bson.D{{Key: "developerId", Value: 1}, {Key: "awardedAt", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "awardedAt", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "customerId", Value: 1}, {Key: "awardedAt", Value: -1}}},
|
||||||
|
},
|
||||||
|
"conversations": {
|
||||||
|
{Keys: bson.D{{Key: "participantIds", Value: 1}, {Key: "lastMessageAt", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "kind", Value: 1}, {Key: "customerId", Value: 1}}},
|
||||||
|
},
|
||||||
|
"messages": {
|
||||||
|
// ULID _id is time-ordered, so this serves history pagination.
|
||||||
|
{Keys: bson.D{{Key: "conversationId", Value: 1}, {Key: "_id", Value: 1}}},
|
||||||
|
},
|
||||||
|
"notifications": {
|
||||||
|
{Keys: bson.D{{Key: "userId", Value: 1}, {Key: "readAt", Value: 1}, {Key: "createdAt", Value: -1}}},
|
||||||
|
},
|
||||||
|
"sessions": {
|
||||||
|
{Keys: bson.D{{Key: "expiresAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)},
|
||||||
|
{Keys: bson.D{{Key: "userId", Value: 1}}},
|
||||||
|
},
|
||||||
|
"auditLog": {
|
||||||
|
{Keys: bson.D{{Key: "at", Value: -1}}},
|
||||||
|
{Keys: bson.D{{Key: "entityId", Value: 1}, {Key: "at", Value: -1}}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for coll, models := range plan {
|
||||||
|
if _, err := db.Collection(coll).Indexes().CreateMany(ctx, models); err != nil {
|
||||||
|
return fmt.Errorf("create indexes for %s: %w", coll, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Optimistic-concurrency errors. HTTP handlers translate ErrVersionConflict
|
||||||
|
// into a 409 with a "conflict" envelope (and the QoL reload toast).
|
||||||
|
var (
|
||||||
|
ErrNotFound = errors.New("document not found")
|
||||||
|
ErrVersionConflict = errors.New("version conflict")
|
||||||
|
)
|
||||||
|
|
||||||
|
// UpdateVersioned applies update to the document only if its version still
|
||||||
|
// matches, bumping version and updatedAt atomically. update uses normal
|
||||||
|
// operators ($set/$push/...); the helper merges its bookkeeping into them.
|
||||||
|
func UpdateVersioned(ctx context.Context, coll *mongo.Collection, id string, version int64, update bson.M) error {
|
||||||
|
merged := bson.M{}
|
||||||
|
for op, fields := range update {
|
||||||
|
merged[op] = fields
|
||||||
|
}
|
||||||
|
set, _ := merged["$set"].(bson.M)
|
||||||
|
if set == nil {
|
||||||
|
set = bson.M{}
|
||||||
|
}
|
||||||
|
set["updatedAt"] = time.Now().UTC()
|
||||||
|
merged["$set"] = set
|
||||||
|
inc, _ := merged["$inc"].(bson.M)
|
||||||
|
if inc == nil {
|
||||||
|
inc = bson.M{}
|
||||||
|
}
|
||||||
|
if _, clash := inc["version"]; clash {
|
||||||
|
return fmt.Errorf("update must not touch version itself")
|
||||||
|
}
|
||||||
|
inc["version"] = int64(1)
|
||||||
|
merged["$inc"] = inc
|
||||||
|
|
||||||
|
res, err := coll.UpdateOne(ctx, bson.M{"_id": id, "version": version}, merged)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("versioned update %s/%s: %w", coll.Name(), id, err)
|
||||||
|
}
|
||||||
|
if res.MatchedCount == 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Distinguish "gone" from "someone got there first".
|
||||||
|
n, err := coll.CountDocuments(ctx, bson.M{"_id": id})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("versioned update %s/%s: existence check: %w", coll.Name(), id, err)
|
||||||
|
}
|
||||||
|
if n == 0 {
|
||||||
|
return fmt.Errorf("%s/%s: %w", coll.Name(), id, ErrNotFound)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s/%s at version %d: %w", coll.Name(), id, version, ErrVersionConflict)
|
||||||
|
}
|
||||||
|
|
||||||
|
// InsertOne inserts a document map, stamping createdAt/updatedAt/version so
|
||||||
|
// every collection follows the §4 conventions.
|
||||||
|
func InsertOne(ctx context.Context, coll *mongo.Collection, id string, doc bson.M) error {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
doc["_id"] = id
|
||||||
|
doc["createdAt"] = now
|
||||||
|
doc["updatedAt"] = now
|
||||||
|
doc["version"] = int64(1)
|
||||||
|
if _, err := coll.InsertOne(ctx, doc); err != nil {
|
||||||
|
return fmt.Errorf("insert %s/%s: %w", coll.Name(), id, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
// Package store owns all MongoDB access: connection lifecycle, index
|
||||||
|
// bootstrap, optimistic-concurrency updates, and (via internal/files) GridFS.
|
||||||
|
// No other package talks to Mongo directly.
|
||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
|
||||||
|
|
||||||
|
"bountyboard/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Store struct {
|
||||||
|
Client *mongo.Client
|
||||||
|
DB *mongo.Database
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect builds the client (the driver dials lazily) and verifies
|
||||||
|
// connectivity with a bounded retry so a compose race at boot does not kill
|
||||||
|
// the app, then ensures all indexes.
|
||||||
|
func Connect(ctx context.Context, cfg *config.Config, log *slog.Logger) (*Store, error) {
|
||||||
|
opts := options.Client().
|
||||||
|
ApplyURI(cfg.MongoURI).
|
||||||
|
SetAppName("bountyboard").
|
||||||
|
SetServerSelectionTimeout(5 * time.Second)
|
||||||
|
if cfg.MongoUsername != "" {
|
||||||
|
opts.SetAuth(options.Credential{Username: cfg.MongoUsername, Password: cfg.MongoPassword})
|
||||||
|
}
|
||||||
|
client, err := mongo.Connect(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("mongo connect: %w", err)
|
||||||
|
}
|
||||||
|
s := &Store{Client: client, DB: client.Database(cfg.MongoDB)}
|
||||||
|
|
||||||
|
const attempts = 6
|
||||||
|
for i := 1; ; i++ {
|
||||||
|
err = s.Ping(ctx)
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if i == attempts || ctx.Err() != nil {
|
||||||
|
return nil, fmt.Errorf("mongo ping (%d attempts): %w", i, err)
|
||||||
|
}
|
||||||
|
log.Warn("mongo not ready, retrying", "attempt", i, "err", err)
|
||||||
|
select {
|
||||||
|
case <-time.After(time.Duration(i) * time.Second):
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := EnsureIndexes(ctx, s.DB); err != nil {
|
||||||
|
return nil, fmt.Errorf("ensure indexes: %w", err)
|
||||||
|
}
|
||||||
|
log.Info("mongo connected", "db", cfg.MongoDB)
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Ping(ctx context.Context) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
return s.Client.Ping(ctx, readpref.Primary())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) Close(ctx context.Context) error {
|
||||||
|
return s.Client.Disconnect(ctx)
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
|
||||||
|
"bountyboard/internal/testutil"
|
||||||
|
"bountyboard/internal/ulid"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEnsureIndexesIsIdempotent(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := EnsureIndexes(ctx, db); err != nil {
|
||||||
|
t.Fatalf("first run: %v", err)
|
||||||
|
}
|
||||||
|
if err := EnsureIndexes(ctx, db); err != nil {
|
||||||
|
t.Fatalf("second run must be a no-op: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spot-check a few critical indexes actually exist.
|
||||||
|
checks := map[string]string{
|
||||||
|
"users": "email_1",
|
||||||
|
"tasks": "external.system_1_external.key_1_customerId_1",
|
||||||
|
"sessions": "expiresAt_1",
|
||||||
|
}
|
||||||
|
for coll, wantName := range checks {
|
||||||
|
cur, err := db.Collection(coll).Indexes().List(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var specs []bson.M
|
||||||
|
if err := cur.All(ctx, &specs); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, s := range specs {
|
||||||
|
if s["name"] == wantName {
|
||||||
|
found = true
|
||||||
|
// the unique upsert key must really be unique+sparse
|
||||||
|
if coll == "tasks" {
|
||||||
|
if s["unique"] != true || s["sparse"] != true {
|
||||||
|
t.Errorf("%s/%s: want unique sparse, got %v", coll, wantName, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("%s: index %s not found in %v", coll, wantName, specs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateVersioned(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
coll := db.Collection("tasks")
|
||||||
|
|
||||||
|
id := ulid.New()
|
||||||
|
if err := InsertOne(ctx, coll, id, bson.M{"title": "original", "status": "imported"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Happy path bumps version and updatedAt.
|
||||||
|
if err := UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"title": "edited"}}); err != nil {
|
||||||
|
t.Fatalf("update at correct version: %v", err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Title string `bson:"title"`
|
||||||
|
Version int64 `bson:"version"`
|
||||||
|
UpdatedAt time.Time `bson:"updatedAt"`
|
||||||
|
CreatedAt time.Time `bson:"createdAt"`
|
||||||
|
}
|
||||||
|
if err := coll.FindOne(ctx, bson.M{"_id": id}).Decode(&doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if doc.Title != "edited" || doc.Version != 2 {
|
||||||
|
t.Fatalf("after update: title=%q version=%d", doc.Title, doc.Version)
|
||||||
|
}
|
||||||
|
if !doc.UpdatedAt.After(doc.CreatedAt) {
|
||||||
|
t.Error("updatedAt must advance")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stale version loses.
|
||||||
|
err := UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"title": "stale write"}})
|
||||||
|
if !errors.Is(err, ErrVersionConflict) {
|
||||||
|
t.Fatalf("want ErrVersionConflict, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Missing document is distinguished from a conflict.
|
||||||
|
err = UpdateVersioned(ctx, coll, ulid.New(), 1, bson.M{"$set": bson.M{"title": "x"}})
|
||||||
|
if !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("want ErrNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-$set operators pass through; version still bumps.
|
||||||
|
if err := UpdateVersioned(ctx, coll, id, 2, bson.M{
|
||||||
|
"$push": bson.M{"timeline": bson.M{"event": "published"}},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("push update: %v", err)
|
||||||
|
}
|
||||||
|
var doc2 struct {
|
||||||
|
Version int64 `bson:"version"`
|
||||||
|
Timeline []bson.M `bson:"timeline"`
|
||||||
|
}
|
||||||
|
if err := coll.FindOne(ctx, bson.M{"_id": id}).Decode(&doc2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if doc2.Version != 3 || len(doc2.Timeline) != 1 {
|
||||||
|
t.Fatalf("after push: version=%d timeline=%d", doc2.Version, len(doc2.Timeline))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Touching version inside the update is a programming error.
|
||||||
|
if err := UpdateVersioned(ctx, coll, id, 3, bson.M{"$inc": bson.M{"version": 5}}); err == nil {
|
||||||
|
t.Fatal("must reject updates that touch version")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateVersionedConcurrentWriters(t *testing.T) {
|
||||||
|
db := testutil.MongoDB(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
coll := db.Collection("tasks")
|
||||||
|
|
||||||
|
id := ulid.New()
|
||||||
|
if err := InsertOne(ctx, coll, id, bson.M{"n": 0}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two writers race at the same version: exactly one wins.
|
||||||
|
results := make(chan error, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go func(i int) {
|
||||||
|
results <- UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"n": i + 1}})
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
var conflicts, oks int
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
switch err := <-results; {
|
||||||
|
case err == nil:
|
||||||
|
oks++
|
||||||
|
case errors.Is(err, ErrVersionConflict):
|
||||||
|
conflicts++
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if oks != 1 || conflicts != 1 {
|
||||||
|
t.Fatalf("oks=%d conflicts=%d, want exactly one of each", oks, conflicts)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
// Package testutil provides shared helpers for integration tests, which run
|
||||||
|
// against the compose Mongo (published on 127.0.0.1 by
|
||||||
|
// docker-compose.test.yml) using a per-run database name that is dropped on
|
||||||
|
// cleanup.
|
||||||
|
package testutil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
|
||||||
|
|
||||||
|
"bountyboard/internal/ulid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MongoDB connects to the test Mongo (MONGO_TEST_URI, default localhost) and
|
||||||
|
// returns a database with a unique per-run name, dropped automatically.
|
||||||
|
func MongoDB(t *testing.T) *mongo.Database {
|
||||||
|
t.Helper()
|
||||||
|
uri := os.Getenv("MONGO_TEST_URI")
|
||||||
|
if uri == "" {
|
||||||
|
uri = "mongodb://127.0.0.1:27017"
|
||||||
|
}
|
||||||
|
client, err := mongo.Connect(options.Client().
|
||||||
|
ApplyURI(uri).
|
||||||
|
SetServerSelectionTimeout(5 * time.Second))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mongo connect: %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := client.Ping(ctx, readpref.Primary()); err != nil {
|
||||||
|
t.Fatalf("mongo at %s not reachable (start it with: docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo): %v", uri, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
db := client.Database("bountyboard_test_" + strings.ToLower(ulid.New()))
|
||||||
|
t.Cleanup(func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := db.Drop(ctx); err != nil {
|
||||||
|
t.Errorf("drop test db: %v", err)
|
||||||
|
}
|
||||||
|
_ = client.Disconnect(ctx)
|
||||||
|
})
|
||||||
|
return db
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user