feat: store layer with indexes, optimistic concurrency, GridFS, credential crypto (phase 2)

- mongo-driver v2 connection with bounded startup retry + /readyz gate
- idempotent creation of all §4.9 indexes
- UpdateVersioned: version-filtered updates, conflict vs not-found errors
- AES-256-GCM Seal/Open for ticketing credentials, base64(nonce|ct)
- GridFS file store: MIME sniffing, MAX_UPLOAD_MB cap, sha256 metadata
- HMAC-signed short-lived file URL tokens (§5.1, 1h TTL)
- integration tests against compose Mongo via docker-compose.test.yml overlay

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
etalon
2026-06-12 18:17:26 +02:00
parent 976f5238f8
commit f2c534f636
18 changed files with 1129 additions and 3 deletions
+21
View File
@@ -36,3 +36,24 @@ Spec-silent choices, recorded as required by the build instructions.
is not an error (containers receive env via compose `env_file`). is not an error (containers receive env via compose `env_file`).
- **Makefile `seed`/`backup`/`restore` are failing stubs until Phase 12** so - **Makefile `seed`/`backup`/`restore` are failing stubs until Phase 12** so
the targets exist but cannot be mistaken for working. the targets exist but cannot be mistaken for working.
## Phase 2
- **Integration tests reach Mongo via `docker-compose.test.yml`**, an explicit
test-only overlay publishing Mongo on `127.0.0.1:27017`. The normal compose
file still never publishes Mongo (§12); the spec's integration-test
requirement (§2.1) needs host access, and a loopback-only opt-in overlay is
the smallest hole. Each run uses a `bountyboard_test_<ulid>` database and
drops it on cleanup.
- **`UpdateVersioned` rejects updates that touch `version`** and merges
`updatedAt`/`$inc version` into the caller's operators; it distinguishes
`ErrNotFound` from `ErrVersionConflict` with a follow-up existence check.
- **File MIME type = sniffed (`http.DetectContentType`) unless inconclusive**
(`application/octet-stream`), in which case the client-declared type is
kept. A confident sniff overrides a lying declaration.
- **sha256 of uploads is recorded post-upload** via an update on `fs.files`
metadata (GridFS metadata must be supplied before streaming; the hash is
only known after).
- **Signed file tokens** are `base64url(exp || hmac-sha256(fileID|exp))` with
a key derived as `sha256("bountyboard/file-url/v1" + SESSION_SECRET)` so
file tokens can never collide with other uses of the session secret.
+2
View File
@@ -11,6 +11,8 @@ run:
test: test:
$(GO) test ./... $(GO) test ./...
# Requires Mongo reachable on 127.0.0.1:27017 (or MONGO_TEST_URI):
# docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo
test-integration: test-integration:
$(GO) test -tags=integration ./... $(GO) test -tags=integration ./...
+1
View File
@@ -1,4 +1,5 @@
# Progress # Progress
- Phase 1 (scaffold): repo layout, Makefile, .env loader + validated config, slog JSON, ULID pkg, trusted-proxy middleware, /healthz /readyz /metricsz, Dockerfile + compose (mongo+app, healthchecks, localhost-bound 8787), graceful shutdown — build/vet/tests green, smoke PASS. - Phase 1 (scaffold): repo layout, Makefile, .env loader + validated config, slog JSON, ULID pkg, trusted-proxy middleware, /healthz /readyz /metricsz, Dockerfile + compose (mongo+app, healthchecks, localhost-bound 8787), graceful shutdown — build/vet/tests green, smoke PASS.
- Phase 2 (store layer): mongo-driver v2 connection with startup retry, §4.9 indexes idempotent, optimistic-concurrency UpdateVersioned (ErrVersionConflict/ErrNotFound), AES-256-GCM credential crypto, HMAC-signed file URL tokens (1h TTL), GridFS store with MIME sniffing + size cap + sha256, /readyz mongo check (verified 503 on stopped Mongo) — unit + integration tests green.
+23 -3
View File
@@ -8,10 +8,12 @@ import (
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time"
"bountyboard/internal/config" "bountyboard/internal/config"
httpx "bountyboard/internal/http" httpx "bountyboard/internal/http"
"bountyboard/internal/metrics" "bountyboard/internal/metrics"
"bountyboard/internal/store"
) )
func main() { func main() {
@@ -33,10 +35,28 @@ func run() error {
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
slog.SetDefault(log) slog.SetDefault(log)
reg := metrics.NewRegistry()
srv := httpx.New(cfg, log, reg)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop() defer stop()
st, err := store.Connect(ctx, cfg, log)
if err != nil {
return err
}
defer func() {
closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := st.Close(closeCtx); err != nil {
log.Error("close mongo", "err", err)
}
}()
reg := metrics.NewRegistry()
srv := httpx.New(cfg, log, reg)
srv.AddReadinessCheck(httpx.ReadinessCheck{
Name: "mongo",
Required: true,
Probe: st.Ping,
})
return srv.Run(ctx) return srv.Run(ctx)
} }
+8
View File
@@ -0,0 +1,8 @@
# Test-only overlay: publishes Mongo on loopback so host-run integration
# tests can reach it. Never used in normal deployment (§12 keeps Mongo
# unpublished):
# docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo
services:
mongo:
ports:
- "127.0.0.1:27017:27017"
+13
View File
@@ -1,3 +1,16 @@
module bountyboard module bountyboard
go 1.26 go 1.26
require go.mongodb.org/mongo-driver/v2 v2.6.0
require (
github.com/klauspost/compress v1.17.6 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.2.0 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
golang.org/x/crypto v0.33.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/text v0.22.0 // indirect
)
+46
View File
@@ -0,0 +1,46 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8=
go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+64
View File
@@ -0,0 +1,64 @@
// Package crypto provides AES-256-GCM sealing for customer ticketing
// credentials stored in Mongo (§4.2): ciphertext format base64(nonce|sealed).
package crypto
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
)
const keyLen = 32
// Seal encrypts plaintext with AES-256-GCM and returns
// base64(nonce|ciphertext) for storage.
func Seal(key, plaintext []byte) (string, error) {
gcm, err := newGCM(key)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", fmt.Errorf("nonce: %w", err)
}
sealed := gcm.Seal(nonce, nonce, plaintext, nil)
return base64.StdEncoding.EncodeToString(sealed), nil
}
// Open decrypts a value produced by Seal.
func Open(key []byte, encoded string) ([]byte, error) {
gcm, err := newGCM(key)
if err != nil {
return nil, err
}
raw, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, fmt.Errorf("decode ciphertext: %w", err)
}
if len(raw) < gcm.NonceSize() {
return nil, errors.New("ciphertext shorter than nonce")
}
plaintext, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], nil)
if err != nil {
return nil, fmt.Errorf("decrypt: %w", err)
}
return plaintext, nil
}
func newGCM(key []byte) (cipher.AEAD, error) {
if len(key) != keyLen {
return nil, fmt.Errorf("key must be %d bytes, got %d", keyLen, len(key))
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("aes: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, fmt.Errorf("gcm: %w", err)
}
return gcm, nil
}
+86
View File
@@ -0,0 +1,86 @@
package crypto
import (
"bytes"
"crypto/rand"
"strings"
"testing"
)
func testKey(t *testing.T) []byte {
t.Helper()
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
return key
}
func TestSealOpenRoundTrip(t *testing.T) {
key := testKey(t)
tests := [][]byte{
[]byte(`{"email":"a@b.c","apiToken":"secret-token"}`),
[]byte(""),
[]byte(strings.Repeat("x", 100000)),
{0x00, 0xFF, 0x10},
}
for _, plaintext := range tests {
sealed, err := Seal(key, plaintext)
if err != nil {
t.Fatalf("Seal: %v", err)
}
got, err := Open(key, sealed)
if err != nil {
t.Fatalf("Open: %v", err)
}
if !bytes.Equal(got, plaintext) {
t.Errorf("round trip mismatch for %d-byte input", len(plaintext))
}
}
}
func TestSealIsNonDeterministic(t *testing.T) {
key := testKey(t)
a, _ := Seal(key, []byte("same"))
b, _ := Seal(key, []byte("same"))
if a == b {
t.Fatal("two seals of the same plaintext must differ (random nonce)")
}
}
func TestOpenRejectsTampering(t *testing.T) {
key := testKey(t)
sealed, err := Seal(key, []byte("credentials"))
if err != nil {
t.Fatal(err)
}
if _, err := Open(testKey(t), sealed); err == nil {
t.Error("wrong key must fail")
}
if _, err := Open(key, "!!not-base64!!"); err == nil {
t.Error("invalid base64 must fail")
}
if _, err := Open(key, "AAAA"); err == nil {
t.Error("too-short ciphertext must fail")
}
corrupted := []byte(sealed)
mid := len(corrupted) / 2
if corrupted[mid] == 'A' {
corrupted[mid] = 'B'
} else {
corrupted[mid] = 'A'
}
if _, err := Open(key, string(corrupted)); err == nil {
t.Error("corrupted ciphertext must fail authentication")
}
}
func TestKeyLengthEnforced(t *testing.T) {
if _, err := Seal(make([]byte, 16), []byte("x")); err == nil {
t.Error("16-byte key must be rejected")
}
if _, err := Open(make([]byte, 31), "AAAA"); err == nil {
t.Error("31-byte key must be rejected")
}
}
+61
View File
@@ -0,0 +1,61 @@
// Package files implements upload storage (GridFS) and the short-lived
// HMAC-signed file URLs from §5.1: /files/{id}?st=<token> lets external
// services (atomizer, work performer) fetch attachments without a session.
package files
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"fmt"
"time"
)
// DefaultTokenTTL is the §5.1 signed-URL lifetime.
const DefaultTokenTTL = time.Hour
// signingKey derives a dedicated key from the session secret so file tokens
// can never be confused with anything else signed by the same secret.
func signingKey(secret []byte) []byte {
h := sha256.New()
h.Write([]byte("bountyboard/file-url/v1"))
h.Write(secret)
return h.Sum(nil)
}
func mac(key []byte, fileID string, exp int64) []byte {
m := hmac.New(sha256.New, key)
fmt.Fprintf(m, "%s|%d", fileID, exp)
return m.Sum(nil)
}
// SignToken returns the st token granting access to fileID until expiry.
// Format: base64url(bigendian-int64-exp || hmac-sha256(fileID|exp)).
func SignToken(secret []byte, fileID string, expiresAt time.Time) string {
exp := expiresAt.Unix()
key := signingKey(secret)
buf := make([]byte, 8, 8+sha256.Size)
binary.BigEndian.PutUint64(buf, uint64(exp))
buf = append(buf, mac(key, fileID, exp)...)
return base64.RawURLEncoding.EncodeToString(buf)
}
// VerifyToken reports whether token grants access to fileID at time now.
func VerifyToken(secret []byte, fileID, token string, now time.Time) bool {
raw, err := base64.RawURLEncoding.DecodeString(token)
if err != nil || len(raw) != 8+sha256.Size {
return false
}
exp := int64(binary.BigEndian.Uint64(raw[:8]))
if now.Unix() > exp {
return false
}
expected := mac(signingKey(secret), fileID, exp)
return hmac.Equal(raw[8:], expected)
}
// SignedURL builds the absolute fetch URL for a stored file.
func SignedURL(appBaseURL string, secret []byte, fileID string, expiresAt time.Time) string {
return fmt.Sprintf("%s/files/%s?st=%s", appBaseURL, fileID, SignToken(secret, fileID, expiresAt))
}
+62
View File
@@ -0,0 +1,62 @@
package files
import (
"strings"
"testing"
"time"
)
func TestSignVerifyRoundTrip(t *testing.T) {
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Now()
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
if !VerifyToken(secret, "01JFILE", token, now) {
t.Fatal("fresh token must verify")
}
if !VerifyToken(secret, "01JFILE", token, now.Add(59*time.Minute)) {
t.Fatal("token must verify just before expiry")
}
}
func TestVerifyRejections(t *testing.T) {
secret := []byte("0123456789abcdef0123456789abcdef")
now := time.Now()
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
tests := []struct {
name string
fileID string
token string
at time.Time
}{
{"expired", "01JFILE", token, now.Add(2 * time.Hour)},
{"wrong file", "01JOTHER", token, now},
{"garbage token", "01JFILE", "not-a-token", now},
{"empty token", "01JFILE", "", now},
{"truncated token", "01JFILE", token[:len(token)-10], now},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if VerifyToken(secret, tt.fileID, tt.token, tt.at) {
t.Error("must not verify")
}
})
}
if VerifyToken([]byte("another-secret-another-secret-xx"), "01JFILE", token, now) {
t.Error("different secret must not verify")
}
}
func TestSignedURLShape(t *testing.T) {
secret := []byte("0123456789abcdef0123456789abcdef")
u := SignedURL("http://localhost:8787", secret, "01JFILE", time.Now().Add(time.Hour))
if !strings.HasPrefix(u, "http://localhost:8787/files/01JFILE?st=") {
t.Fatalf("unexpected URL %q", u)
}
token := u[strings.Index(u, "?st=")+4:]
if !VerifyToken(secret, "01JFILE", token, time.Now()) {
t.Fatal("token embedded in URL must verify")
}
}
+167
View File
@@ -0,0 +1,167 @@
package files
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"time"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
"bountyboard/internal/ulid"
)
// ErrTooLarge is returned when an upload exceeds MAX_UPLOAD_MB.
var ErrTooLarge = errors.New("file exceeds maximum upload size")
// ErrNotFound is returned when no stored file has the given id.
var ErrNotFound = errors.New("file not found")
// Valid metadata scopes (§4.7); access control at the HTTP layer keys off
// these.
const (
ScopeAvatar = "avatar"
ScopeChat = "chat"
ScopeTask = "task"
)
type Meta struct {
ID string
Name string
OwnerID string
Scope string
MimeType string
Size int64
SHA256 string
}
// Store wraps the GridFS bucket with MIME sniffing, a size cap, and sha256
// recording.
type Store struct {
bucket *mongo.GridFSBucket
maxBytes int64
}
func NewStore(db *mongo.Database, maxUploadMB int) *Store {
return &Store{
bucket: db.GridFSBucket(),
maxBytes: int64(maxUploadMB) * 1024 * 1024,
}
}
// Save streams r into GridFS. The content type is sniffed from the first
// 512 bytes; declaredMime is used only when sniffing is inconclusive.
func (s *Store) Save(ctx context.Context, scope, ownerID, name, declaredMime string, r io.Reader) (*Meta, error) {
head := make([]byte, 512)
n, err := io.ReadFull(r, head)
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
return nil, fmt.Errorf("read upload head: %w", err)
}
head = head[:n]
mimeType := http.DetectContentType(head)
if mimeType == "application/octet-stream" && declaredMime != "" {
mimeType = declaredMime
}
id := ulid.New()
hasher := sha256.New()
// Allow one byte over the cap so an oversize upload is detectable.
body := &countingReader{r: io.TeeReader(
io.LimitReader(io.MultiReader(bytes.NewReader(head), r), s.maxBytes+1),
hasher)}
err = s.bucket.UploadFromStreamWithID(ctx, id, name, body,
options.GridFSUpload().SetMetadata(bson.M{
"ownerId": ownerID,
"scope": scope,
"mimeType": mimeType,
}))
if err != nil {
return nil, fmt.Errorf("gridfs upload: %w", err)
}
size := body.n
if size > s.maxBytes {
if delErr := s.bucket.Delete(ctx, id); delErr != nil {
return nil, fmt.Errorf("delete oversize upload: %w", delErr)
}
return nil, fmt.Errorf("%w (limit %d bytes)", ErrTooLarge, s.maxBytes)
}
sum := hex.EncodeToString(hasher.Sum(nil))
_, err = s.bucket.GetFilesCollection().UpdateOne(ctx,
bson.M{"_id": id},
bson.M{"$set": bson.M{"metadata.sha256": sum}})
if err != nil {
return nil, fmt.Errorf("record sha256: %w", err)
}
return &Meta{
ID: id, Name: name, OwnerID: ownerID, Scope: scope,
MimeType: mimeType, Size: size, SHA256: sum,
}, nil
}
// Open returns a reader over the stored file plus its metadata. The caller
// must Close the stream.
func (s *Store) Open(ctx context.Context, id string) (io.ReadCloser, *Meta, error) {
ds, err := s.bucket.OpenDownloadStream(ctx, id)
if err != nil {
if errors.Is(err, mongo.ErrFileNotFound) {
return nil, nil, fmt.Errorf("%s: %w", id, ErrNotFound)
}
return nil, nil, fmt.Errorf("gridfs open %s: %w", id, err)
}
f := ds.GetFile()
var md struct {
OwnerID string `bson:"ownerId"`
Scope string `bson:"scope"`
MimeType string `bson:"mimeType"`
SHA256 string `bson:"sha256"`
}
if f.Metadata != nil {
if err := bson.Unmarshal(f.Metadata, &md); err != nil {
ds.Close()
return nil, nil, fmt.Errorf("decode file metadata %s: %w", id, err)
}
}
return ds, &Meta{
ID: id, Name: f.Name, OwnerID: md.OwnerID, Scope: md.Scope,
MimeType: md.MimeType, Size: f.Length, SHA256: md.SHA256,
}, nil
}
// Delete removes the file and its chunks.
func (s *Store) Delete(ctx context.Context, id string) error {
if err := s.bucket.Delete(ctx, id); err != nil {
if errors.Is(err, mongo.ErrFileNotFound) {
return fmt.Errorf("%s: %w", id, ErrNotFound)
}
return fmt.Errorf("gridfs delete %s: %w", id, err)
}
return nil
}
// countingReader tracks how many bytes the upload actually consumed, since
// the driver's upload helper does not report a size.
type countingReader struct {
r io.Reader
n int64
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += int64(n)
return n, err
}
// SignedURLFor is a convenience for handing attachments to external services.
func (s *Store) SignedURLFor(appBaseURL string, secret []byte, fileID string) string {
return SignedURL(appBaseURL, secret, fileID, time.Now().Add(DefaultTokenTTL))
}
+145
View File
@@ -0,0 +1,145 @@
//go:build integration
package files
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"strings"
"testing"
"bountyboard/internal/testutil"
)
func TestSaveOpenRoundTrip(t *testing.T) {
db := testutil.MongoDB(t)
s := NewStore(db, 25)
ctx := context.Background()
content := []byte("%PDF-1.4 fake pdf content for sniffing")
meta, err := s.Save(ctx, ScopeTask, "01JOWNER", "spec.pdf", "", bytes.NewReader(content))
if err != nil {
t.Fatalf("Save: %v", err)
}
if meta.MimeType != "application/pdf" {
t.Errorf("sniffed mime = %q, want application/pdf", meta.MimeType)
}
if meta.Size != int64(len(content)) {
t.Errorf("size = %d, want %d", meta.Size, len(content))
}
wantSum := sha256.Sum256(content)
if meta.SHA256 != hex.EncodeToString(wantSum[:]) {
t.Errorf("sha256 mismatch")
}
rc, got, err := s.Open(ctx, meta.ID)
if err != nil {
t.Fatalf("Open: %v", err)
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, content) {
t.Error("downloaded content differs")
}
if got.OwnerID != "01JOWNER" || got.Scope != ScopeTask || got.Name != "spec.pdf" {
t.Errorf("metadata round trip: %+v", got)
}
if got.SHA256 != meta.SHA256 || got.MimeType != meta.MimeType {
t.Errorf("stored metadata mismatch: %+v vs %+v", got, meta)
}
}
func TestDeclaredMimeUsedWhenSniffInconclusive(t *testing.T) {
db := testutil.MongoDB(t)
s := NewStore(db, 25)
meta, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "data.bin",
"application/x-custom", bytes.NewReader([]byte{0x01, 0x02, 0x03}))
if err != nil {
t.Fatal(err)
}
if meta.MimeType != "application/x-custom" {
t.Errorf("mime = %q, want declared application/x-custom", meta.MimeType)
}
// But a confident sniff overrides a lying declaration.
meta2, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "fake.txt",
"text/plain", strings.NewReader("\x89PNG\r\n\x1a\n123456789"))
if err != nil {
t.Fatal(err)
}
if meta2.MimeType != "image/png" {
t.Errorf("mime = %q, want sniffed image/png", meta2.MimeType)
}
}
func TestSizeLimitEnforced(t *testing.T) {
db := testutil.MongoDB(t)
s := NewStore(db, 1) // 1 MiB cap
big := io.LimitReader(neverEnding('x'), 1<<20+1)
_, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "big.bin", "", big)
if !errors.Is(err, ErrTooLarge) {
t.Fatalf("want ErrTooLarge, got %v", err)
}
// No orphaned file documents may remain.
n, err := s.bucket.GetFilesCollection().CountDocuments(context.Background(), struct{}{})
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Errorf("oversize upload left %d file docs behind", n)
}
// Exactly at the limit is fine.
ok := io.LimitReader(neverEnding('x'), 1<<20)
if _, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "exact.bin", "", ok); err != nil {
t.Fatalf("upload at exactly the limit: %v", err)
}
}
func TestOpenMissingFile(t *testing.T) {
db := testutil.MongoDB(t)
s := NewStore(db, 25)
if _, _, err := s.Open(context.Background(), "01JNOPE"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestDelete(t *testing.T) {
db := testutil.MongoDB(t)
s := NewStore(db, 25)
ctx := context.Background()
meta, err := s.Save(ctx, ScopeAvatar, "01JOWNER", "a.txt", "", strings.NewReader("hello"))
if err != nil {
t.Fatal(err)
}
if err := s.Delete(ctx, meta.ID); err != nil {
t.Fatal(err)
}
if _, _, err := s.Open(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound after delete, got %v", err)
}
if err := s.Delete(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("double delete: want ErrNotFound, got %v", err)
}
}
// neverEnding is an infinite reader of one repeated byte.
type neverEnding byte
func (b neverEnding) Read(p []byte) (int, error) {
for i := range p {
p[i] = byte(b)
}
return len(p), nil
}
+72
View File
@@ -0,0 +1,72 @@
package store
import (
"context"
"fmt"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
)
// EnsureIndexes creates every index from spec §4.9. CreateMany is idempotent:
// existing identical indexes are no-ops, so this runs safely at every startup.
func EnsureIndexes(ctx context.Context, db *mongo.Database) error {
plan := map[string][]mongo.IndexModel{
"users": {
{Keys: bson.D{{Key: "email", Value: 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{Key: "auth.oidc.issuer", Value: 1}, {Key: "auth.oidc.subject", Value: 1}},
Options: options.Index().SetUnique(true).SetSparse(true)},
},
"customers": {
{Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{Key: "consultantIds", Value: 1}}},
},
"pools": {
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "developerId", Value: 1}},
Options: options.Index().SetUnique(true)},
{Keys: bson.D{{Key: "developerId", Value: 1}}},
},
"tasks": {
{Keys: bson.D{{Key: "customerId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}},
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}},
{Keys: bson.D{{Key: "assignee.userId", Value: 1}, {Key: "status", Value: 1}}},
{Keys: bson.D{{Key: "rootId", Value: 1}}},
{Keys: bson.D{{Key: "parentId", Value: 1}}},
{Keys: bson.D{{Key: "external.system", Value: 1}, {Key: "external.key", Value: 1}, {Key: "customerId", Value: 1}},
Options: options.Index().SetUnique(true).SetSparse(true)},
{Keys: bson.D{{Key: "title", Value: "text"}, {Key: "description", Value: "text"}}},
},
"bountyAwards": {
{Keys: bson.D{{Key: "developerId", Value: 1}, {Key: "awardedAt", Value: -1}}},
{Keys: bson.D{{Key: "consultantId", Value: 1}, {Key: "awardedAt", Value: -1}}},
{Keys: bson.D{{Key: "customerId", Value: 1}, {Key: "awardedAt", Value: -1}}},
},
"conversations": {
{Keys: bson.D{{Key: "participantIds", Value: 1}, {Key: "lastMessageAt", Value: -1}}},
{Keys: bson.D{{Key: "kind", Value: 1}, {Key: "customerId", Value: 1}}},
},
"messages": {
// ULID _id is time-ordered, so this serves history pagination.
{Keys: bson.D{{Key: "conversationId", Value: 1}, {Key: "_id", Value: 1}}},
},
"notifications": {
{Keys: bson.D{{Key: "userId", Value: 1}, {Key: "readAt", Value: 1}, {Key: "createdAt", Value: -1}}},
},
"sessions": {
{Keys: bson.D{{Key: "expiresAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)},
{Keys: bson.D{{Key: "userId", Value: 1}}},
},
"auditLog": {
{Keys: bson.D{{Key: "at", Value: -1}}},
{Keys: bson.D{{Key: "entityId", Value: 1}, {Key: "at", Value: -1}}},
},
}
for coll, models := range plan {
if _, err := db.Collection(coll).Indexes().CreateMany(ctx, models); err != nil {
return fmt.Errorf("create indexes for %s: %w", coll, err)
}
}
return nil
}
+75
View File
@@ -0,0 +1,75 @@
package store
import (
"context"
"errors"
"fmt"
"time"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
)
// Optimistic-concurrency errors. HTTP handlers translate ErrVersionConflict
// into a 409 with a "conflict" envelope (and the QoL reload toast).
var (
ErrNotFound = errors.New("document not found")
ErrVersionConflict = errors.New("version conflict")
)
// UpdateVersioned applies update to the document only if its version still
// matches, bumping version and updatedAt atomically. update uses normal
// operators ($set/$push/...); the helper merges its bookkeeping into them.
func UpdateVersioned(ctx context.Context, coll *mongo.Collection, id string, version int64, update bson.M) error {
merged := bson.M{}
for op, fields := range update {
merged[op] = fields
}
set, _ := merged["$set"].(bson.M)
if set == nil {
set = bson.M{}
}
set["updatedAt"] = time.Now().UTC()
merged["$set"] = set
inc, _ := merged["$inc"].(bson.M)
if inc == nil {
inc = bson.M{}
}
if _, clash := inc["version"]; clash {
return fmt.Errorf("update must not touch version itself")
}
inc["version"] = int64(1)
merged["$inc"] = inc
res, err := coll.UpdateOne(ctx, bson.M{"_id": id, "version": version}, merged)
if err != nil {
return fmt.Errorf("versioned update %s/%s: %w", coll.Name(), id, err)
}
if res.MatchedCount == 1 {
return nil
}
// Distinguish "gone" from "someone got there first".
n, err := coll.CountDocuments(ctx, bson.M{"_id": id})
if err != nil {
return fmt.Errorf("versioned update %s/%s: existence check: %w", coll.Name(), id, err)
}
if n == 0 {
return fmt.Errorf("%s/%s: %w", coll.Name(), id, ErrNotFound)
}
return fmt.Errorf("%s/%s at version %d: %w", coll.Name(), id, version, ErrVersionConflict)
}
// InsertOne inserts a document map, stamping createdAt/updatedAt/version so
// every collection follows the §4 conventions.
func InsertOne(ctx context.Context, coll *mongo.Collection, id string, doc bson.M) error {
now := time.Now().UTC()
doc["_id"] = id
doc["createdAt"] = now
doc["updatedAt"] = now
doc["version"] = int64(1)
if _, err := coll.InsertOne(ctx, doc); err != nil {
return fmt.Errorf("insert %s/%s: %w", coll.Name(), id, err)
}
return nil
}
+73
View File
@@ -0,0 +1,73 @@
// Package store owns all MongoDB access: connection lifecycle, index
// bootstrap, optimistic-concurrency updates, and (via internal/files) GridFS.
// No other package talks to Mongo directly.
package store
import (
"context"
"fmt"
"log/slog"
"time"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
"bountyboard/internal/config"
)
type Store struct {
Client *mongo.Client
DB *mongo.Database
}
// Connect builds the client (the driver dials lazily) and verifies
// connectivity with a bounded retry so a compose race at boot does not kill
// the app, then ensures all indexes.
func Connect(ctx context.Context, cfg *config.Config, log *slog.Logger) (*Store, error) {
opts := options.Client().
ApplyURI(cfg.MongoURI).
SetAppName("bountyboard").
SetServerSelectionTimeout(5 * time.Second)
if cfg.MongoUsername != "" {
opts.SetAuth(options.Credential{Username: cfg.MongoUsername, Password: cfg.MongoPassword})
}
client, err := mongo.Connect(opts)
if err != nil {
return nil, fmt.Errorf("mongo connect: %w", err)
}
s := &Store{Client: client, DB: client.Database(cfg.MongoDB)}
const attempts = 6
for i := 1; ; i++ {
err = s.Ping(ctx)
if err == nil {
break
}
if i == attempts || ctx.Err() != nil {
return nil, fmt.Errorf("mongo ping (%d attempts): %w", i, err)
}
log.Warn("mongo not ready, retrying", "attempt", i, "err", err)
select {
case <-time.After(time.Duration(i) * time.Second):
case <-ctx.Done():
return nil, ctx.Err()
}
}
if err := EnsureIndexes(ctx, s.DB); err != nil {
return nil, fmt.Errorf("ensure indexes: %w", err)
}
log.Info("mongo connected", "db", cfg.MongoDB)
return s, nil
}
func (s *Store) Ping(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
return s.Client.Ping(ctx, readpref.Primary())
}
func (s *Store) Close(ctx context.Context) error {
return s.Client.Disconnect(ctx)
}
+157
View File
@@ -0,0 +1,157 @@
//go:build integration
package store
import (
"context"
"errors"
"testing"
"time"
"go.mongodb.org/mongo-driver/v2/bson"
"bountyboard/internal/testutil"
"bountyboard/internal/ulid"
)
func TestEnsureIndexesIsIdempotent(t *testing.T) {
db := testutil.MongoDB(t)
ctx := context.Background()
if err := EnsureIndexes(ctx, db); err != nil {
t.Fatalf("first run: %v", err)
}
if err := EnsureIndexes(ctx, db); err != nil {
t.Fatalf("second run must be a no-op: %v", err)
}
// Spot-check a few critical indexes actually exist.
checks := map[string]string{
"users": "email_1",
"tasks": "external.system_1_external.key_1_customerId_1",
"sessions": "expiresAt_1",
}
for coll, wantName := range checks {
cur, err := db.Collection(coll).Indexes().List(ctx)
if err != nil {
t.Fatal(err)
}
var specs []bson.M
if err := cur.All(ctx, &specs); err != nil {
t.Fatal(err)
}
found := false
for _, s := range specs {
if s["name"] == wantName {
found = true
// the unique upsert key must really be unique+sparse
if coll == "tasks" {
if s["unique"] != true || s["sparse"] != true {
t.Errorf("%s/%s: want unique sparse, got %v", coll, wantName, s)
}
}
}
}
if !found {
t.Errorf("%s: index %s not found in %v", coll, wantName, specs)
}
}
}
func TestUpdateVersioned(t *testing.T) {
db := testutil.MongoDB(t)
ctx := context.Background()
coll := db.Collection("tasks")
id := ulid.New()
if err := InsertOne(ctx, coll, id, bson.M{"title": "original", "status": "imported"}); err != nil {
t.Fatal(err)
}
// Happy path bumps version and updatedAt.
if err := UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"title": "edited"}}); err != nil {
t.Fatalf("update at correct version: %v", err)
}
var doc struct {
Title string `bson:"title"`
Version int64 `bson:"version"`
UpdatedAt time.Time `bson:"updatedAt"`
CreatedAt time.Time `bson:"createdAt"`
}
if err := coll.FindOne(ctx, bson.M{"_id": id}).Decode(&doc); err != nil {
t.Fatal(err)
}
if doc.Title != "edited" || doc.Version != 2 {
t.Fatalf("after update: title=%q version=%d", doc.Title, doc.Version)
}
if !doc.UpdatedAt.After(doc.CreatedAt) {
t.Error("updatedAt must advance")
}
// Stale version loses.
err := UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"title": "stale write"}})
if !errors.Is(err, ErrVersionConflict) {
t.Fatalf("want ErrVersionConflict, got %v", err)
}
// Missing document is distinguished from a conflict.
err = UpdateVersioned(ctx, coll, ulid.New(), 1, bson.M{"$set": bson.M{"title": "x"}})
if !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
// Non-$set operators pass through; version still bumps.
if err := UpdateVersioned(ctx, coll, id, 2, bson.M{
"$push": bson.M{"timeline": bson.M{"event": "published"}},
}); err != nil {
t.Fatalf("push update: %v", err)
}
var doc2 struct {
Version int64 `bson:"version"`
Timeline []bson.M `bson:"timeline"`
}
if err := coll.FindOne(ctx, bson.M{"_id": id}).Decode(&doc2); err != nil {
t.Fatal(err)
}
if doc2.Version != 3 || len(doc2.Timeline) != 1 {
t.Fatalf("after push: version=%d timeline=%d", doc2.Version, len(doc2.Timeline))
}
// Touching version inside the update is a programming error.
if err := UpdateVersioned(ctx, coll, id, 3, bson.M{"$inc": bson.M{"version": 5}}); err == nil {
t.Fatal("must reject updates that touch version")
}
}
func TestUpdateVersionedConcurrentWriters(t *testing.T) {
db := testutil.MongoDB(t)
ctx := context.Background()
coll := db.Collection("tasks")
id := ulid.New()
if err := InsertOne(ctx, coll, id, bson.M{"n": 0}); err != nil {
t.Fatal(err)
}
// Two writers race at the same version: exactly one wins.
results := make(chan error, 2)
for i := 0; i < 2; i++ {
go func(i int) {
results <- UpdateVersioned(ctx, coll, id, 1, bson.M{"$set": bson.M{"n": i + 1}})
}(i)
}
var conflicts, oks int
for i := 0; i < 2; i++ {
switch err := <-results; {
case err == nil:
oks++
case errors.Is(err, ErrVersionConflict):
conflicts++
default:
t.Fatalf("unexpected error: %v", err)
}
}
if oks != 1 || conflicts != 1 {
t.Fatalf("oks=%d conflicts=%d, want exactly one of each", oks, conflicts)
}
}
+53
View File
@@ -0,0 +1,53 @@
//go:build integration
// Package testutil provides shared helpers for integration tests, which run
// against the compose Mongo (published on 127.0.0.1 by
// docker-compose.test.yml) using a per-run database name that is dropped on
// cleanup.
package testutil
import (
"context"
"os"
"strings"
"testing"
"time"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
"bountyboard/internal/ulid"
)
// MongoDB connects to the test Mongo (MONGO_TEST_URI, default localhost) and
// returns a database with a unique per-run name, dropped automatically.
func MongoDB(t *testing.T) *mongo.Database {
t.Helper()
uri := os.Getenv("MONGO_TEST_URI")
if uri == "" {
uri = "mongodb://127.0.0.1:27017"
}
client, err := mongo.Connect(options.Client().
ApplyURI(uri).
SetServerSelectionTimeout(5 * time.Second))
if err != nil {
t.Fatalf("mongo connect: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := client.Ping(ctx, readpref.Primary()); err != nil {
t.Fatalf("mongo at %s not reachable (start it with: docker compose -f docker-compose.yml -f docker-compose.test.yml up -d mongo): %v", uri, err)
}
db := client.Database("bountyboard_test_" + strings.ToLower(ulid.New()))
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := db.Drop(ctx); err != nil {
t.Errorf("drop test db: %v", err)
}
_ = client.Disconnect(ctx)
})
return db
}