feat: store layer with indexes, optimistic concurrency, GridFS, credential crypto (phase 2)
- mongo-driver v2 connection with bounded startup retry + /readyz gate - idempotent creation of all §4.9 indexes - UpdateVersioned: version-filtered updates, conflict vs not-found errors - AES-256-GCM Seal/Open for ticketing credentials, base64(nonce|ct) - GridFS file store: MIME sniffing, MAX_UPLOAD_MB cap, sha256 metadata - HMAC-signed short-lived file URL tokens (§5.1, 1h TTL) - integration tests against compose Mongo via docker-compose.test.yml overlay Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
etalon
@@ -0,0 +1,61 @@
|
||||
// Package files implements upload storage (GridFS) and the short-lived
|
||||
// HMAC-signed file URLs from §5.1: /files/{id}?st=<token> lets external
|
||||
// services (atomizer, work performer) fetch attachments without a session.
|
||||
package files
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultTokenTTL is the §5.1 signed-URL lifetime.
|
||||
const DefaultTokenTTL = time.Hour
|
||||
|
||||
// signingKey derives a dedicated key from the session secret so file tokens
|
||||
// can never be confused with anything else signed by the same secret.
|
||||
func signingKey(secret []byte) []byte {
|
||||
h := sha256.New()
|
||||
h.Write([]byte("bountyboard/file-url/v1"))
|
||||
h.Write(secret)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
func mac(key []byte, fileID string, exp int64) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
fmt.Fprintf(m, "%s|%d", fileID, exp)
|
||||
return m.Sum(nil)
|
||||
}
|
||||
|
||||
// SignToken returns the st token granting access to fileID until expiry.
|
||||
// Format: base64url(bigendian-int64-exp || hmac-sha256(fileID|exp)).
|
||||
func SignToken(secret []byte, fileID string, expiresAt time.Time) string {
|
||||
exp := expiresAt.Unix()
|
||||
key := signingKey(secret)
|
||||
buf := make([]byte, 8, 8+sha256.Size)
|
||||
binary.BigEndian.PutUint64(buf, uint64(exp))
|
||||
buf = append(buf, mac(key, fileID, exp)...)
|
||||
return base64.RawURLEncoding.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// VerifyToken reports whether token grants access to fileID at time now.
|
||||
func VerifyToken(secret []byte, fileID, token string, now time.Time) bool {
|
||||
raw, err := base64.RawURLEncoding.DecodeString(token)
|
||||
if err != nil || len(raw) != 8+sha256.Size {
|
||||
return false
|
||||
}
|
||||
exp := int64(binary.BigEndian.Uint64(raw[:8]))
|
||||
if now.Unix() > exp {
|
||||
return false
|
||||
}
|
||||
expected := mac(signingKey(secret), fileID, exp)
|
||||
return hmac.Equal(raw[8:], expected)
|
||||
}
|
||||
|
||||
// SignedURL builds the absolute fetch URL for a stored file.
|
||||
func SignedURL(appBaseURL string, secret []byte, fileID string, expiresAt time.Time) string {
|
||||
return fmt.Sprintf("%s/files/%s?st=%s", appBaseURL, fileID, SignToken(secret, fileID, expiresAt))
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSignVerifyRoundTrip(t *testing.T) {
|
||||
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||
now := time.Now()
|
||||
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
|
||||
|
||||
if !VerifyToken(secret, "01JFILE", token, now) {
|
||||
t.Fatal("fresh token must verify")
|
||||
}
|
||||
if !VerifyToken(secret, "01JFILE", token, now.Add(59*time.Minute)) {
|
||||
t.Fatal("token must verify just before expiry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyRejections(t *testing.T) {
|
||||
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||
now := time.Now()
|
||||
token := SignToken(secret, "01JFILE", now.Add(time.Hour))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fileID string
|
||||
token string
|
||||
at time.Time
|
||||
}{
|
||||
{"expired", "01JFILE", token, now.Add(2 * time.Hour)},
|
||||
{"wrong file", "01JOTHER", token, now},
|
||||
{"garbage token", "01JFILE", "not-a-token", now},
|
||||
{"empty token", "01JFILE", "", now},
|
||||
{"truncated token", "01JFILE", token[:len(token)-10], now},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if VerifyToken(secret, tt.fileID, tt.token, tt.at) {
|
||||
t.Error("must not verify")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if VerifyToken([]byte("another-secret-another-secret-xx"), "01JFILE", token, now) {
|
||||
t.Error("different secret must not verify")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignedURLShape(t *testing.T) {
|
||||
secret := []byte("0123456789abcdef0123456789abcdef")
|
||||
u := SignedURL("http://localhost:8787", secret, "01JFILE", time.Now().Add(time.Hour))
|
||||
if !strings.HasPrefix(u, "http://localhost:8787/files/01JFILE?st=") {
|
||||
t.Fatalf("unexpected URL %q", u)
|
||||
}
|
||||
token := u[strings.Index(u, "?st=")+4:]
|
||||
if !VerifyToken(secret, "01JFILE", token, time.Now()) {
|
||||
t.Fatal("token embedded in URL must verify")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
|
||||
"bountyboard/internal/ulid"
|
||||
)
|
||||
|
||||
// ErrTooLarge is returned when an upload exceeds MAX_UPLOAD_MB.
|
||||
var ErrTooLarge = errors.New("file exceeds maximum upload size")
|
||||
|
||||
// ErrNotFound is returned when no stored file has the given id.
|
||||
var ErrNotFound = errors.New("file not found")
|
||||
|
||||
// Valid metadata scopes (§4.7); access control at the HTTP layer keys off
|
||||
// these.
|
||||
const (
|
||||
ScopeAvatar = "avatar"
|
||||
ScopeChat = "chat"
|
||||
ScopeTask = "task"
|
||||
)
|
||||
|
||||
type Meta struct {
|
||||
ID string
|
||||
Name string
|
||||
OwnerID string
|
||||
Scope string
|
||||
MimeType string
|
||||
Size int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// Store wraps the GridFS bucket with MIME sniffing, a size cap, and sha256
|
||||
// recording.
|
||||
type Store struct {
|
||||
bucket *mongo.GridFSBucket
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func NewStore(db *mongo.Database, maxUploadMB int) *Store {
|
||||
return &Store{
|
||||
bucket: db.GridFSBucket(),
|
||||
maxBytes: int64(maxUploadMB) * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
// Save streams r into GridFS. The content type is sniffed from the first
|
||||
// 512 bytes; declaredMime is used only when sniffing is inconclusive.
|
||||
func (s *Store) Save(ctx context.Context, scope, ownerID, name, declaredMime string, r io.Reader) (*Meta, error) {
|
||||
head := make([]byte, 512)
|
||||
n, err := io.ReadFull(r, head)
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return nil, fmt.Errorf("read upload head: %w", err)
|
||||
}
|
||||
head = head[:n]
|
||||
|
||||
mimeType := http.DetectContentType(head)
|
||||
if mimeType == "application/octet-stream" && declaredMime != "" {
|
||||
mimeType = declaredMime
|
||||
}
|
||||
|
||||
id := ulid.New()
|
||||
hasher := sha256.New()
|
||||
// Allow one byte over the cap so an oversize upload is detectable.
|
||||
body := &countingReader{r: io.TeeReader(
|
||||
io.LimitReader(io.MultiReader(bytes.NewReader(head), r), s.maxBytes+1),
|
||||
hasher)}
|
||||
err = s.bucket.UploadFromStreamWithID(ctx, id, name, body,
|
||||
options.GridFSUpload().SetMetadata(bson.M{
|
||||
"ownerId": ownerID,
|
||||
"scope": scope,
|
||||
"mimeType": mimeType,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gridfs upload: %w", err)
|
||||
}
|
||||
size := body.n
|
||||
if size > s.maxBytes {
|
||||
if delErr := s.bucket.Delete(ctx, id); delErr != nil {
|
||||
return nil, fmt.Errorf("delete oversize upload: %w", delErr)
|
||||
}
|
||||
return nil, fmt.Errorf("%w (limit %d bytes)", ErrTooLarge, s.maxBytes)
|
||||
}
|
||||
|
||||
sum := hex.EncodeToString(hasher.Sum(nil))
|
||||
_, err = s.bucket.GetFilesCollection().UpdateOne(ctx,
|
||||
bson.M{"_id": id},
|
||||
bson.M{"$set": bson.M{"metadata.sha256": sum}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("record sha256: %w", err)
|
||||
}
|
||||
|
||||
return &Meta{
|
||||
ID: id, Name: name, OwnerID: ownerID, Scope: scope,
|
||||
MimeType: mimeType, Size: size, SHA256: sum,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Open returns a reader over the stored file plus its metadata. The caller
|
||||
// must Close the stream.
|
||||
func (s *Store) Open(ctx context.Context, id string) (io.ReadCloser, *Meta, error) {
|
||||
ds, err := s.bucket.OpenDownloadStream(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongo.ErrFileNotFound) {
|
||||
return nil, nil, fmt.Errorf("%s: %w", id, ErrNotFound)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("gridfs open %s: %w", id, err)
|
||||
}
|
||||
f := ds.GetFile()
|
||||
var md struct {
|
||||
OwnerID string `bson:"ownerId"`
|
||||
Scope string `bson:"scope"`
|
||||
MimeType string `bson:"mimeType"`
|
||||
SHA256 string `bson:"sha256"`
|
||||
}
|
||||
if f.Metadata != nil {
|
||||
if err := bson.Unmarshal(f.Metadata, &md); err != nil {
|
||||
ds.Close()
|
||||
return nil, nil, fmt.Errorf("decode file metadata %s: %w", id, err)
|
||||
}
|
||||
}
|
||||
return ds, &Meta{
|
||||
ID: id, Name: f.Name, OwnerID: md.OwnerID, Scope: md.Scope,
|
||||
MimeType: md.MimeType, Size: f.Length, SHA256: md.SHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Delete removes the file and its chunks.
|
||||
func (s *Store) Delete(ctx context.Context, id string) error {
|
||||
if err := s.bucket.Delete(ctx, id); err != nil {
|
||||
if errors.Is(err, mongo.ErrFileNotFound) {
|
||||
return fmt.Errorf("%s: %w", id, ErrNotFound)
|
||||
}
|
||||
return fmt.Errorf("gridfs delete %s: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// countingReader tracks how many bytes the upload actually consumed, since
|
||||
// the driver's upload helper does not report a size.
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// SignedURLFor is a convenience for handing attachments to external services.
|
||||
func (s *Store) SignedURLFor(appBaseURL string, secret []byte, fileID string) string {
|
||||
return SignedURL(appBaseURL, secret, fileID, time.Now().Add(DefaultTokenTTL))
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//go:build integration
|
||||
|
||||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bountyboard/internal/testutil"
|
||||
)
|
||||
|
||||
func TestSaveOpenRoundTrip(t *testing.T) {
|
||||
db := testutil.MongoDB(t)
|
||||
s := NewStore(db, 25)
|
||||
ctx := context.Background()
|
||||
|
||||
content := []byte("%PDF-1.4 fake pdf content for sniffing")
|
||||
meta, err := s.Save(ctx, ScopeTask, "01JOWNER", "spec.pdf", "", bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
if meta.MimeType != "application/pdf" {
|
||||
t.Errorf("sniffed mime = %q, want application/pdf", meta.MimeType)
|
||||
}
|
||||
if meta.Size != int64(len(content)) {
|
||||
t.Errorf("size = %d, want %d", meta.Size, len(content))
|
||||
}
|
||||
wantSum := sha256.Sum256(content)
|
||||
if meta.SHA256 != hex.EncodeToString(wantSum[:]) {
|
||||
t.Errorf("sha256 mismatch")
|
||||
}
|
||||
|
||||
rc, got, err := s.Open(ctx, meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, content) {
|
||||
t.Error("downloaded content differs")
|
||||
}
|
||||
if got.OwnerID != "01JOWNER" || got.Scope != ScopeTask || got.Name != "spec.pdf" {
|
||||
t.Errorf("metadata round trip: %+v", got)
|
||||
}
|
||||
if got.SHA256 != meta.SHA256 || got.MimeType != meta.MimeType {
|
||||
t.Errorf("stored metadata mismatch: %+v vs %+v", got, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeclaredMimeUsedWhenSniffInconclusive(t *testing.T) {
|
||||
db := testutil.MongoDB(t)
|
||||
s := NewStore(db, 25)
|
||||
|
||||
meta, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "data.bin",
|
||||
"application/x-custom", bytes.NewReader([]byte{0x01, 0x02, 0x03}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.MimeType != "application/x-custom" {
|
||||
t.Errorf("mime = %q, want declared application/x-custom", meta.MimeType)
|
||||
}
|
||||
|
||||
// But a confident sniff overrides a lying declaration.
|
||||
meta2, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "fake.txt",
|
||||
"text/plain", strings.NewReader("\x89PNG\r\n\x1a\n123456789"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta2.MimeType != "image/png" {
|
||||
t.Errorf("mime = %q, want sniffed image/png", meta2.MimeType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeLimitEnforced(t *testing.T) {
|
||||
db := testutil.MongoDB(t)
|
||||
s := NewStore(db, 1) // 1 MiB cap
|
||||
|
||||
big := io.LimitReader(neverEnding('x'), 1<<20+1)
|
||||
_, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "big.bin", "", big)
|
||||
if !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("want ErrTooLarge, got %v", err)
|
||||
}
|
||||
|
||||
// No orphaned file documents may remain.
|
||||
n, err := s.bucket.GetFilesCollection().CountDocuments(context.Background(), struct{}{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("oversize upload left %d file docs behind", n)
|
||||
}
|
||||
|
||||
// Exactly at the limit is fine.
|
||||
ok := io.LimitReader(neverEnding('x'), 1<<20)
|
||||
if _, err := s.Save(context.Background(), ScopeChat, "01JOWNER", "exact.bin", "", ok); err != nil {
|
||||
t.Fatalf("upload at exactly the limit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMissingFile(t *testing.T) {
|
||||
db := testutil.MongoDB(t)
|
||||
s := NewStore(db, 25)
|
||||
if _, _, err := s.Open(context.Background(), "01JNOPE"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
db := testutil.MongoDB(t)
|
||||
s := NewStore(db, 25)
|
||||
ctx := context.Background()
|
||||
|
||||
meta, err := s.Save(ctx, ScopeAvatar, "01JOWNER", "a.txt", "", strings.NewReader("hello"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Delete(ctx, meta.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := s.Open(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound after delete, got %v", err)
|
||||
}
|
||||
if err := s.Delete(ctx, meta.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("double delete: want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// neverEnding is an infinite reader of one repeated byte.
|
||||
type neverEnding byte
|
||||
|
||||
func (b neverEnding) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = byte(b)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
Reference in New Issue
Block a user