diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6d5891e --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,25 @@ +version: "2" +linters: + settings: + errcheck: + exclude-functions: + - (*net/http.Response).Body.Close + - io.Closer.Close + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/AGENTS.md b/AGENTS.md index 23f17b9..317bf8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,8 @@ internal/server/ MCP handlers + x402 + analytics (~1800 lines in serv types.go request/response types, platform enum auth.go / webhook.go / cache*.go / *Middleware.go / networkInfo.go *_test.go split suites: mcp_test.go, analytics_test.go, validation_test.go, etc. -internal/{bluesky,nostr,activitypub,twitter,ollama,config,utils,actor,x402} +internal/activitypub/ AP client; throttle.go per-host token bucket (1/s) for background GTS warming +internal/{bluesky,nostr,twitter,ollama,config,utils,actor,x402} sql/schema/ 001..007 goose migrations sql/queries/ sqlc query files → internal/store/ config/models.env single source of truth for EMBEDDING_MODEL / STYLE_MODEL @@ -91,5 +92,6 @@ Loaded in order: `config/models.env` → `.env` (gitignored, via `godotenv`) → - Unix socket MCP + TCP readyz sidecar; `mcpServer.service` expects `bin/main` at repo root. - Platform inferred from author handle in `determinePlatform`/`fetch*` — not caller-supplied. - Bluesky auth refresh via `isAuthError`/`withRefresh` — 401/expired token retried once. -- Twitter client has per-limit clamp (`>100`/`>50` rejected) and `x-rate-limit-*`/`Retry-After` backoff. -- Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). +- Twitter client clamps `limit` (`>100` → 100), retries 429/5xx/transport errors with `x-rate-limit-*`/`Retry-After` backoff (bounded by `maxRateLimitWait`), and has a 60s per-request timeout. + - ActivityPub client: GTS Mastodon API only serves locally-federated statuses. Remote actors are resolved via `/accounts/search?resolve=true` (not `/accounts/lookup`, which skips WebFinger). When the local cache can't satisfy full `limit` (clamped to 200), `GetLatestNPosts`/`GetPostsInTimeframe` fall back to walking the remote outbox directly (webfinger → actor → `OrderedCollection`/`OrderedCollectionPage`), filtering `Create`/`Note`-like activities and dereferencing IRI-only objects; `GetPost` derefs status URLs over AP when GTS `/search` can't resolve them. The `read:search` scope is needed for status-URL resolution. Outbox-fetched posts are warmed into GTS in background via `GET /api/v2/search?resolve=true&type=statuses&limit=1` per post, throttled at 1 req/s per remote host via per-host token bucket (`internal/activitypub/throttle.go`) with cross-host concurrency, no follows. + - Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). diff --git a/cdpScripts/rescueFunds.py b/cdpScripts/rescueFunds.py deleted file mode 100644 index 18ef5fd..0000000 --- a/cdpScripts/rescueFunds.py +++ /dev/null @@ -1,25 +0,0 @@ -from cdp import cdp_client -import asyncio -import inspect -from dotenv import load_dotenv - -load_dotenv() - -OLD = "0x3C17021dF1071b7F45cd3D1e4Db0bcBd03C6ECF4" -NETWORK = "base-sepolia" - - -async def main(): - cdp = cdp_client.CdpClient() - - try: - private_key = await cdp.evm.export_account( - name="api-naco-li" - ) - print(f"Private key: {private_key}") - - finally: - await cdp.close() - - -asyncio.run(main()) \ No newline at end of file diff --git a/cmd/check-facilitator/main.go b/cmd/check-facilitator/main.go index db8aa29..717cf00 100644 --- a/cmd/check-facilitator/main.go +++ b/cmd/check-facilitator/main.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/nacorid/naco-api/internal/server" ) @@ -116,18 +117,22 @@ func main() { os.Exit(1) } - resp, err := http.DefaultClient.Do(req) + httpClient := &http.Client{ + Timeout: 30 * time.Second, + } + resp, err := httpClient.Do(req) if err != nil { fmt.Fprintf(os.Stderr, "Request failed: %v\n", err) os.Exit(1) } defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - fmt.Printf("Status: %d\n", resp.StatusCode) - fmt.Printf("Response: %s\n", string(body)) - - if resp.StatusCode != http.StatusOK { - os.Exit(1) + maxBytes := int64(5 * 1024 * 1024) + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes)) + if err != nil { + fmt.Printf("failed reading response body: %v", err) } + + fmt.Printf("Status: %d\n", resp.StatusCode) + fmt.Printf("Response: %s\n", body) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 300f84c..b42c61f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,9 @@ package main import ( "context" "database/sql" + "errors" + "fmt" + "io" "net/http" "net/url" "os" @@ -20,25 +23,45 @@ import ( "github.com/nacorid/naco-api/internal/server" "github.com/nacorid/naco-api/internal/store" "github.com/nacorid/naco-api/internal/twitter" + "golang.org/x/sync/errgroup" ) func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // backgroundCtx is for long-lived clients whose lifetime is the process, + // not a single request. It must not inherit signal cancellation so that a + // SIGTERM during startup does not poison an in-flight session creation. + backgroundCtx := context.Background() cfg := config.Load() - log.Init(log.Config{ + if err := log.Init(log.Config{ LogFilePath: cfg.LogFile, ConsoleLevel: log.LevelDebug, FileLevel: log.LevelTrace, - }) + }); err != nil { + log.Fatalf("Failed to init logger: %v", err) + } defer log.Close() + conn, err := sql.Open("postgres", cfg.DatabaseURL) if err != nil { - log.Fatalf("Failed to connect to database: %v", err) + log.Fatalf("Failed to initialize database connection: %v", err) } defer conn.Close() + + conn.SetMaxOpenConns(10) + conn.SetMaxIdleConns(10) + conn.SetConnMaxLifetime(5 * time.Minute) + + dbCtx, dbCancel := context.WithTimeout(ctx, 5*time.Second) + err = conn.PingContext(dbCtx) + dbCancel() + if err != nil { + log.Fatalf("Failed to ping database: %v", err) + } db := store.New(conn) if cfg.MigrateOnStart { @@ -48,7 +71,7 @@ func main() { log.Info("Database migrations applied", "dir", cfg.MigrationsDir) } - bskyClient, err := bluesky.NewBlueskyClient(ctx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) + bskyClient, err := bluesky.NewBlueskyClient(backgroundCtx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) if err != nil { log.Fatal("Failed to create bluesky client", "error", err) } @@ -60,9 +83,9 @@ func main() { ollamaHTTPClient := &http.Client{ Timeout: 600 * time.Second, } - ollamaClient := ollama.New(ctx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) + ollamaClient := ollama.New(uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) - apClient, err := activitypub.NewAPClient(ctx, cfg.ActivityPubHost, cfg.ActivityPubToken) + apClient, err := activitypub.NewAPClient(cfg.ActivityPubHost, cfg.ActivityPubToken) if err != nil { log.Fatal("Failed to create activitypub client", "error", err) } @@ -78,48 +101,86 @@ func main() { srv.StartPrewarm(ctx) - go func() { - mux := http.NewServeMux() - mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { - if err := conn.PingContext(r.Context()); err != nil { - http.Error(w, "database unavailable", http.StatusServiceUnavailable) - return - } - req, _ := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) - resp, err := http.DefaultClient.Do(req) - if err != nil || resp.StatusCode != http.StatusOK { - if resp != nil { - resp.Body.Close() - } - http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) - return - } - resp.Body.Close() - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - }) - mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") - w.WriteHeader(http.StatusOK) - w.Write([]byte(srv.MetricsText())) - }) - addr := ":" + cfg.ReadyzPort - log.Info("Readiness server starting", "address", addr) - if err := http.ListenAndServe(addr, mux); err != nil { - log.Error("Readiness server error", "error", err) - } - }() + g, gCtx := errgroup.WithContext(ctx) + startReadyzServer(g, ctx, cfg, conn) - go func() { + g.Go(func() error { log.Info("Server starting", "address", cfg.ListenAddress) - if err := srv.Start(); err != http.ErrServerClosed { - log.Fatal("HTTP server error", "error", err) + if err := srv.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("main server error: %w", err) } - }() + return nil + }) - stopChan := make(chan os.Signal, 1) - signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM) - <-stopChan + g.Go(func() error { + <-gCtx.Done() + log.Info("Shutting down main server...") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + return srv.Shutdown(shutdownCtx) + }) - log.Info("Shutting down server...") + if err := g.Wait(); err != nil && !errors.Is(err, context.Canceled) { + log.Error("Application terminated with error", "error", err) + } + + log.Info("Application shutdown complete") +} + +func startReadyzServer(g *errgroup.Group, shutdownCtx context.Context, cfg *config.Config, conn *sql.DB) { + mux := http.NewServeMux() + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + if err := conn.PingContext(r.Context()); err != nil { + http.Error(w, "database unavailable", http.StatusServiceUnavailable) + return + } + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) + if err != nil { + http.Error(w, "invalid ollama configuration", http.StatusInternalServerError) + return + } + + resp, err := client.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + if resp != nil { + resp.Body.Close() + } + http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) + return + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + _, _ = w.Write([]byte("ok")) + }) + + readyServer := &http.Server{ + Addr: cfg.ReadyzAddress + ":" + cfg.ReadyzPort, + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + } + + g.Go(func() error { + log.Info("Readiness server starting", "address", readyServer.Addr) + if err := readyServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("readiness server failed: %w", err) + } + return nil + }) + + g.Go(func() error { + <-shutdownCtx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return readyServer.Shutdown(shutdownCtx) + }) } diff --git a/go.mod b/go.mod index 5a95b31..13124e1 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 github.com/modelcontextprotocol/go-sdk v1.6.1 - github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 + github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 github.com/ollama/ollama v0.12.10 github.com/x402-foundation/x402/go/v2 v2.19.0 golang.org/x/time v0.14.0 @@ -28,7 +28,7 @@ require ( github.com/klauspost/compress v1.18.1 // indirect github.com/supranational/blst v0.3.16 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - golang.org/x/sync v0.19.0 // indirect + golang.org/x/sync v0.19.0 ) require ( diff --git a/go.sum b/go.sum index 9ef062a..d43b247 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7B github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 h1:nZwGInvJNQzPSFYpstVR/TLumqBEZt0+yUXchdyBZiI= -github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= +github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 h1:o7CvgRLUikfsbyuZIJTTrfRac5rMXhca0b61VLBWSIA= +github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/ollama/ollama v0.12.10 h1:Dd0/SeCc+nv+FffxmWuQTGiRreib7Gt3nBhIIFuKwZA= github.com/ollama/ollama v0.12.10/go.mod h1:RUSmYywUWx/YZMaHrqtnT1ZChu+iSz/7jx2aO9+Mgfg= diff --git a/internal/activitypub/client.go b/internal/activitypub/client.go index cc45f07..c3684dd 100644 --- a/internal/activitypub/client.go +++ b/internal/activitypub/client.go @@ -7,189 +7,374 @@ import ( "io" "net/http" "net/url" + "strings" + "sync" "time" "github.com/nacorid/naco-api/internal/actor" ) +const maxMastodonPageSize = 40 +const maxTimeframePages = 100 +const maxResponseBody = 1 << 20 +const maxLimit = 200 + type APClient struct { client *http.Client host string bearerToken string + outbox *outboxClient + limiter *hostLimiter } -func NewAPClient(ctx context.Context, host, apikey string) (*APClient, error) { +func NewAPClient(host, apikey string) (*APClient, error) { if host == "" || apikey == "" { return nil, fmt.Errorf("host and apikey must not be empty") } + return &APClient{ - client: http.DefaultClient, - host: host, - bearerToken: fmt.Sprintf("Bearer %s", apikey), + client: &http.Client{Timeout: 30 * time.Second}, + host: strings.TrimRight(host, "/"), + bearerToken: "Bearer " + apikey, + outbox: newOutboxClient(), + limiter: newHostLimiter(), }, nil } -func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { - url := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, postID) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) +func (ap *APClient) get(ctx context.Context, endpoint string, target any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Authorization", ap.bearerToken) + resp, err := ap.client.Do(req) if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + + return nil +} + +func (ap *APClient) accountSearch(ctx context.Context, query string) (*Account, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=1", ap.host, url.QueryEscape(query)) + var accounts []*Account + if err := ap.get(ctx, endpoint, &accounts); err != nil { + return nil, fmt.Errorf("failed to search accounts: %w", err) + } + for _, acc := range accounts { + if acc != nil { + return acc, nil + } + } + return nil, nil +} + +func (ap *APClient) resolveAccount(ctx context.Context, actorHandle string) (*Account, error) { + handle := strings.TrimPrefix(actorHandle, "@") + + acc, err := ap.accountSearch(ctx, handle) + if err != nil { + return nil, err + } + if acc != nil { + return acc, nil + } + + if !strings.Contains(handle, "@") { + var byID Account + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(handle)) + if err := ap.get(ctx, endpoint, &byID); err != nil { + return nil, fmt.Errorf("account %q not found by search or id: %w", actorHandle, err) + } + if byID.ID == "" { + return nil, fmt.Errorf("account %q not found", actorHandle) + } + return &byID, nil + } + return nil, fmt.Errorf("account %q not found", actorHandle) +} + +func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { + status, err := ap.resolveStatus(ctx, postID) + if err != nil { + return nil, err + } + return status.toPost(), nil +} + +func (ap *APClient) resolveStatus(ctx context.Context, postID string) (*internalPost, error) { + if strings.HasPrefix(postID, "http://") || strings.HasPrefix(postID, "https://") || strings.HasPrefix(postID, "//") { + if status, err := ap.searchStatus(ctx, postID); err == nil && status != nil { + return status, nil + } + note, derr := ap.outbox.resolveNote(ctx, postID) + if derr != nil { + return nil, fmt.Errorf("status %q not resolvable: %w", postID, derr) + } + author := extractHandleFromURL(postID) + return noteToInternalPost(note, author), nil + } + + endpoint := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, url.PathEscape(postID)) + var status internalPost + if err := ap.get(ctx, endpoint, &status); err != nil { return nil, fmt.Errorf("failed to get post: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get post: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var internal = &internalPost{} - if err = json.Unmarshal(body, internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal json response: %w", err) - } - return internal.toPost(), nil + return &status, nil } -func (ap *APClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { - url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, actor, limit) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) +func (ap *APClient) searchStatus(ctx context.Context, postID string) (*internalPost, error) { + endpoint := fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(postID)) + var res searchResult + if err := ap.get(ctx, endpoint, &res); err != nil { + return nil, fmt.Errorf("failed to resolve status %q: %w", postID, err) } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get latest posts: %w", err) + for _, s := range res.Statuses { + if s != nil && (s.URL == postID || s.URI == postID) { + return s, nil + } } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get latest posts: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var internal []*internalPost - if err = json.Unmarshal(body, &internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal json response: %w", err) - } - posts := make([]*Post, len(internal)) - for i, p := range internal { - posts[i] = p.toPost() - } - return posts, nil + return nil, nil } -func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]Post, error) { - var posts []Post - var maxID string +func (ap *APClient) GetLatestNPosts(ctx context.Context, actorID string, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + if limit > maxLimit { + limit = maxLimit + } - for { + acc, err := ap.resolveAccount(ctx, actorID) + if err != nil { + return nil, err + } + + cached, cacheErr := ap.cacheLatestPosts(ctx, acc, limit) + if cacheErr != nil { + return nil, cacheErr + } + if len(cached) >= limit { + return cachePostsToPosts(cached), nil + } + + if !isRemoteAccount(acc) || acc.URL == "" { + return cachePostsToPosts(cached), nil + } + + outboxPosts, apErr := ap.outbox.getLatest(ctx, acc.URL, acc.Acct, limit-len(cached)) + if apErr != nil { + return nil, fmt.Errorf("outbox fallback failed: %w", apErr) + } + merged := mergePostsBySource(cached, outboxPosts) + if len(outboxPosts) > 0 { + urls := make([]string, 0, len(outboxPosts)) + for _, p := range outboxPosts { + if p != nil && p.ID != "" { + urls = append(urls, p.ID) + } + } + go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) + } + return merged, nil +} + +func (ap *APClient) cacheLatestPosts(ctx context.Context, acc *Account, limit int) ([]*internalPost, error) { + var ( + internal []*internalPost + maxID string + ) + + for len(internal) < limit { if ctx.Err() != nil { return nil, ctx.Err() } - if len(posts) >= limit { - break + + pageSize := limit - len(internal) + if pageSize > maxMastodonPageSize { + pageSize = maxMastodonPageSize } - url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=50", ap.host, actor) + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), pageSize) if maxID != "" { - url += fmt.Sprintf("&max_id=%s", maxID) + endpoint += "&max_id=" + url.QueryEscape(maxID) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + var page []*internalPost + if err := ap.get(ctx, endpoint, &page); err != nil { + return nil, err } - req.Header.Set("Authorization", ap.bearerToken) - - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get posts: %w", err) - } - // !!IMPORTANT!! Manually close body instead of deferring, due to loop - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("failed to get posts: %s", resp.Status) - } - - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - var internal []*internalPost - if err := json.Unmarshal(body, &internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal response body: %w", err) - } - - if len(internal) == 0 { + if len(page) == 0 { break } - for _, i := range internal { + newMaxID := page[len(page)-1].ID + if newMaxID == "" { + break + } + if maxID != "" && newMaxID == maxID { + break + } + + internal = append(internal, page...) + maxID = newMaxID + } + if len(internal) > limit { + internal = internal[:limit] + } + return internal, nil +} + +func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, start, end time.Time, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + if limit > maxLimit { + limit = maxLimit + } + + acc, err := ap.resolveAccount(ctx, actorID) + if err != nil { + return nil, err + } + + cached, cacheErr := ap.cachePostsInTimeframe(ctx, acc, start, end, limit) + if cacheErr != nil { + return nil, cacheErr + } + if len(cached) >= limit { + return cachePostsToPosts(cached), nil + } + + if !isRemoteAccount(acc) || acc.URL == "" { + return cachePostsToPosts(cached), nil + } + + outboxPosts, apErr := ap.outbox.getInTimeframe(ctx, acc.URL, acc.Acct, start, end, limit-len(cached)) + if apErr != nil { + return nil, fmt.Errorf("outbox fallback failed: %w", apErr) + } + merged := mergePostsBySource(cached, outboxPosts) + if len(outboxPosts) > 0 { + urls := make([]string, 0, len(outboxPosts)) + for _, p := range outboxPosts { + if p != nil && p.ID != "" { + urls = append(urls, p.ID) + } + } + go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) + } + return merged, nil +} + +func (ap *APClient) cachePostsInTimeframe(ctx context.Context, acc *Account, start, end time.Time, limit int) ([]*internalPost, error) { + var ( + internal []*internalPost + maxID string + ) + + for pages := 0; pages < maxTimeframePages; pages++ { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if len(internal) >= limit { + break + } + + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), maxMastodonPageSize) + if maxID != "" { + endpoint += "&max_id=" + url.QueryEscape(maxID) + } + + var page []*internalPost + if err := ap.get(ctx, endpoint, &page); err != nil { + return nil, err + } + if len(page) == 0 { + break + } + + newMaxID := page[len(page)-1].ID + if newMaxID == "" { + break + } + if maxID != "" && newMaxID == maxID { + break + } + + for _, i := range page { if i.CreatedAt.After(end) { continue } - if i.CreatedAt.Before(start) { - return posts, nil + return internal, nil } - - posts = append(posts, *i.toPost()) - - if len(posts) >= limit { - return posts, nil + internal = append(internal, i) + if len(internal) >= limit { + return internal, nil } } - - maxID = internal[len(internal)-1].ID + maxID = newMaxID } - return posts, nil + return internal, nil } -func (ap *APClient) GetActor(ctx context.Context, actor string) (*actor.Profile, error) { - // Mastodon exposes actor lookup by acct; fall back to a direct id lookup. - prof, err := ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actor))) - if err != nil { - prof, err = ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actor))) - if err != nil { - return nil, err +func isRemoteAccount(acc *Account) bool { + return acc != nil && strings.Contains(acc.Acct, "@") +} + +func cachePostsToPosts(cached []*internalPost) []*Post { + if len(cached) == 0 { + return nil + } + posts := make([]*Post, 0, len(cached)) + for _, c := range cached { + if c == nil { + continue } + posts = append(posts, c.toPost()) } - return prof, nil + return posts } -func (ap *APClient) fetchAccount(ctx context.Context, endpoint string) (*actor.Profile, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) +func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { + acc, err := ap.resolveAccount(ctx, actorHandle) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return nil, err } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get actor: %w", err) + return ap.toProfile(acc), nil +} + +func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=%d", ap.host, url.QueryEscape(query), limit) + var accounts []*Account + if err := ap.get(ctx, endpoint, &accounts); err != nil { + return nil, fmt.Errorf("failed to search actors: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get actor: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var acc Account - if err := json.Unmarshal(body, &acc); err != nil { - return nil, fmt.Errorf("failed to unmarshal account: %w", err) + + results := make([]*actor.Profile, 0, len(accounts)) + for _, acc := range accounts { + if acc == nil { + continue + } + results = append(results, ap.toProfile(acc)) } + return results, nil +} + +func (ap *APClient) toProfile(acc *Account) *actor.Profile { return &actor.Profile{ Platform: "ActivityPub", Handle: acc.Acct, @@ -200,45 +385,74 @@ func (ap *APClient) fetchAccount(ctx context.Context, endpoint string) (*actor.P FollowingCount: acc.FollowingCount, PostCount: acc.StatusesCount, URL: acc.URL, - }, nil + } } -func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&limit=%d", ap.host, url.QueryEscape(query), limit) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) +func extractHandleFromURL(postURL string) string { + u, err := url.Parse(postURL) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return "" } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to search actors: %w", err) + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + for _, p := range parts { + if strings.HasPrefix(p, "@") { + return strings.TrimPrefix(p, "@") + } } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to search actors: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var accounts []Account - if err := json.Unmarshal(body, &accounts); err != nil { - return nil, fmt.Errorf("failed to unmarshal accounts: %w", err) - } - results := make([]actor.Profile, 0, len(accounts)) - for _, acc := range accounts { - results = append(results, actor.Profile{ - Platform: "ActivityPub", - Handle: acc.Acct, - DisplayName: acc.DisplayName, - AvatarURL: acc.Avatar, - Bio: acc.Note, - FollowersCount: acc.FollowersCount, - FollowingCount: acc.FollowingCount, - PostCount: acc.StatusesCount, - URL: acc.URL, - }) - } - return results, nil + return "" +} + +func (ap *APClient) warmGTSAsync(ctx context.Context, urls []string) { + if len(urls) == 0 { + return + } + if ap.limiter == nil { + ap.limiter = newHostLimiter() + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + seen := make(map[string]struct{}, len(urls)) + uniq := make([]string, 0, len(urls)) + for _, u := range urls { + if _, ok := seen[u]; ok { + continue + } + seen[u] = struct{}{} + uniq = append(uniq, u) + } + if len(uniq) > 50 { + uniq = uniq[:50] + } + + byHost := make(map[string][]string) + for _, u := range uniq { + parsed, err := url.Parse(u) + if err != nil || parsed.Host == "" { + continue + } + byHost[parsed.Host] = append(byHost[parsed.Host], u) + } + if len(byHost) == 0 { + return + } + + var wg sync.WaitGroup + for host, us := range byHost { + wg.Add(1) + go func(host string, us []string) { + defer wg.Done() + for _, u := range us { + if err := ap.limiter.Wait(ctx, host); err != nil { + return + } + if ctx.Err() != nil { + return + } + var res searchResult + _ = ap.get(ctx, fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(u)), &res) + } + }(host, us) + } + wg.Wait() } diff --git a/internal/activitypub/client_test.go b/internal/activitypub/client_test.go index 2fdfa11..04fbffd 100644 --- a/internal/activitypub/client_test.go +++ b/internal/activitypub/client_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -16,7 +17,7 @@ type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func newTestClient(serverURL string) *APClient { - ap, err := NewAPClient(context.Background(), "https://example.com", "tok") + ap, err := NewAPClient("https://example.com", "tok") if err != nil { panic(err) } @@ -35,22 +36,29 @@ func newTestClient(serverURL string) *APClient { } func TestNewAPClientEmpty(t *testing.T) { - if _, err := NewAPClient(context.Background(), "", "tok"); err == nil { + if _, err := NewAPClient("", "tok"); err == nil { t.Fatal("expected error for empty host") } - if _, err := NewAPClient(context.Background(), "host", ""); err == nil { + if _, err := NewAPClient("host", ""); err == nil { t.Fatal("expected error for empty apikey") } } func TestGetLatestNPosts(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer tok" { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -68,10 +76,92 @@ func TestGetLatestNPosts(t *testing.T) { } } +func TestGetLatestNPostsPaginates(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get("max_id") { + case "40": + ids := make([]map[string]any, 0, 10) + for id := 41; id <= 50; id++ { + ids = append(ids, map[string]any{ + "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", + "account": map[string]any{"acct": "alice"}, + }) + } + _ = json.NewEncoder(w).Encode(ids) + case "": + ids := make([]map[string]any, 0, 40) + for id := 1; id <= 40; id++ { + ids = append(ids, map[string]any{ + "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", + "account": map[string]any{"acct": "alice"}, + }) + } + _ = json.NewEncoder(w).Encode(ids) + default: + _ = json.NewEncoder(w).Encode([]map[string]any{}) + } + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 50) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 50 { + t.Fatalf("expected 50 posts across 2 pages, got %d", len(posts)) + } + if posts[49].Content != "post 50" { + t.Fatalf("expected last post 'post 50', got %q", posts[49].Content) + } +} + +func TestGetLatestNPostsZeroLimit(t *testing.T) { + ap := newTestClient("http://example.com") + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 0) + if err != nil { + t.Fatalf("GetLatestNPosts with zero limit error: %v", err) + } + if len(posts) != 0 { + t.Fatalf("expected no posts for zero limit, got %d", len(posts)) + } +} + +func TestGetLatestNPostsLimitClamp(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + limit := r.URL.Query().Get("limit") + l, _ := strconv.Atoi(limit) + if l > maxLimit { + t.Fatalf("expected limit <= %d, got %s", maxLimit, limit) + } + _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 500) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) == 0 { + t.Fatal("expected at least one post") + } +} + func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/statuses/42", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) + _, _ = fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -86,34 +176,215 @@ func TestGetPost(t *testing.T) { } } -func TestGetLatestNPostsHTTPError(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) +func TestGetPostURLResolves(t *testing.T) { + url := "https://inst.social/@alice/statuses/42" + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("q") != url { + w.WriteHeader(http.StatusBadRequest) + return + } + if r.URL.Query().Get("type") != "statuses" || r.URL.Query().Get("resolve") != "true" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"42","uri":"`+url+`","url":"`+url+`","content":"resolved post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob@inst.social"}}]}`) + }) + server := httptest.NewServer(mux) defer server.Close() ap := newTestClient(server.URL) - if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { - t.Fatal("expected error on non-200 status") + post, err := ap.GetPost(context.Background(), url) + if err != nil { + t.Fatalf("GetPost error: %v", err) + } + if post.Content != "resolved post" || post.Author != "bob@inst.social" { + t.Fatalf("unexpected post: %+v", post) + } +} + +func TestGetPostURLNotFound(t *testing.T) { + derefTarget := httptest.NewServer(http.NotFoundHandler()) + defer derefTarget.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[]}`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + if _, err := ap.GetPost(context.Background(), derefTarget.URL+"/missed"); err == nil { + t.Fatal("expected error when search and dereference both return nothing") + } +} + +func TestGetPostURLSearchReturnsWrongPost(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"wrong","uri":"https://other.com/1","content":"wrong post"}]}`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + if _, err := ap.GetPost(context.Background(), "https://inst.social/@alice/statuses/42"); err == nil { + t.Fatal("expected error when search returns no matching post") + } +} + +func TestGetPostURLDerefFallback(t *testing.T) { + remote, _ := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + post, err := ap.GetPost(context.Background(), remote.URL+"/notes/2") + if err != nil { + t.Fatalf("GetPost error: %v", err) + } + if post.Content != "deref note 2" { + t.Fatalf("expected dereferenced note, got %+v", post) + } +} + +func TestGetLatestNPostsRemoteFallsBackToOutbox(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached","uri":`+`"`+remoteBase+`/notes/9","created_at":"2024-03-10T00:00:00Z","account":{"acct":"alice@example.social"}}]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 5) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 5 { + t.Fatalf("expected 5 merged posts, got %d: %+v", len(posts), posts) + } + if posts[0].Content != "cached" { + t.Fatalf("expected newest cached first, got %q", posts[0].Content) + } + if posts[4].Content != "note 4" { + t.Fatalf("expected oldest outbox post last, got %q", posts[4].Content) + } +} + +func TestGetLatestNPostsRemoteEmptyCacheFallsBack(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 4) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected all 4 posts from outbox, got %d", len(posts)) + } + if posts[0].Content != "bare note 3" { + t.Fatalf("expected outbox post newest-first, got %q", posts[0].Content) + } +} + +func TestGetPostsInTimeframeRemoteFallsBackToOutbox(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached in window","uri":`+`"`+remoteBase+`/notes/5","created_at":"2024-02-20T00:00:00Z","account":{"acct":"alice@example.social"}}]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) + posts, err := ap.GetPostsInTimeframe(context.Background(), "@alice@example.social", start, end, 3) + if err != nil { + t.Fatalf("GetPostsInTimeframe error: %v", err) + } + if len(posts) != 3 { + t.Fatalf("expected 3 in-window posts, got %d: %+v", len(posts), posts) + } + if posts[0].Content != "created note 1" || posts[1].Content != "cached in window" || posts[2].Content != "deref note 2" { + t.Fatalf("unexpected merged order: %+v", posts) } } func TestGetPostsInTimeframe(t *testing.T) { - // Feed is reverse-chronological: newer than `end`, two in-window, then older - // than `start`. The client must skip the too-new post, return both in-window - // posts, and stop when it crosses the `start` boundary. mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { feed := []map[string]any{} switch r.URL.Query().Get("max_id") { case "": - // Page 1: too-new post and one in-window post. feed = []map[string]any{ {"id": "5", "content": "too-new", "text": "too-new", "created_at": "2025-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "4", "content": "mid", "text": "mid", "created_at": "2024-06-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, } case "4": - // Page 2: in-window post then too-old post. feed = []map[string]any{ {"id": "3", "content": "early-mid", "text": "early-mid", "created_at": "2024-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "2", "content": "too-old", "text": "too-old", "created_at": "2023-01-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, @@ -142,14 +413,31 @@ func TestGetPostsInTimeframe(t *testing.T) { } } +func TestGetLatestNPostsHTTPError(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { + t.Fatal("expected error on non-200 status from timeline") + } +} + func TestGetActor(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("acct") != "alice" { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("q") != "alice" { w.WriteHeader(http.StatusBadRequest) return } - fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}`) + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -175,11 +463,11 @@ func TestGetActor(t *testing.T) { func TestGetActorIDFallback(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[]`) }) mux.HandleFunc("/api/v1/accounts/alice", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) + _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -194,6 +482,29 @@ func TestGetActorIDFallback(t *testing.T) { } } +func TestGetActorAtLocalHandle(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if q != "@alice" && q != "alice" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + prof, err := ap.GetActor(context.Background(), "@alice") + if err != nil { + t.Fatalf("GetActor error: %v", err) + } + if prof.Handle != "alice" { + t.Fatalf("expected local handle, got %q", prof.Handle) + } +} + func TestSearchActors(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { @@ -201,7 +512,11 @@ func TestSearchActors(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) + if r.URL.Query().Get("resolve") != "true" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -218,3 +533,65 @@ func TestSearchActors(t *testing.T) { t.Fatalf("unexpected result: %+v", results[0]) } } + +func TestExtractHandleFromURL(t *testing.T) { + tests := []struct { + url string + expected string + }{ + {"https://inst.social/@alice/statuses/42", "alice"}, + {"http://example.com/@bob/posts/1", "bob"}, + {"https://no-at-here.com/notes/1", ""}, + {"https://host/@user/note", "user"}, + } + for _, tt := range tests { + got := extractHandleFromURL(tt.url) + if got != tt.expected { + t.Fatalf("extractHandleFromURL(%q) = %q, want %q", tt.url, got, tt.expected) + } + } +} + +func TestGetPostBareLocalHandle(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if q != "@alice" && q != "alice" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/statuses/alice", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"id":"1","content":"local post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + _, err := ap.GetPost(context.Background(), "alice") + if err != nil { + t.Fatalf("GetPost with local handle error: %v", err) + } +} + +func TestSearchActorsNilEntries(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[null,{"id":"1","acct":"alice@inst.social"}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + results, err := ap.SearchActors(context.Background(), "alice", 10) + if err != nil { + t.Fatalf("SearchActors error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result after filtering nil, got %d", len(results)) + } + if results[0].Handle != "alice@inst.social" { + t.Fatalf("unexpected result: %+v", results[0]) + } +} diff --git a/internal/activitypub/outbox.go b/internal/activitypub/outbox.go new file mode 100644 index 0000000..f7dc81f --- /dev/null +++ b/internal/activitypub/outbox.go @@ -0,0 +1,501 @@ +package activitypub + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const outboxMaxPages = 100 + +var webfingerScheme = "https" + +type outboxClient struct { + client *http.Client +} + +type apStringOrArray []string + +func (s *apStringOrArray) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + *s = nil + return nil + } + if b[0] == '"' { + var v string + if err := json.Unmarshal(b, &v); err != nil { + return err + } + *s = []string{v} + return nil + } + var arr []string + if err := json.Unmarshal(b, &arr); err != nil { + return err + } + *s = arr + return nil +} + +type apIRI string + +func (i *apIRI) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + *i = "" + return nil + } + if b[0] == '"' { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + *i = apIRI(s) + return nil + } + var o struct { + ID string `json:"id"` + } + if err := json.Unmarshal(b, &o); err != nil { + return err + } + *i = apIRI(o.ID) + return nil +} + +type apRef struct { + IRI string + Inline json.RawMessage +} + +func (r *apRef) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + r.IRI = "" + r.Inline = nil + return nil + } + if b[0] == '"' { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + r.IRI = s + return nil + } + r.Inline = append([]byte(nil), b...) + return nil +} + +type apActor struct { + ID string `json:"id"` + Type string `json:"type"` + PreferredUsername string `json:"preferredUsername"` + URL string `json:"url"` + Outbox apIRI `json:"outbox"` + Inbox apIRI `json:"inbox"` +} + +type apActivity struct { + ID string `json:"id"` + Type string `json:"type"` + Published string `json:"published"` + Content string `json:"content"` + URL apStringOrArray `json:"url"` + Object json.RawMessage `json:"object"` +} + +type apCollection struct { + Type string `json:"type"` + First apRef `json:"first"` +} + +type apPage struct { + Type string `json:"type"` + OrderedItems json.RawMessage `json:"orderedItems"` + Next apRef `json:"next"` +} + +type apNote struct { + ID string `json:"id"` + Type string `json:"type"` + Published string `json:"published"` + Content string `json:"content"` + URL apStringOrArray `json:"url"` +} + +type webfingerResponse struct { + Subject string `json:"subject"` + Links []struct { + Rel string `json:"rel"` + Type string `json:"type"` + Href string `json:"href"` + } `json:"links"` +} + +func newOutboxClient() *outboxClient { + return &outboxClient{client: &http.Client{Timeout: 30 * time.Second}} +} + +func (oc *outboxClient) getJSON(ctx context.Context, endpoint string, target any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", `application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/json`) + + resp, err := oc.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + return nil +} + +func (oc *outboxClient) webfinger(ctx context.Context, handle string) (string, error) { + full := strings.TrimPrefix(handle, "@") + at := strings.LastIndex(full, "@") + if at <= 0 || at == len(full)-1 { + return "", fmt.Errorf("invalid handle %q", handle) + } + domain := full[at+1:] + + endpoint := fmt.Sprintf("%s://%s/.well-known/webfinger?resource=%s", webfingerScheme, domain, url.QueryEscape("acct:"+full)) + var wf webfingerResponse + if err := oc.getJSON(ctx, endpoint, &wf); err != nil { + return "", fmt.Errorf("webfinger lookup for %q failed: %w", full, err) + } + + var htmlSelf string + for _, l := range wf.Links { + if l.Rel != "self" || l.Href == "" { + continue + } + if strings.Contains(l.Type, "activity+json") || strings.Contains(l.Type, "ld+json") { + return l.Href, nil + } + if htmlSelf == "" && strings.Contains(l.Type, "html") { + htmlSelf = l.Href + } + } + if htmlSelf != "" { + return htmlSelf, nil + } + return "", fmt.Errorf("webfinger for %q returned no self link", full) +} + +func (oc *outboxClient) getActor(ctx context.Context, actorURL string) (*apActor, error) { + var actor apActor + if err := oc.getJSON(ctx, actorURL, &actor); err != nil { + return nil, fmt.Errorf("failed to fetch actor %q: %w", actorURL, err) + } + if actor.ID == "" && actor.Outbox == "" { + return nil, fmt.Errorf("response from %q is not an actor document", actorURL) + } + return &actor, nil +} + +func (oc *outboxClient) resolveActor(ctx context.Context, actorURL, handle string) (*apActor, error) { + if actorURL == "" { + var err error + actorURL, err = oc.webfinger(ctx, handle) + if err != nil { + return nil, err + } + } + return oc.getActor(ctx, actorURL) +} + +func (oc *outboxClient) getLatest(ctx context.Context, actorURL, author string, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + return oc.walk(ctx, actorURL, author, limit, func(*Post) bool { return true }) +} + +func (oc *outboxClient) getInTimeframe(ctx context.Context, actorURL, author string, start, end time.Time, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + return oc.walk(ctx, actorURL, author, limit, func(p *Post) bool { + return !p.CreatedAt.Before(start) && !p.CreatedAt.After(end) + }) +} + +func (oc *outboxClient) walk(ctx context.Context, actorURL, author string, limit int, include func(*Post) bool) ([]*Post, error) { + actor, err := oc.resolveActor(ctx, actorURL, author) + if err != nil { + return nil, err + } + if actor.Outbox == "" { + return nil, fmt.Errorf("actor %q exposes no outbox", actorURL) + } + + var posts []*Post + seen := make(map[string]struct{}) + + cur, err := oc.firstPage(ctx, string(actor.Outbox)) + if err != nil { + return nil, err + } + + for pages := 0; pages < outboxMaxPages; pages++ { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if len(posts) >= limit { + break + } + + page, ok, err := oc.getPage(ctx, cur) + if err != nil { + return nil, err + } + if !ok { + break + } + + var items []apActivity + if len(page.OrderedItems) > 0 { + if err := json.Unmarshal(page.OrderedItems, &items); err != nil { + return nil, fmt.Errorf("failed to decode outbox page items: %w", err) + } + } + + for _, item := range items { + p, valid, err := oc.itemToPost(ctx, item, author) + if err != nil { + return nil, err + } + if !valid || p == nil { + continue + } + if _, dup := seen[p.ID]; dup { + continue + } + seen[p.ID] = struct{}{} + if include(p) { + posts = append(posts, p) + if len(posts) >= limit { + return posts, nil + } + } + } + + cur = page.Next + if cur.IRI == "" && len(cur.Inline) == 0 { + break + } + } + + return posts, nil +} + +func (oc *outboxClient) firstPage(ctx context.Context, outbox string) (apRef, error) { + var coll apCollection + if err := oc.getJSON(ctx, outbox, &coll); err != nil { + return apRef{}, fmt.Errorf("failed to fetch outbox %q: %w", outbox, err) + } + if coll.Type != "" && coll.Type != "OrderedCollection" { + return apRef{}, fmt.Errorf("outbox %q has unexpected type %q", outbox, coll.Type) + } + if coll.First.IRI == "" && len(coll.First.Inline) == 0 { + return apRef{}, fmt.Errorf("outbox %q exposes no pages", outbox) + } + return coll.First, nil +} + +func (oc *outboxClient) getPage(ctx context.Context, ref apRef) (apPage, bool, error) { + var page apPage + switch { + case ref.IRI != "": + if err := oc.getJSON(ctx, ref.IRI, &page); err != nil { + return apPage{}, false, fmt.Errorf("failed to fetch outbox page %q: %w", ref.IRI, err) + } + case len(ref.Inline) > 0: + if err := json.Unmarshal(ref.Inline, &page); err != nil { + return apPage{}, false, fmt.Errorf("failed to decode inline outbox page: %w", err) + } + default: + return apPage{}, false, nil + } + return page, true, nil +} + +func (oc *outboxClient) itemToPost(ctx context.Context, item apActivity, author string) (*Post, bool, error) { + var note *apNote + var err error + + switch item.Type { + case "Create", "Update": + note, err = oc.noteFromObject(ctx, item.Object) + case "Note", "Article", "Question", "Page", "": + note = &apNote{ + ID: item.ID, + Type: item.Type, + Published: item.Published, + Content: item.Content, + URL: item.URL, + } + default: + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if note == nil || !isPostNote(note.Type) { + return nil, false, nil + } + + id := note.ID + if id == "" && len(note.URL) > 0 { + id = note.URL[0] + } + if id == "" { + return nil, false, nil + } + + created, ok := parseActivityTime(note.Published) + if !ok || created.IsZero() { + return nil, false, nil + } + + return &Post{ + ID: id, + Author: author, + Content: note.Content, + CreatedAt: created, + }, true, nil +} + +func (oc *outboxClient) noteFromObject(ctx context.Context, raw json.RawMessage) (*apNote, error) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + if raw[0] == '"' { + var id string + if err := json.Unmarshal(raw, &id); err != nil { + return nil, err + } + return oc.fetchNote(ctx, id) + } + var n apNote + if err := json.Unmarshal(raw, &n); err != nil { + return nil, err + } + return &n, nil +} + +func (oc *outboxClient) fetchNote(ctx context.Context, iri string) (*apNote, error) { + var n apNote + if err := oc.getJSON(ctx, iri, &n); err != nil { + return nil, fmt.Errorf("failed to dereference object %q: %w", iri, err) + } + return &n, nil +} + +func (oc *outboxClient) resolveNote(ctx context.Context, url string) (*apNote, error) { + var act apActivity + if err := oc.getJSON(ctx, url, &act); err != nil { + return nil, fmt.Errorf("failed to dereference status %q: %w", url, err) + } + switch act.Type { + case "Create": + return oc.noteFromObject(ctx, act.Object) + case "Note", "Article", "Question", "Page", "": + return &apNote{ + ID: act.ID, + Type: act.Type, + Published: act.Published, + Content: act.Content, + URL: act.URL, + }, nil + default: + return nil, fmt.Errorf("document at %q is not a post (type %q)", url, act.Type) + } +} + +func isPostNote(t string) bool { + switch t { + case "", "Note", "Article", "Question", "Page": + return true + } + return false +} + +func parseActivityTime(s string) (time.Time, bool) { + if s == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05"} { + if t, err := time.Parse(layout, s); err == nil { + return t, true + } + } + return time.Time{}, false +} + +func noteToInternalPost(n *apNote, author string) *internalPost { + p := &internalPost{ + ID: n.ID, + URI: n.ID, + Content: n.Content, + } + if len(n.URL) > 0 { + p.URL = n.URL[0] + } + if t, ok := parseActivityTime(n.Published); ok { + p.CreatedAt = t + } + if author != "" { + p.Account = Account{Acct: author} + } + return p +} + +func mergePostsBySource(cached []*internalPost, outbox []*Post) []*Post { + seen := make(map[string]struct{}, len(cached)) + res := make([]*Post, 0, len(cached)+len(outbox)) + for _, c := range cached { + key := c.URI + if key == "" { + key = c.ID + } + if key != "" { + seen[key] = struct{}{} + } + if c != nil { + res = append(res, c.toPost()) + } + } + for _, p := range outbox { + if p == nil || p.ID == "" { + continue + } + if _, dup := seen[p.ID]; dup { + continue + } + seen[p.ID] = struct{}{} + res = append(res, p) + } + sort.SliceStable(res, func(i, j int) bool { return res[i].CreatedAt.After(res[j].CreatedAt) }) + return res +} diff --git a/internal/activitypub/outbox_test.go b/internal/activitypub/outbox_test.go new file mode 100644 index 0000000..fb8c733 --- /dev/null +++ b/internal/activitypub/outbox_test.go @@ -0,0 +1,224 @@ +package activitypub + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// newRemoteServer serves a synthetic ActivityPub instance: a webfinger +// endpoint, an actor with an outbox collection, two outbox pages (with mixed +// activity types and an IRI-referenced object) and a dereferenceable note. +// Returns the server and the actor's canonical URL. +func newRemoteServer() (*httptest.Server, string) { + base := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body string + switch { + case strings.HasSuffix(r.URL.Path, "/.well-known/webfinger"): + body = `{"subject":"acct:alice@example.social","links":[` + + `{"rel":"self","type":"application/activity+json","href":"` + base + `/users/alice"},` + + `{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"` + base + `/@alice"}]}` + case r.URL.Path == "/users/alice": + body = `{"id":"` + base + `/users/alice","type":"Person","preferredUsername":"alice","url":"` + base + `/@alice","outbox":"` + base + `/users/alice/outbox"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "": + body = `{"type":"OrderedCollection","totalItems":5,"first":"` + base + `/users/alice/outbox?page=1"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "1": + body = `{"type":"OrderedCollectionPage","partOf":"` + base + `/users/alice/outbox","orderedItems":[` + + `{"type":"Create","published":"2024-03-01T00:00:00Z","object":{"type":"Note","id":"` + base + `/notes/1","published":"2024-03-01T00:00:00Z","content":"created note 1","url":"` + base + `/@alice/statuses/1"}},` + + `{"type":"Announce","object":"` + base + `/notes/9"},` + + `{"type":"Like","object":"` + base + `/notes/8"},` + + `{"type":"Create","object":"` + base + `/notes/2"},` + + `{"type":"Note","id":"` + base + `/notes/3","published":"2024-03-03T00:00:00Z","content":"bare note 3","url":["` + base + `/@alice/statuses/3"]}` + + `],"next":"` + base + `/users/alice/outbox?page=2"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "2": + body = `{"type":"OrderedCollectionPage","orderedItems":[` + + `{"type":"Create","object":{"type":"Note","id":"` + base + `/notes/4","published":"2024-01-15T00:00:00Z","content":"note 4"}}` + + `]}` + case r.URL.Path == "/notes/2": + body = `{"type":"Note","id":"` + base + `/notes/2","published":"2024-02-01T00:00:00Z","content":"deref note 2","url":"` + base + `/@alice/statuses/2"}` + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/activity+json") + _, _ = fmt.Fprint(w, body) + })) + base = server.URL + return server, base + "/users/alice" +} + +func TestOutboxGetLatest(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), actorURL, "alice@example.social", 10) + if err != nil { + t.Fatalf("getLatest error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected 4 Create/Note posts (skipping Announce/Like), got %d: %+v", len(posts), posts) + } + want := []string{"created note 1", "deref note 2", "bare note 3", "note 4"} + for i, w := range want { + if posts[i].Content != w { + t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) + } + } + if posts[0].Author != "alice@example.social" { + t.Fatalf("Author = %q, want handle", posts[0].Author) + } +} + +func TestOutboxGetLatestLimit(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), actorURL, "h", 2) + if err != nil { + t.Fatalf("getLatest error: %v", err) + } + if len(posts) != 2 { + t.Fatalf("expected 2 posts, got %d", len(posts)) + } + want := []string{"created note 1", "deref note 2"} + for i, w := range want { + if posts[i].Content != w { + t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) + } + } +} + +func TestOutboxGetInTimeframe(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + start := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) + posts, err := oc.getInTimeframe(context.Background(), actorURL, "h", start, end, 10) + if err != nil { + t.Fatalf("getInTimeframe error: %v", err) + } + if len(posts) != 2 { + t.Fatalf("expected 2 in-window posts (note 3 and 4 are outside), got %d", len(posts)) + } + if posts[0].Content != "created note 1" || posts[1].Content != "deref note 2" { + t.Fatalf("unexpected in-window posts: %+v", posts) + } +} + +func TestOutboxResolveViaWebfinger(t *testing.T) { + server, _ := newRemoteServer() + defer server.Close() + + scheme := webfingerScheme + webfingerScheme = "http" + defer func() { webfingerScheme = scheme }() + + handle := "@alice@" + strings.TrimPrefix(server.URL, "http://") + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), "", handle, 10) + if err != nil { + t.Fatalf("getLatest via webfinger error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected 4 posts via webfinger, got %d", len(posts)) + } +} + +func TestOutboxResolveNote(t *testing.T) { + server, _ := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + note, err := oc.resolveNote(context.Background(), server.URL+"/notes/2") + if err != nil { + t.Fatalf("resolveNote error: %v", err) + } + if note.Content != "deref note 2" { + t.Fatalf("unexpected note: %+v", note) + } +} + +func TestNoteToInternalPostSetsAuthor(t *testing.T) { + note := &apNote{ + ID: "https://example.social/notes/1", + Content: "hello", + Published: "2024-01-01T00:00:00Z", + } + p := noteToInternalPost(note, "alice@example.social") + if p.Account.Acct != "alice@example.social" { + t.Fatalf("expected Account.Acct = 'alice@example.social', got %q", p.Account.Acct) + } + if p.toPost().Author != "alice@example.social" { + t.Fatalf("expected toPost().Author = 'alice@example.social', got %q", p.toPost().Author) + } +} + +func TestNoteToInternalPostNoAuthor(t *testing.T) { + note := &apNote{ + ID: "https://example.social/notes/1", + Content: "hello", + Published: "2024-01-01T00:00:00Z", + } + p := noteToInternalPost(note, "") + if p.Account.Acct != "" { + t.Fatalf("expected empty Account.Acct when no author, got %q", p.Account.Acct) + } + if p.toPost().Author != "" { + t.Fatalf("expected empty author when none provided, got %q", p.toPost().Author) + } +} + +func TestMergePostsBySourceDedup(t *testing.T) { + created := time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC) + cached := []*internalPost{ + {ID: "local-1", URI: "https://example.social/notes/1", Content: "cached", CreatedAt: created}, + } + outbox := []*Post{ + {ID: "https://example.social/notes/1", Content: "cached dup", CreatedAt: created}, + {ID: "https://example.social/notes/2", Content: "outbox only", CreatedAt: created.Add(-time.Hour)}, + } + merged := mergePostsBySource(cached, outbox) + if len(merged) != 2 { + t.Fatalf("expected dedup to drop the duplicate, got %d", len(merged)) + } + if merged[0].Content != "cached" { + t.Fatalf("newest first: expected cached first, got %q", merged[0].Content) + } + if merged[1].Content != "outbox only" { + t.Fatalf("expected outbox only second, got %q", merged[1].Content) + } +} + +func TestParseActivityTime(t *testing.T) { + for _, s := range []string{"2024-01-01T00:00:00Z", "2024-01-01T00:00:00.000Z", "2024-01-01T00:00:00+02:00"} { + if _, ok := parseActivityTime(s); !ok { + t.Fatalf("parseActivityTime(%q) failed", s) + } + } + if _, ok := parseActivityTime("not a date"); ok { + t.Fatal("expected invalid timestamp to fail") + } +} + +func TestAPStringOrArray(t *testing.T) { + var sa apStringOrArray + if err := json.Unmarshal([]byte(`"https://a"`), &sa); err != nil || len(sa) != 1 || sa[0] != "https://a" { + t.Fatalf("string form failed: %v %v", sa, err) + } + if err := json.Unmarshal([]byte(`["https://a","https://b"]`), &sa); err != nil || len(sa) != 2 { + t.Fatalf("array form failed: %v %v", sa, err) + } + if err := json.Unmarshal([]byte(`null`), &sa); err != nil || len(sa) != 0 { + t.Fatalf("null form failed: %v %v", sa, err) + } +} diff --git a/internal/activitypub/throttle.go b/internal/activitypub/throttle.go new file mode 100644 index 0000000..19287d5 --- /dev/null +++ b/internal/activitypub/throttle.go @@ -0,0 +1,58 @@ +package activitypub + +import ( + "context" + "sync" + "time" +) + +type hostLimiter struct { + mu sync.Mutex + buckets map[string]*bucket +} + +type bucket struct { + mu sync.Mutex + last time.Time +} + +func newHostLimiter() *hostLimiter { + return &hostLimiter{buckets: make(map[string]*bucket)} +} + +func (h *hostLimiter) Wait(ctx context.Context, host string) error { + h.mu.Lock() + b, ok := h.buckets[host] + if !ok { + b = &bucket{} + h.buckets[host] = b + if len(h.buckets) > 1024 { + for k, v := range h.buckets { + if time.Since(v.last) > 10*time.Minute { + delete(h.buckets, k) + } + } + } + } + h.mu.Unlock() + + b.mu.Lock() + defer b.mu.Unlock() + + if b.last.IsZero() { + b.last = time.Now() + return nil + } + wait := time.Until(b.last.Add(time.Second)) + if wait <= 0 { + b.last = time.Now() + return nil + } + select { + case <-time.After(wait): + b.last = time.Now() + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/activitypub/types.go b/internal/activitypub/types.go index 269604d..1d9fc27 100644 --- a/internal/activitypub/types.go +++ b/internal/activitypub/types.go @@ -6,7 +6,6 @@ type Post struct { ID string `json:"id"` Author string `json:"author"` Content string `json:"content"` - Text string `json:"text"` CreatedAt time.Time `json:"created_at"` FavouritesCount int `json:"favourites_count"` ReblogsCount int `json:"reblogs_count"` @@ -14,16 +13,18 @@ type Post struct { } func (i *internalPost) toPost() *Post { - return &Post{ + p := &Post{ ID: i.ID, - Author: i.Account.Acct, Content: i.Content, - Text: i.Text, CreatedAt: i.CreatedAt, FavouritesCount: i.FavouritesCount, ReblogsCount: i.ReblogsCount, RepliesCount: i.RepliesCount, } + if i.Account.Acct != "" { + p.Author = i.Account.Acct + } + return p } type internalPost struct { @@ -57,7 +58,6 @@ type internalPost struct { Sensitive bool `json:"sensitive"` SpoilerText string `json:"spoiler_text"` Tags []Tag `json:"tags"` - Text string `json:"text"` URI string `json:"uri"` URL string `json:"url"` Visibility string `json:"visibility"` @@ -270,3 +270,10 @@ type Tag struct { Name string `json:"name"` URL string `json:"url"` } + +// searchResult mirrors the /api/v2/search response: statuses are the api +// `status` objects served by the search endpoint. +type searchResult struct { + Accounts []*Account `json:"accounts"` + Statuses []*internalPost `json:"statuses"` +} diff --git a/internal/bluesky/client.go b/internal/bluesky/client.go index 2ddc4bf..edb6359 100644 --- a/internal/bluesky/client.go +++ b/internal/bluesky/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "strings" + "sync" "time" "github.com/bluesky-social/indigo/api/atproto" @@ -16,8 +17,36 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) +const ( + // maxBskyFeedLimit is the upper bound the Bluesky API accepts for ?limit=. + // Higher values are silently clamped server-side. + maxBskyFeedLimit = 100 + + // maxAuthorFeedPages caps the number of author-feed pages GetPostsInTimeframe + // scans. At 50 posts/page this allows up to ~5,000 statuses per window before + // giving up. + maxAuthorFeedPages = 100 + + // didCacheTTL bounds how long a handle->DID resolution is reused. Handles + // almost never re-map, but the TTL guarantees a long-running process still + // notices handle changes instead of pinning a stale DID forever. + didCacheTTL = 24 * time.Hour +) + type BlueskyClient struct { - client *xrpc.Client + client *xrpc.Client + authMu sync.RWMutex + refreshMu sync.Mutex + cacheMu sync.Mutex + didCache map[string]didCacheEntry +} + +// didCacheEntry holds a positive (successfully resolved) handle->DID mapping +// and when it expires. Failures are never cached, so transient resolution +// errors are retried on the next call. +type didCacheEntry struct { + did string + expiresAt time.Time } func NewBlueskyClient(ctx context.Context, handle, password, host string) (*BlueskyClient, error) { @@ -39,37 +68,81 @@ func NewBlueskyClient(ctx context.Context, handle, password, host string) (*Blue Handle: session.Handle, Did: session.Did, } - return &BlueskyClient{client: client}, nil + return &BlueskyClient{ + client: client, + didCache: make(map[string]didCacheEntry), + }, nil } -// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the session is -// refreshed once and fn is retried. This makes long-running and background enrichment calls -// resilient to Bluesky access-token expiry without callers having to handle it. +// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the +// session is refreshed once and fn is retried. +// +// refreshMu serializes refresh calls: Bluesky refresh tokens are single-use and +// rotated, so two concurrent refreshes would race and one should fail for no +// reason. After acquiring the lock the error is re-checked, because a previous +// goroutine may already have refreshed the session while this one waited. func (c *BlueskyClient) withRefresh(ctx context.Context, fn func() error) error { - firstErr := fn() - if firstErr == nil { + if err := fn(); err == nil { + return nil + } else if !isAuthError(err) { + return err + } + + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + + // Another goroutine may have refreshed while we waited for the lock. + err := fn() + if err == nil { return nil } - if !isAuthError(firstErr) { - return firstErr + if !isAuthError(err) { + return err } + if rerr := c.refreshSession(ctx); rerr != nil { log.WarnWithContext(ctx, "bluesky session refresh failed", "error", rerr) - return firstErr + return err } return fn() } +// clientSnapshot returns a copy of the xrpc client that requests must use. It +// pins the current, immutable AuthInfo under the read lock: refreshSession +// replaces the Auth pointer under the exclusive lock rather than mutating the +// old value in place, so every goroutine sharing a pinned AuthInfo reads a +// complete, consistent token. xrpc.Client.Do reads client.Auth without any +// locking of its own, so no request may touch c.client.Auth directly. +func (c *BlueskyClient) clientSnapshot() *xrpc.Client { + c.authMu.RLock() + defer c.authMu.RUnlock() + cp := *c.client + return &cp +} + func (c *BlueskyClient) refreshSession(ctx context.Context) error { - refreshed, err := atproto.ServerRefreshSession(ctx, c.client) + refreshed, err := atproto.ServerRefreshSession(ctx, c.clientSnapshot()) if err != nil { return fmt.Errorf("failed to refresh bluesky session: %w", err) } - c.client.Auth.AccessJwt = refreshed.AccessJwt - c.client.Auth.RefreshJwt = refreshed.RefreshJwt + + c.authMu.Lock() + defer c.authMu.Unlock() + c.client.Auth = &xrpc.AuthInfo{ + AccessJwt: refreshed.AccessJwt, + RefreshJwt: refreshed.RefreshJwt, + Handle: c.client.Auth.Handle, + Did: c.client.Auth.Did, + } return nil } +// isAuthError reports whether err is an auth/expired-token XRPC error. The +// ErrStr switch covers the canonical cases; the message-substring fallback is a +// deliberate heuristic for third-party PDS instances that return nonstandard +// error strings. It can false-positive (an unrelated message that happens to +// contain "expired"/"unauthorized"/"token"), which only costs one gratuitous +// refresh + retry. func isAuthError(err error) bool { var xe *xrpc.XRPCError if !errors.As(err, &xe) { @@ -91,14 +164,18 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe return err } - var uri string + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) if collection != "" { uri = fmt.Sprintf("at://%s/%s/%s", repo.String(), collection, recordKey) - } else { - uri = fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) } + // collection and recordKey are caller-supplied; reject a malformed + // at-uri before spending a network round-trip. + if _, err := syntax.ParseATURI(uri); err != nil { + return fmt.Errorf("invalid at uri %q: %w", uri, err) + } + log.DebugWithContext(ctx, "Fetching post", "URI", uri) - resp, err := bsky.FeedGetPosts(ctx, c.client, []string{uri}) + resp, err := bsky.FeedGetPosts(ctx, c.clientSnapshot(), []string{uri}) if err != nil { return fmt.Errorf("failed to get post: %w", err) } @@ -112,13 +189,21 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe } func (c *BlueskyClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*bsky.FeedDefs_FeedViewPost, error) { + if limit <= 0 { + return nil, nil + } + pageSize := limit + if pageSize > maxBskyFeedLimit { + pageSize = maxBskyFeedLimit + } + var result []*bsky.FeedDefs_FeedViewPost err := c.withRefresh(ctx, func() error { repo, err := c.resolveUser(ctx, actor) if err != nil { return err } - resp, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), "", "posts", false, int64(limit)) + resp, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), "", "posts", false, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get latest posts: %w", err) } @@ -139,8 +224,11 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s var posts []*bsky.FeedDefs_FeedViewPost var cursor string - for len(posts) < limit { - feed, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), cursor, "posts", false, 50) + // The author feed is reverse-chronological, so the first post older + // than start terminates the scan; maxAuthorFeedPages is a belt-and- + // braces guard against a misbehaving server. + for pages := 0; pages < maxAuthorFeedPages; pages++ { + feed, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), cursor, "posts", false, 50) if err != nil { return err } @@ -161,12 +249,18 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s continue } - if createdAt.After(start) && createdAt.Before(end) { - posts = append(posts, post) - if len(posts) >= limit { - result = posts - return nil - } + if createdAt.After(end) { + continue + } + if createdAt.Before(start) { + // Reverse-chronological: nothing older is in-window. + result = posts + return nil + } + posts = append(posts, post) + if len(posts) >= limit { + result = posts + return nil } } @@ -186,9 +280,17 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) if err != nil { return nil, err } + pageSize := limit + if pageSize <= 0 { + pageSize = 50 + } + if pageSize > maxBskyFeedLimit { + pageSize = maxBskyFeedLimit + } + var result *bsky.FeedGetFeed_Output err = c.withRefresh(ctx, func() error { - resp, err := bsky.FeedGetFeed(ctx, c.client, "", uri, int64(50)) + resp, err := bsky.FeedGetFeed(ctx, c.clientSnapshot(), "", uri, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get feed: %w", err) } @@ -198,20 +300,33 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) return result, err } +// resolveUser converts a handle or DID into a syntax.DID. Handle resolutions +// are cached for didCacheTTL; DIDs pass through unmodified. Only successful +// resolutions are cached. func (c *BlueskyClient) resolveUser(ctx context.Context, user string) (syntax.DID, error) { - var did syntax.DID if d, err := syntax.ParseDID(user); err == nil { - did = d - } else { - resp, err := atproto.IdentityResolveHandle(ctx, c.client, user) - if err != nil { - return "", fmt.Errorf("failed to resolve handle to DID: %w", err) - } - did, err = syntax.ParseDID(resp.Did) - if err != nil { - return "", fmt.Errorf("failed to parse resolved DID: %w", err) - } + return d, nil } + + c.cacheMu.Lock() + if e, ok := c.didCache[user]; ok && time.Now().Before(e.expiresAt) { + c.cacheMu.Unlock() + return syntax.DID(e.did), nil + } + c.cacheMu.Unlock() + + resp, err := atproto.IdentityResolveHandle(ctx, c.clientSnapshot(), user) + if err != nil { + return "", fmt.Errorf("failed to resolve handle to DID: %w", err) + } + did, err := syntax.ParseDID(resp.Did) + if err != nil { + return "", fmt.Errorf("failed to parse resolved DID: %w", err) + } + + c.cacheMu.Lock() + c.didCache[user] = didCacheEntry{did: did.String(), expiresAt: time.Now().Add(didCacheTTL)} + c.cacheMu.Unlock() return did, nil } @@ -224,7 +339,7 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro err := c.withRefresh(ctx, func() error { params := map[string]any{"actor": author} var prof bskyActorProfile - if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { + if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { return fmt.Errorf("failed to get actor profile: %w", err) } result = &actor.Profile{ @@ -236,30 +351,30 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro FollowersCount: prof.FollowersCount, FollowingCount: prof.FollowsCount, PostCount: prof.PostsCount, - URL: prof.URL, + URL: blueskyProfileURL(prof.Handle), } return nil }) return result, err } -func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - var result []actor.Profile +func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { + var result []*actor.Profile err := c.withRefresh(ctx, func() error { params := map[string]any{"q": query, "limit": limit} var out bskyActorSearch - if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { + if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { return fmt.Errorf("failed to search actors: %w", err) } - results := make([]actor.Profile, 0, len(out.Actors)) + results := make([]*actor.Profile, 0, len(out.Actors)) for _, a := range out.Actors { - results = append(results, actor.Profile{ + results = append(results, &actor.Profile{ Platform: "Bluesky", Handle: a.Handle, DisplayName: a.DisplayName, AvatarURL: a.Avatar, Bio: a.Description, - URL: a.URL, + URL: blueskyProfileURL(a.Handle), }) } result = results @@ -268,6 +383,15 @@ func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit in return result, err } +// blueskyProfileURL derives the canonical public profile URL from the handle. +// app.bsky.actor.getProfile/searchActors do not return a url field. +func blueskyProfileURL(handle string) string { + if handle == "" { + return "" + } + return "https://bsky.app/profile/" + strings.TrimPrefix(handle, "@") +} + type bskyActorProfile struct { Did string `json:"did"` Handle string `json:"handle"` @@ -277,7 +401,6 @@ type bskyActorProfile struct { FollowersCount int `json:"followersCount"` FollowsCount int `json:"followsCount"` PostsCount int `json:"postsCount"` - URL string `json:"url"` } type bskyActorSearch struct { @@ -287,7 +410,6 @@ type bskyActorSearch struct { DisplayName string `json:"displayName"` Avatar string `json:"avatar"` Description string `json:"description"` - URL string `json:"url"` } `json:"actors"` } @@ -303,6 +425,10 @@ func (c *BlueskyClient) sanitizeFeedURI(ctx context.Context, input string) (stri return "", fmt.Errorf("invalid URI: %w", err) } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("unrecognized URI form (scheme must be http/https): %s", input) + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) >= 4 && parts[0] == "profile" && parts[2] == "feed" { didRaw := parts[1] diff --git a/internal/bluesky/client_test.go b/internal/bluesky/client_test.go index 73e9977..599edbb 100644 --- a/internal/bluesky/client_test.go +++ b/internal/bluesky/client_test.go @@ -7,6 +7,8 @@ import ( "net/http" "net/http/httptest" "strconv" + "sync" + "sync/atomic" "testing" "time" @@ -31,7 +33,7 @@ func setupMockServer() *httptest.Server { if input.Identifier == "fail.test" { w.WriteHeader(http.StatusUnauthorized) - fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) + _, _ = fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) return } @@ -43,7 +45,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:test", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetAuthorFeed @@ -81,7 +83,7 @@ func setupMockServer() *httptest.Server { Feed: feed, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetFeed @@ -107,7 +109,7 @@ func setupMockServer() *httptest.Server { }, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetPosts (used by GetPost) @@ -140,7 +142,7 @@ func setupMockServer() *httptest.Server { Posts: posts, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.getProfile (used by GetActor) @@ -160,10 +162,9 @@ func setupMockServer() *httptest.Server { "followersCount": 100, "followsCount": 50, "postsCount": 200, - "url": "https://bsky.app/profile/" + actor, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.searchActors (used by SearchActors) @@ -188,13 +189,12 @@ func setupMockServer() *httptest.Server { "displayName": fmt.Sprintf("Test User %d", i), "avatar": "https://example.com/avatar.jpg", "description": fmt.Sprintf("Bio for user %d", i), - "url": fmt.Sprintf("https://bsky.app/profile/%s%d.test", query, i), }) } resp := map[string]any{"actors": actors} w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.identity.resolveHandle (used by ResolveUser) @@ -209,7 +209,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:resolved", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.server.refreshSession (used by withRefresh) @@ -218,7 +218,7 @@ func setupMockServer() *httptest.Server { auth := r.Header.Get("Authorization") if auth == "" { w.WriteHeader(http.StatusUnauthorized) - json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) return } @@ -227,7 +227,7 @@ func setupMockServer() *httptest.Server { RefreshJwt: "refreshed_refresh_jwt", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) return httptest.NewServer(mux) @@ -292,6 +292,7 @@ func newTestClient(t *testing.T, serverURL string) *BlueskyClient { AccessJwt: "test_access_jwt", }, }, + didCache: make(map[string]didCacheEntry), } } @@ -302,7 +303,7 @@ func TestGetLatestNPosts(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() actor := "did:plc:testactor" - var limit int = 5 + limit := 5 output, err := client.GetLatestNPosts(ctx, actor, limit) if err != nil { @@ -329,7 +330,7 @@ func TestGetFeed(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() feedURI := "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.generator/with-friends" - var limit int = 10 + limit := 10 output, err := client.GetFeed(ctx, feedURI, limit) if err != nil { @@ -385,13 +386,14 @@ func TestGetPost(t *testing.T) { } func TestGetPostsInTimeframe(t *testing.T) { - // The mock author-feed returns two pages: the first contains out-of-range posts, - // then in-range posts, verifying both date filtering and cursor pagination. + // The mock author-feed is reverse-chronological: page 1 has posts newer than + // the window end (skipped, cursor advances), page 2 has the in-range posts + // followed by one older than start (which terminates the scan). requests := 0 mux := http.NewServeMux() mux.HandleFunc("/xrpc/com.atproto.server.createSession", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) + _ = json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) }) mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { requests++ @@ -413,22 +415,22 @@ func TestGetPostsInTimeframe(t *testing.T) { var feed []*bsky.FeedDefs_FeedViewPost var cursor string if requests == 1 { - // Page 1: one out-of-range (before start), one out-of-range (after end), and a cursor. + // Page 1: one post after the window end (skipped), with a cursor. feed = []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/4", "2024-01-01T00:00:00Z"), } cursor = "page-2-cursor" } else { - // Page 2: two in-range posts and no cursor (end of feed). + // Page 2: two in-range posts, then one older than start (terminates). feed = []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/2", "2023-08-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), } } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) }) server := httptest.NewServer(mux) @@ -493,6 +495,9 @@ func TestGetActor(t *testing.T) { if profile.Platform != "Bluesky" { t.Errorf("Expected platform 'Bluesky', got %q", profile.Platform) } + if profile.URL != "https://bsky.app/profile/"+actor { + t.Errorf("Expected URL %q, got %q", "https://bsky.app/profile/"+actor, profile.URL) + } if profile.FollowersCount != 100 || profile.FollowingCount != 50 || profile.PostCount != 200 { t.Errorf("Unexpected counts: followers=%d, following=%d, posts=%d", profile.FollowersCount, profile.FollowingCount, profile.PostCount) } @@ -529,6 +534,10 @@ func TestSearchActors(t *testing.T) { if result.Platform != "Bluesky" { t.Errorf("Result %d: expected platform 'Bluesky', got %q", i, result.Platform) } + expectedURL := "https://bsky.app/profile/" + expectedHandle + if result.URL != expectedURL { + t.Errorf("Result %d: expected URL %q, got %q", i, expectedURL, result.URL) + } } } @@ -583,6 +592,12 @@ func TestSanitizeFeedURI(t *testing.T) { expected: "at://did:plc:abc/app.bsky.feed.generator/xyz", wantErr: false, }, + { + name: "Non-HTTP scheme", + input: "gopher://bsky.app/profile/did:plc:abc/feed/xyz", + expected: "", + wantErr: true, + }, { name: "Invalid URI", input: "not-a-uri", @@ -609,3 +624,322 @@ func TestSanitizeFeedURI(t *testing.T) { }) } } + +func TestGetLatestNPostsLimitClamp(t *testing.T) { + server := setupMockServer() + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + actor := "did:plc:testactor" + + t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, 250) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output) != maxBskyFeedLimit { + t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output)) + } + }) + + t.Run("ZeroLimitReturnsNil", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, 0) + if err != nil { + t.Fatalf("Expected no error with zero limit, got %v", err) + } + if output != nil { + t.Errorf("Expected nil output for zero limit, got %d posts", len(output)) + } + }) + + t.Run("NegativeLimitReturnsNil", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, -5) + if err != nil { + t.Fatalf("Expected no error with negative limit, got %v", err) + } + if output != nil { + t.Errorf("Expected nil output for negative limit, got %d posts", len(output)) + } + }) +} + +func TestGetFeedHonorsLimit(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getFeed", func(w http.ResponseWriter, r *http.Request) { + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + feed := make([]*bsky.FeedDefs_FeedViewPost, 0, limit) + for i := 0; i < limit; i++ { + feed = append(feed, &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: fmt.Sprintf("at://did:plc:test/app.bsky.feed.post/%d", i), + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:test", Handle: "test.handle"}, + }, + }) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + feedURI := "at://did:plc:abc/app.bsky.feed.generator/xyz" + + t.Run("ExplicitLimitPassedThrough", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 75) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != 75 { + t.Errorf("Expected 75 posts, got %d", len(output.Feed)) + } + }) + + t.Run("ZeroLimitDefaultsTo50", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 0) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != 50 { + t.Errorf("Expected 50 posts (default), got %d", len(output.Feed)) + } + }) + + t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 1000) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != maxBskyFeedLimit { + t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output.Feed)) + } + }) +} + +func TestGetPostInvalidRecordKey(t *testing.T) { + requests := 0 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getPosts", func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, `{"error":"InternalServerError"}`, http.StatusInternalServerError) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + post, err := client.GetPost(ctx, "did:plc:testactor", "", "bad/key") + if post != nil { + t.Fatalf("Expected nil post, got %v", post) + } + if err == nil { + t.Fatal("Expected error for invalid record key, got nil") + } + if requests != 0 { + t.Errorf("Expected no network request for invalid record key, got %d", requests) + } +} + +func TestWithRefreshConcurrent(t *testing.T) { + var refreshCount atomic.Int32 + + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.actor.getProfile", func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth != "Bearer refreshed_access_jwt" { + // The initial access token is expired; the session must be refreshed + // before the retry succeeds. + w.WriteHeader(http.StatusUnauthorized) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "ExpiredToken", "message": "Token expired, refresh required"}) + return + } + resp := map[string]any{ + "did": "did:plc:test", + "handle": "test.handle", + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + mux.HandleFunc("/xrpc/com.atproto.server.refreshSession", func(w http.ResponseWriter, r *http.Request) { + refreshCount.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(atproto.ServerRefreshSession_Output{ + AccessJwt: "refreshed_access_jwt", + RefreshJwt: "refreshed_refresh_jwt", + }) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := client.GetActor(ctx, "test.handle"); err != nil { + t.Errorf("expected no error, got %v", err) + } + }() + } + wg.Wait() + + if refreshCount.Load() != 1 { + t.Errorf("expected exactly 1 session refresh under concurrency, got %d", refreshCount.Load()) + } +} + +func TestGetPostsInTimeframeStopsAtStart(t *testing.T) { + requests := 0 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { + requests++ + mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { + return &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: uri, + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, + Record: &util.LexiconTypeDecoder{ + Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, + }, + }, + } + } + feed := []*bsky.FeedDefs_FeedViewPost{ + mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/0", "2022-01-01T00:00:00Z"), + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) + + posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(posts) != 1 { + t.Fatalf("expected 1 in-range post before early exit, got %d", len(posts)) + } + if requests != 1 { + t.Errorf("expected scan to stop after the first page, made %d requests", requests) + } +} + +func TestGetPostsInTimeframeIncludesBoundaries(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { + mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { + return &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: uri, + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, + Record: &util.LexiconTypeDecoder{ + Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, + }, + }, + } + } + feed := []*bsky.FeedDefs_FeedViewPost{ + mk("at://did:plc:testactor/app.bsky.feed.post/end", "2023-12-31T23:59:59Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/start", "2023-01-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/older", "2022-12-31T23:59:59Z"), + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) + + posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + // Both boundary posts are inclusive; the post one second before start is not. + if len(posts) != 2 { + t.Fatalf("expected boundary posts to be included (got %d)", len(posts)) + } + got := map[string]bool{} + for _, p := range posts { + got[p.Post.Uri] = true + } + if !got["at://did:plc:testactor/app.bsky.feed.post/start"] { + t.Error("expected post exactly at start to be included") + } + if !got["at://did:plc:testactor/app.bsky.feed.post/end"] { + t.Error("expected post exactly at end to be included") + } +} + +func TestResolveUserCache(t *testing.T) { + var resolveRequests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/com.atproto.identity.resolveHandle", func(w http.ResponseWriter, r *http.Request) { + resolveRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(atproto.IdentityResolveHandle_Output{Did: "did:plc:resolved"}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + // First resolution hits the network and populates the cache. + did, err := client.ResolveUser(ctx, "test.handle") + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if did.String() != "did:plc:resolved" { + t.Errorf("Unexpected DID %q", did.String()) + } + if resolveRequests.Load() != 1 { + t.Errorf("expected 1 resolve request, got %d", resolveRequests.Load()) + } + + // Second resolution is served from cache. + if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { + t.Fatalf("Expected no error on cached resolution, got %v", err) + } + if resolveRequests.Load() != 1 { + t.Errorf("expected cached resolution to skip the network, got %d requests", resolveRequests.Load()) + } + + // An expired cache entry is refetched. + client.cacheMu.Lock() + entry := client.didCache["test.handle"] + entry.expiresAt = time.Now().Add(-time.Minute) + client.didCache["test.handle"] = entry + client.cacheMu.Unlock() + + if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { + t.Fatalf("Expected no error after expiry, got %v", err) + } + if resolveRequests.Load() != 2 { + t.Errorf("expected a fresh resolve after TTL expiry, got %d requests", resolveRequests.Load()) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index afe01b9..1c5ab7f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { OllamaHost string MigrateOnStart bool MigrationsDir string + ReadyzAddress string ReadyzPort string MetricsPort string X402FacilitatorURL string @@ -86,13 +87,14 @@ func Load() *Config { BlueskyAppPassword: getEnv("BLUESKY_APP_PASSWORD", ""), BlueskyHost: getEnv("BLUESKY_HOST", "https://bsky.social"), NostrRelays: splitComma(getEnv("NOSTR_RELAYS", "wss://relay.damus.io,wss://nos.lol")), - TwitterBearerToken: getEnv("TWITTER_BEARER_TOKEN", ""), - PrewarmAuthors: splitComma(getEnv("PREWARM_AUTHORS", "")), - PrewarmFeedURI: getEnv("PREWARM_FEED_URI", ""), + TwitterBearerToken: getEnvOptional("TWITTER_BEARER_TOKEN"), + PrewarmAuthors: splitComma(getEnvOptional("PREWARM_AUTHORS")), + PrewarmFeedURI: getEnvOptional("PREWARM_FEED_URI"), EnrichCrossPlatform: getEnvBool("ENRICH_CROSS_PLATFORM", "true"), OllamaHost: getEnv("OLLAMA_HOST", "http://localhost:11434"), MigrateOnStart: getEnvBool("MIGRATE_ON_START", "true"), MigrationsDir: getEnv("MIGRATIONS_DIR", "sql/schema"), + ReadyzAddress: getEnv("READYZ_ADDRESS", "127.0.0.1"), ReadyzPort: getEnv("READYZ_PORT", "9090"), MetricsPort: getEnv("METRICS_PORT", "9091"), X402FacilitatorURL: getEnv("X402_FACILITATOR_URL", "https://api.cdp.coinbase.com/platform/v2/x402"), @@ -112,6 +114,12 @@ func Load() *Config { AnalyticsTTLDrivers: getEnvDuration("ANALYTICS_TTL_DRIVERS", "1h"), } + for _, port := range []string{cfg.ReadyzPort, cfg.MetricsPort, cfg.ListenPort} { + if _, err := strconv.Atoi(port); err != nil { + log.Fatalf("Invalid port %q: must be a numeric port value", port) + } + } + // Default MatrixServerID to system hostname if not explicitly set if cfg.MatrixServerID == "" { hostname, err := os.Hostname() @@ -133,6 +141,14 @@ func getEnv(key, fallback string) string { return fallback } +func getEnvOptional(key string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + log.Warn("Environment variable is not set (optional)", "key", key) + return "" +} + func getEnvBool(key, fallback string) bool { v := getEnv(key, fallback) switch strings.ToLower(strings.TrimSpace(v)) { diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 5c22907..952045d 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "net/url" + "strings" "time" "fiatjaf.com/nostr" @@ -12,15 +14,44 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) +const ( + // relayQueryTimeout bounds connect + subscribe + collect against a single + // relay. The websocket dial, subscription handshake and event collection all + // share this budget. + relayQueryTimeout = 5 * time.Second + + // maxTotalRelayTime caps the fan-out across all relays when the caller's + // context carries no deadline of its own. Without it a long relay list could + // multiply the per-relay budget unboundedly. + maxTotalRelayTime = 20 * time.Second +) + type Client struct { relays []string } func NewNostrClient(relays []string) (*Client, error) { - if len(relays) == 0 { + seen := make(map[string]struct{}, len(relays)) + cleaned := make([]string, 0, len(relays)) + for _, r := range relays { + r = strings.TrimSpace(r) + if r == "" { + continue + } + u, err := url.Parse(r) + if err != nil || (u.Scheme != "ws" && u.Scheme != "wss") { + return nil, fmt.Errorf("invalid relay URL %q: must be a ws:// or wss:// endpoint", r) + } + if _, dup := seen[r]; dup { + continue + } + seen[r] = struct{}{} + cleaned = append(cleaned, r) + } + if len(cleaned) == 0 { return nil, fmt.Errorf("at least one relay is required") } - return &Client{relays: relays}, nil + return &Client{relays: cleaned}, nil } type Post struct { @@ -30,145 +61,92 @@ type Post struct { CreatedAt time.Time } -func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { - pubkey, err := resolvePubkey(actor) - if err != nil { - return nil, err +// queryBudget returns a context bounded to maxTotalRelayTime when ctx has no +// deadline, so the relay fan-out cannot run away across many relays. Callers +// must invoke the returned cancel. +func (c *Client) queryBudget(ctx context.Context) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return context.WithCancel(ctx) } - - posts := make([]*Post, 0, limit) - seen := make(map[string]struct{}) - - connected := false - - for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue - } - connected = true - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - - func() { - defer sub.Unsub() - defer relay.Close() - - collect: - for { - select { - case <-sub.EndOfStoredEvents: - break collect - case ev, ok := <-sub.Events: - if !ok { - break collect - } - id := ev.ID.Hex() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - posts = append(posts, &Post{ - ID: id, - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }) - case <-ctx.Done(): - break collect - } - } - }() - - if len(posts) >= limit { - break - } - } - - if !connected { - return nil, fmt.Errorf("could not connect to any relay") - } - return posts, nil + return context.WithTimeout(ctx, maxTotalRelayTime) } -func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { - pubkey, err := resolvePubkey(actor) +// collectFromRelay connects to url within ctx, subscribes with filter and +// appends matching events to posts (deduplicated by event id) until the limit is +// reached, EOSE arrives, or ctx is done. It reports whether the relay was +// actually queryable (connected + subscribed), letting callers distinguish a +// total relay outage from a legitimate empty result. +func (c *Client) collectFromRelay(ctx context.Context, url string, filter nostr.Filter, limit int, posts *[]*Post, seen map[string]struct{}) (used bool) { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) if err != nil { - return nil, err + return false } + defer relay.Close() - since := nostr.Timestamp(start.Unix()) - until := nostr.Timestamp(end.Unix()) + sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) + if err != nil { + return false + } + defer sub.Unsub() + for { + select { + case <-sub.EndOfStoredEvents: + return true + case ev, ok := <-sub.Events: + if !ok { + return true + } + id := ev.ID.Hex() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + *posts = append(*posts, &Post{ + ID: id, + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }) + if len(*posts) >= limit { + return true + } + case <-ctx.Done(): + return true + } + } +} + +// collectPosts subscribes to filter across every relay, stopping once limit posts +// are collected or the query context is done. It returns the deduplicated posts, +// or an error if no relay could be queried at all or the operation was aborted. +func (c *Client) collectPosts(ctx context.Context, filter nostr.Filter, limit int) ([]*Post, error) { posts := make([]*Post, 0, limit) seen := make(map[string]struct{}) - connected := false + queryCtx, cancel := c.queryBudget(ctx) + defer cancel() + var anyUsed bool for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue + if queryCtx.Err() != nil { + break } - connected = true - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Since: since, - Until: until, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - - func() { - defer sub.Unsub() - defer relay.Close() - - collect: - for { - select { - case <-sub.EndOfStoredEvents: - break collect - case ev, ok := <-sub.Events: - if !ok { - break collect - } - id := ev.ID.Hex() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - posts = append(posts, &Post{ - ID: id, - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }) - case <-ctx.Done(): - break collect - } - } - }() - + relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) + used := c.collectFromRelay(relayCtx, url, filter, limit, &posts, seen) + relayCancel() + anyUsed = anyUsed || used if len(posts) >= limit { break } } - if !connected { + if err := queryCtx.Err(); err != nil { + return nil, fmt.Errorf("nostr query aborted: %w", err) + } + if !anyUsed { return nil, fmt.Errorf("could not connect to any relay") } - return posts, nil } @@ -178,42 +156,54 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { return nil, err } - for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue - } - sub, err := relay.Subscribe(ctx, nostr.Filter{ - IDs: []nostr.ID{eid}, - Kinds: []nostr.Kind{1}, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - defer sub.Unsub() - defer relay.Close() - - for { - select { - case <-sub.EndOfStoredEvents: - return nil, fmt.Errorf("post %q not found", id) - case ev, ok := <-sub.Events: - if !ok { - return nil, fmt.Errorf("post %q not found", id) - } - return &Post{ - ID: ev.ID.Hex(), - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }, nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } + filter := nostr.Filter{ + IDs: []nostr.ID{eid}, + Kinds: []nostr.Kind{1}, } - return nil, fmt.Errorf("could not connect to any relay") + posts, err := c.collectPosts(ctx, filter, 1) + if err != nil { + return nil, err + } + if len(posts) == 0 { + return nil, fmt.Errorf("post %q not found across configured relays", id) + } + return posts[0], nil +} + +func (c *Client) GetLatestNPosts(ctx context.Context, author string, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + pubkey, err := resolvePubkey(author) + if err != nil { + return nil, err + } + + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Limit: limit, + } + return c.collectPosts(ctx, filter, limit) +} + +func (c *Client) GetPostsInTimeframe(ctx context.Context, author string, start, end time.Time, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + pubkey, err := resolvePubkey(author) + if err != nil { + return nil, err + } + + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Since: nostr.Timestamp(start.Unix()), + Until: nostr.Timestamp(end.Unix()), + Limit: limit, + } + return c.collectPosts(ctx, filter, limit) } func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -223,83 +213,128 @@ func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, e } npub := nip19.EncodeNpub(pubkey) - for _, u := range c.relays { - relay, err := nostr.RelayConnect(ctx, u, nostr.RelayOptions{}) - if err != nil { - continue - } - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{0}, - Limit: 1, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - defer sub.Unsub() - defer relay.Close() + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{0}, + Limit: 1, + } - collect: - for { - select { - case <-sub.EndOfStoredEvents: - break collect - case ev, ok := <-sub.Events: - if !ok { - break collect - } - var meta struct { - Name string `json:"name"` - DisplayName string `json:"display_name"` - About string `json:"about"` - Picture string `json:"picture"` - Website string `json:"website"` - } - if err := json.Unmarshal([]byte(ev.Content), &meta); err == nil { - return &actor.Profile{ - Platform: "Nostr", - Handle: npub, - DisplayName: meta.DisplayName, - AvatarURL: meta.Picture, - Bio: meta.About, - URL: "https://njump.me/" + npub, - }, nil - } - break collect - case <-ctx.Done(): - return nil, ctx.Err() - } + queryCtx, cancel := c.queryBudget(ctx) + defer cancel() + + var anyUsed bool + for _, url := range c.relays { + if queryCtx.Err() != nil { + break } + relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) + profile, used := c.actorFromRelay(relayCtx, url, filter, npub) + relayCancel() + anyUsed = anyUsed || used + if profile != nil { + return profile, nil + } + } + + if err := queryCtx.Err(); err != nil { + return nil, fmt.Errorf("nostr query aborted: %w", err) + } + if !anyUsed { + return nil, fmt.Errorf("could not connect to any relay") } return nil, fmt.Errorf("no profile metadata found for %q", author) } -func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - // Nostr has no global actor search; best-effort: if the query resolves to a - // pubkey/npub, return that single profile. +// actorFromRelay looks up one kind-0 metadata event on url. It returns a +// non-nil profile when a usable metadata event was found. used reports whether +// the relay was queried at all (so a fully unreachable relay list is +// distinguishable from an author without metadata). +func (c *Client) actorFromRelay(ctx context.Context, url string, filter nostr.Filter, npub string) (profile *actor.Profile, used bool) { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + return nil, false + } + defer relay.Close() + + sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) + if err != nil { + return nil, false + } + defer sub.Unsub() + + select { + case <-sub.EndOfStoredEvents: + return nil, true + case ev, ok := <-sub.Events: + if !ok { + return nil, true + } + return profileFromEvent(ev, npub), true + case <-ctx.Done(): + return nil, true + } +} + +// profileFromEvent builds an actor.Profile from a kind-0 metadata event, or nil +// when the content is not valid JSON or carries no usable fields. +func profileFromEvent(ev nostr.Event, npub string) *actor.Profile { + var meta struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + About string `json:"about"` + Picture string `json:"picture"` + Website string `json:"website"` + } + if err := json.Unmarshal([]byte(ev.Content), &meta); err != nil { + return nil + } + if meta.Name == "" && meta.DisplayName == "" && meta.About == "" && meta.Picture == "" { + return nil + } + displayName := meta.DisplayName + if displayName == "" { + displayName = meta.Name + } + return &actor.Profile{ + Platform: "Nostr", + Handle: npub, + DisplayName: displayName, + AvatarURL: meta.Picture, + Bio: meta.About, + URL: "https://njump.me/" + npub, + } +} + +func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { if _, err := resolvePubkey(query); err != nil { - return []actor.Profile{}, nil + // Not a pubkey we can resolve (e.g. a name query). Treat it as no match + // rather than an error so a cross-platform search isn't failed by a + // non-nostr query. + return nil, nil } prof, err := c.GetActor(ctx, query) if err != nil { - return []actor.Profile{}, nil + return nil, err } - return []actor.Profile{*prof}, nil + return []*actor.Profile{prof}, nil } func resolvePubkey(actor string) (nostr.PubKey, error) { prefix, value, err := nip19.Decode(actor) if err == nil { - if prefix != "npub" { - return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) + switch prefix { + case "npub": + if pk, ok := value.(nostr.PubKey); ok { + return pk, nil + } + case "nprofile": + if pp, ok := value.(nostr.ProfilePointer); ok { + return pp.PublicKey, nil + } } - pk, ok := value.(nostr.PubKey) - if !ok { - return nostr.PubKey{}, fmt.Errorf("unexpected decoded value type %T", value) - } - return pk, nil + return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) } + if len(actor) == 64 { pk, e := nostr.PubKeyFromHex(actor) if e != nil { @@ -313,15 +348,19 @@ func resolvePubkey(actor string) (nostr.PubKey, error) { func resolveEventID(id string) (nostr.ID, error) { prefix, value, err := nip19.Decode(id) if err == nil { - if prefix != "note" { - return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) + switch prefix { + case "note": + if eid, ok := value.(nostr.ID); ok { + return eid, nil + } + case "nevent": + if ep, ok := value.(nostr.EventPointer); ok { + return ep.ID, nil + } } - eid, ok := value.(nostr.ID) - if !ok { - return nostr.ID{}, fmt.Errorf("unexpected decoded value type %T", value) - } - return eid, nil + return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) } + if len(id) == 64 { eid, e := nostr.IDFromHex(id) if e != nil { diff --git a/internal/nostr/client_test.go b/internal/nostr/client_test.go index 0980361..e65bbb1 100644 --- a/internal/nostr/client_test.go +++ b/internal/nostr/client_test.go @@ -189,6 +189,80 @@ func TestRelayConnectionFailureReturnsError(t *testing.T) { } } +func TestNewNostrClientInvalidScheme(t *testing.T) { + _, err := NewNostrClient([]string{"http://relay.example"}) + if err == nil { + t.Fatal("expected error for non-websocket relay URL") + } + if !strings.Contains(err.Error(), "invalid relay URL") { + t.Fatalf("expected 'invalid relay URL' error, got %v", err) + } + + // Blank and duplicate entries are dropped, the scheme is validated. + c, err := NewNostrClient([]string{ + "wss://relay.a", + " wss://relay.a ", + "ws://relay.b", + " ", + }) + if err != nil { + t.Fatal(err) + } + if len(c.relays) != 2 { + t.Fatalf("expected 2 unique relays after cleaning, got %d: %v", len(c.relays), c.relays) + } +} + +func TestGetLatestNPostsZeroOrNegativeLimit(t *testing.T) { + c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) + if err != nil { + t.Fatal(err) + } + + sk := nostr.Generate() + npub := nip19.EncodeNpub(sk.Public()) + + if posts, err := c.GetLatestNPosts(context.Background(), npub, 0); err != nil || posts != nil { + t.Fatalf("expected nil, nil for zero limit, got %v, %v", posts, err) + } + if posts, err := c.GetLatestNPosts(context.Background(), npub, -5); err != nil || posts != nil { + t.Fatalf("expected nil, nil for negative limit, got %v, %v", posts, err) + } + + start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC) + if posts, err := c.GetPostsInTimeframe(context.Background(), npub, start, end, 0); err != nil || posts != nil { + t.Fatalf("expected nil, nil for zero timeframe limit, got %v, %v", posts, err) + } +} + +func TestSearchActors(t *testing.T) { + c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + + // A non-key query (e.g. a display name) is a valid no-match, not an error. + results, err := c.SearchActors(ctx, "alice", 1) + if err != nil { + t.Fatalf("expected no error for non-key query, got %v", err) + } + if results != nil { + t.Fatalf("expected nil results for non-key query, got %v", results) + } + + // A valid pubkey that fails on every relay must surface the outage. + sk := nostr.Generate() + npub := nip19.EncodeNpub(sk.Public()) + if _, err := c.SearchActors(ctx, npub, 1); err == nil { + t.Fatal("expected error when no relay is reachable") + } else if !strings.Contains(err.Error(), "could not connect to any relay") { + t.Fatalf("expected 'could not connect to any relay' error, got %v", err) + } +} + func TestGetPostWithHexID(t *testing.T) { c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) if err != nil { @@ -223,9 +297,9 @@ func TestGetActorWithHexPubkey(t *testing.T) { if err == nil { t.Fatal("expected error when relay unavailable") } - // GetActor surfaces the failure as an absence of a profile after the relay - // loop runs without connecting. - if !strings.Contains(err.Error(), "no profile metadata found") { - t.Fatalf("expected 'no profile metadata found' error, got %v", err) + // GetActor reports a total relay outage as a connection failure rather than + // pretending the author has no metadata. + if !strings.Contains(err.Error(), "could not connect to any relay") { + t.Fatalf("expected 'could not connect to any relay' error, got %v", err) } } diff --git a/internal/ollama/client.go b/internal/ollama/client.go index 69df20c..60515ae 100644 --- a/internal/ollama/client.go +++ b/internal/ollama/client.go @@ -2,52 +2,150 @@ package ollama import ( "context" + "errors" + "fmt" "net/http" "net/url" "strings" + "sync" + "time" "github.com/nacorid/naco-api/internal/clustering" "github.com/ollama/ollama/api" ) +const ( + // embedConcurrency caps how many embedding requests run in parallel. Ollama + // serializes model requests server-side (OLLAMA_NUM_PARALLEL defaults to 1), + // so unbounded fan-out would only add latency and memory for no throughput. + embedConcurrency = 4 + + // embedRetries and embedRetryBackoff cover transient server-side embedding + // failures (e.g. cold model load, transient 5xx/429). + embedRetries = 2 + embedRetryBackoff = 150 * time.Millisecond + + // generateRetries and generateRetryBackoff cover transient failures that + // produce no output at all; a truncated stream is never retried because the + // partial response cannot be safely replayed. + generateRetries = 1 + generateRetryBackoff = 500 * time.Millisecond +) + type OllamaClient struct { client api.Client embeddingModel string styleModel string } -func New(ctx context.Context, host *url.URL, http *http.Client, em, sm string) *OllamaClient { - client := api.NewClient(host, http) - return &OllamaClient{client: *client, embeddingModel: em, styleModel: sm} +func New(host *url.URL, httpClient *http.Client, embeddingModel, styleModel string) *OllamaClient { + client := api.NewClient(host, httpClient) + return &OllamaClient{client: *client, embeddingModel: embeddingModel, styleModel: styleModel} } +// EmbedTexts computes one embedding per post. Embeddings are returned in the +// same order as posts; callers rely on index correspondence (e.g. +// items[i].Embedding = embeddings[i]) so posts are never skipped. Requests run +// in a bounded worker pool and transient failures are retried with backoff. func (o *OllamaClient) EmbedTexts(ctx context.Context, posts []clustering.Item) ([][]float64, error) { if len(posts) == 0 { return nil, nil } - embeddings := make([][]float64, 0, len(posts)) - for _, post := range posts { + embeddings := make([][]float64, len(posts)) + + // poolCtx is a child of ctx: cancelling it on the first failure must not + // cancel the caller's request context. + poolCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + mu sync.Mutex + firstErr error + sema = make(chan struct{}, embedConcurrency) + wg sync.WaitGroup + ) + for i, post := range posts { + wg.Add(1) + go func(idx int, text string) { + defer wg.Done() + select { + case sema <- struct{}{}: + case <-poolCtx.Done(): + return + } + defer func() { <-sema }() + + emb, err := o.embedWithRetry(poolCtx, strings.TrimSpace(text)) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("embedding post %d (%q): %w", idx, text, err) + } + mu.Unlock() + cancel() + return + } + embeddings[idx] = emb + }(i, post.Text) + } + wg.Wait() + + if firstErr := atomicErr(&mu, &firstErr); firstErr != nil { + return embeddings, firstErr + } + // Every worker may have exited via the poolCtx.Done() branch without + // recording an error, e.g. when the caller's context is already canceled. + if err := ctx.Err(); err != nil { + return nil, err + } + return embeddings, nil +} + +func atomicErr(mu *sync.Mutex, err *error) error { + mu.Lock() + defer mu.Unlock() + return *err +} + +func (o *OllamaClient) embedWithRetry(ctx context.Context, text string) ([]float64, error) { + backoff := embedRetryBackoff + var lastErr error + for attempt := 0; attempt <= embedRetries; attempt++ { + if attempt > 0 { + if err := sleepCtx(ctx, backoff); err != nil { + return nil, err + } + backoff *= 2 + } resp, err := o.client.Embeddings(ctx, &api.EmbeddingRequest{ Model: o.embeddingModel, - Prompt: post.Text, + Prompt: text, }) - if err != nil { + if err == nil { + return resp.Embedding, nil + } + lastErr = err + if !isTransientErr(err) { return nil, err } - embeddings = append(embeddings, resp.Embedding) } - - return embeddings, nil + return nil, lastErr } func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.Item) (string, error) { if len(posts) == 0 { return "", nil } - var promptParts []string + + promptParts := make([]string, 0, len(posts)) for _, post := range posts { - promptParts = append(promptParts, post.Text) + if text := strings.TrimSpace(post.Text); text != "" { + promptParts = append(promptParts, text) + } + } + if len(promptParts) == 0 { + return "", fmt.Errorf("no post text to summarize") } prompt := "Summarize the user's writing style. Analyze:\n" + @@ -62,16 +160,74 @@ func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.It "\n\nStyle summary:" var fullResponse strings.Builder - err := o.client.Generate(ctx, &api.GenerateRequest{ - Model: o.styleModel, - Prompt: prompt, - }, func(resp api.GenerateResponse) error { - fullResponse.WriteString(resp.Response) - return nil - }) - if err != nil { - return "", err - } + backoff := generateRetryBackoff + var lastErr error + for attempt := 0; attempt <= generateRetries; attempt++ { + if attempt > 0 { + if err := sleepCtx(ctx, backoff); err != nil { + return "", err + } + } - return fullResponse.String(), nil + // Reset between attempts so a failed run never leaks a partial summary. + fullResponse.Reset() + done := false + doneReason := "" + err := o.client.Generate(ctx, &api.GenerateRequest{ + Model: o.styleModel, + Prompt: prompt, + }, func(resp api.GenerateResponse) error { + fullResponse.WriteString(resp.Response) + if resp.Done { + done = true + doneReason = resp.DoneReason + } + return nil + }) + if err != nil { + lastErr = err + // Only retry transport/5xx failures that produced no output; a + // partial stream must not be replayed. + if !isTransientErr(err) || fullResponse.Len() > 0 { + return "", fmt.Errorf("ollama generate with model %q failed: %w", o.styleModel, err) + } + continue + } + + // The underlying stream reader can finish without an error when the + // request context is canceled or the connection drops mid-stream (the + // library never inspects bufio.Scanner.Err), so a bare nil from + // Generate does not mean the response is complete. + if ctxErr := ctx.Err(); ctxErr != nil { + return "", ctxErr + } + if !done { + return "", fmt.Errorf("ollama stream for model %q ended before completion", o.styleModel) + } + if doneReason == "length" { + return "", fmt.Errorf("ollama response for model %q was truncated (context limit reached)", o.styleModel) + } + return fullResponse.String(), nil + } + return "", fmt.Errorf("ollama generate with model %q failed after retries: %w", o.styleModel, lastErr) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-time.After(d): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// isTransientErr reports whether err is a retryable server-side failure. Non-2xx +// HTTP errors that carry a StatusCode (429, 5xx) qualify; context and transport +// errors below that never do, so they surface immediately. +func isTransientErr(err error) bool { + var se api.StatusError + if errors.As(err, &se) { + return se.StatusCode == http.StatusTooManyRequests || se.StatusCode >= http.StatusInternalServerError + } + return false } diff --git a/internal/ollama/client_test.go b/internal/ollama/client_test.go index 16b18f3..9ade0e3 100644 --- a/internal/ollama/client_test.go +++ b/internal/ollama/client_test.go @@ -2,27 +2,89 @@ package ollama import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" "net/url" + "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/nacorid/naco-api/internal/clustering" ) +// setupTestServer creates a test server and an initialized client. +func setupTestServer(t *testing.T, handler http.Handler) (*httptest.Server, *OllamaClient) { + t.Helper() + server := httptest.NewServer(handler) + + host, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server URL: %v", err) + } + + client := New(host, http.DefaultClient, "test-embed", "test-style") + return server, client +} + +// ============================================================================ +// EmbedTexts Tests +// ============================================================================ + func TestEmbedTexts(t *testing.T) { + type embedRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + } + + var mu sync.Mutex + var receivedRequests []embedRequest + mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + // 1. Validate HTTP Method + if r.Method != http.MethodPost { + t.Errorf("expected method POST, got %s", r.Method) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // 2. Validate Headers + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) + http.Error(w, "invalid content-type", http.StatusBadRequest) + return + } + + // 3. Validate Request Payload + var req embedRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + if req.Model != "test-embed" { + t.Errorf("expected model 'test-embed', got %q", req.Model) + } + + mu.Lock() + receivedRequests = append(receivedRequests, req) + mu.Unlock() + + // 4. Return valid response w.Header().Set("Content-Type", "application/json") - w.Write([]byte(`{"embedding":[0.1,0.2,0.3]}`)) + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{0.1, 0.2, 0.3}, + }) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - posts := []clustering.Item{ {ID: "1", Text: "hello", Embedding: nil}, {ID: "2", Text: "world", Embedding: nil}, @@ -33,177 +95,291 @@ func TestEmbedTexts(t *testing.T) { t.Fatalf("EmbedTexts failed: %v", err) } + // Verify outgoing requests: ensure both prompts were sent + mu.Lock() + if len(receivedRequests) != 2 { + t.Errorf("expected 2 HTTP requests, got %d", len(receivedRequests)) + } + promptsReceived := map[string]bool{} + for _, req := range receivedRequests { + promptsReceived[req.Prompt] = true + } + mu.Unlock() + + if !promptsReceived["hello"] || !promptsReceived["world"] { + t.Errorf("expected prompts 'hello' and 'world' to be sent, got: %+v", receivedRequests) + } + + // Verify parsed results if len(embeddings) != 2 { t.Fatalf("expected 2 embeddings, got %d", len(embeddings)) } - for i, emb := range embeddings { if len(emb) != 3 { - t.Errorf("embedding %d has wrong dim: %d", i, len(emb)) + t.Errorf("embedding %d has wrong dimension: %d", i, len(emb)) } if emb[0] != 0.1 || emb[1] != 0.2 || emb[2] != 0.3 { - t.Errorf("embedding %d wrong values: %v", i, emb) + t.Errorf("embedding %d has wrong values: %v", i, emb) } } } -func TestEmbedTexts_Error(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} - - _, err := client.EmbedTexts(context.Background(), posts) - if err == nil { - t.Fatal("expected error from failing server") - } -} - -func TestSummarizeStyle(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, ok := w.(http.Flusher) - if !ok { - t.Fatal("response writer doesn't support flushing") - } - w.Header().Set("Content-Type", "application/x-ndjson") - // Stream response as newline-delimited JSON - w.Write([]byte(`{"response":"Style summary: "}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"concise and clear."}` + "\n")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{ - {ID: "1", Text: "First post", Embedding: nil}, - {ID: "2", Text: "Second post", Embedding: nil}, - } - - summary, err := client.SummarizeStyle(context.Background(), posts) - if err != nil { - t.Fatalf("SummarizeStyle failed: %v", err) - } - - if summary != "Style summary: concise and clear." { - t.Errorf("unexpected summary: %q", summary) - } -} - -func TestSummarizeStyle_Error(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error from failing server") - } -} - -func TestSummarizeStyle_Streaming(t *testing.T) { - callCount := 0 - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - callCount++ - flusher, ok := w.(http.Flusher) - if !ok { - t.Fatal("response writer doesn't support flushing") - } - w.Header().Set("Content-Type", "application/x-ndjson") - // Simulate streaming response - w.Write([]byte(`{"response":"part1"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"part2"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"part3","done":true}` + "\n")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "test", Embedding: nil}} - - summary, err := client.SummarizeStyle(context.Background(), posts) - if err != nil { - t.Fatalf("SummarizeStyle failed: %v", err) - } - - if summary != "part1part2part3" { - t.Errorf("streaming summary = %q, want %q", summary, "part1part2part3") - } - - if callCount != 1 { - t.Errorf("generate endpoint called %d times, want 1", callCount) - } -} - func TestEmbedTexts_EmptyInput(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - t.Fatal("embeddings endpoint should not be called for empty input") + t.Error("embeddings endpoint should not be called when input is empty") + http.Error(w, "should not be called", http.StatusInternalServerError) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - embeddings, err := client.EmbedTexts(context.Background(), []clustering.Item{}) if err != nil { t.Fatalf("EmbedTexts with empty input failed: %v", err) } if len(embeddings) != 0 { - t.Errorf("expected empty embeddings, got %d", len(embeddings)) + t.Errorf("expected 0 embeddings, got %d", len(embeddings)) + } +} + +func TestEmbedTexts_Error(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error": "model test-embed not found"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.EmbedTexts(context.Background(), posts) + if err == nil { + t.Fatal("expected error from failing server, got nil") + } +} + +func TestEmbedTexts_ContextCanceled(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + // Hang until client cancels the request + <-r.Context().Done() + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + _, err := client.EmbedTexts(ctx, posts) + if err == nil { + t.Fatal("expected error due to canceled context, got nil") + } +} + +func TestEmbedTexts_RetryThenSuccess(t *testing.T) { + var requests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{9.9}, + }) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + embeddings, err := client.EmbedTexts(context.Background(), posts) + if err != nil { + t.Fatalf("EmbedTexts failed after transient error: %v", err) + } + if len(embeddings) != 1 || len(embeddings[0]) != 1 || embeddings[0][0] != 9.9 { + t.Fatalf("unexpected embeddings after retry: %v", embeddings) + } + if got := requests.Load(); got != 2 { + t.Errorf("expected 2 requests (1 failure + 1 retry), got %d", got) + } +} + +func TestEmbedTexts_RetryExhausted(t *testing.T) { + var requests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + _, err := client.EmbedTexts(context.Background(), posts) + if err == nil { + t.Fatal("expected error after retries exhausted, got nil") + } + if got := requests.Load(); got != embedRetries+1 { + t.Errorf("expected %d attempts (initial + %d retries), got %d", embedRetries+1, embedRetries, got) + } +} + +func TestEmbedTexts_ParallelOrder(t *testing.T) { + // Shake ordering: handler sleeps and varies the embedding by prompt length, + // so a naive (unordered) worker pool would misalign results. + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + var req struct { + Prompt string `json:"prompt"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + time.Sleep(30 * time.Millisecond) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{float64(len(req.Prompt))}, + }) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := make([]clustering.Item, 8) + want := make([]float64, 8) + for i := range posts { + posts[i] = clustering.Item{ID: fmt.Sprintf("%d", i), Text: strings.Repeat("x", i+1)} + want[i] = float64(i + 1) + } + + embeddings, err := client.EmbedTexts(context.Background(), posts) + if err != nil { + t.Fatalf("EmbedTexts failed: %v", err) + } + if len(embeddings) != len(posts) { + t.Fatalf("expected %d embeddings, got %d", len(posts), len(embeddings)) + } + for i, emb := range embeddings { + if len(emb) != 1 || emb[0] != want[i] { + t.Errorf("embedding %d out of order or wrong value: got %v, want %v", i, emb, want[i]) + } + } +} + +// ============================================================================ +// SummarizeStyle Tests +// ============================================================================ + +func TestSummarizeStyle(t *testing.T) { + type generateRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + } + + var reqBody generateRequest + callCount := 0 + + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + callCount++ + + // 1. Validate HTTP Method + if r.Method != http.MethodPost { + t.Errorf("expected method POST, got %s", r.Method) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // 2. Validate Headers + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) + http.Error(w, "invalid content-type", http.StatusBadRequest) + return + } + + // 3. Validate Request Payload + if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + if reqBody.Model != "test-style" { + t.Errorf("expected model 'test-style', got %q", reqBody.Model) + } + + // Ensure the prompt includes input text + if !strings.Contains(reqBody.Prompt, "First post") || !strings.Contains(reqBody.Prompt, "Second post") { + t.Errorf("prompt missing post contents: %q", reqBody.Prompt) + } + + // 4. Stream NDJSON Response + flusher, ok := w.(http.Flusher) + if !ok { + t.Error("ResponseWriter is not a Flusher") + return + } + + w.Header().Set("Content-Type", "application/x-ndjson") + + chunks := []string{ + `{"response":"Style summary: "}` + "\n", + `{"response":"concise and "}` + "\n", + `{"response":"clear.","done":true}` + "\n", + } + + for _, chunk := range chunks { + _, _ = w.Write([]byte(chunk)) + flusher.Flush() + } + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{ + {ID: "1", Text: "First post"}, + {ID: "2", Text: "Second post"}, + } + + summary, err := client.SummarizeStyle(context.Background(), posts) + if err != nil { + t.Fatalf("SummarizeStyle failed: %v", err) + } + + expected := "Style summary: concise and clear." + if summary != expected { + t.Errorf("got summary %q, want %q", summary, expected) + } + + if callCount != 1 { + t.Errorf("expected 1 call to /api/generate, got %d", callCount) } } func TestSummarizeStyle_EmptyInput(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - t.Fatal("generate endpoint should not be called for empty input") + t.Error("generate endpoint should not be called when input is empty") + http.Error(w, "should not be called", http.StatusInternalServerError) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - summary, err := client.SummarizeStyle(context.Background(), []clustering.Item{}) if err != nil { t.Fatalf("SummarizeStyle with empty input failed: %v", err) @@ -213,3 +389,128 @@ func TestSummarizeStyle_EmptyInput(t *testing.T) { t.Errorf("expected empty summary, got %q", summary) } } + +func TestSummarizeStyle_Error(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"internal model failure"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error from failing server, got nil") + } +} + +func TestSummarizeStyle_MalformedJSONStream(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + + // First line is valid JSON, second is corrupt + _, _ = w.Write([]byte(`{"response":"valid part"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + _, _ = w.Write([]byte(`{corrupt_json` + "\n")) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error on malformed JSON stream, got nil") + } +} + +func TestSummarizeStyle_ContextCanceled(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + + // Send initial chunk + _, _ = w.Write([]byte(`{"response":"waiting..."}` + "\n")) + if flusher != nil { + flusher.Flush() + } + + // Wait until client context cancels + <-r.Context().Done() + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(ctx, posts) + if err == nil { + t.Fatal("expected error on canceled context, got nil") + } +} + +func TestSummarizeStyle_StreamNeverCompletes(t *testing.T) { + // Streaming ends without a done:true chunk (simulates a dropped connection + // or the http.Client timeout firing mid-generation). + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = w.Write([]byte(`{"response":"partial answer"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + }) // handler returns without ever writing done:true + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error when stream ends without completion, got nil") + } + if !strings.Contains(err.Error(), "ended before completion") { + t.Errorf("expected 'ended before completion' error, got %v", err) + } +} + +func TestSummarizeStyle_LengthTruncated(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = w.Write([]byte(`{"response":"summary...","done":true,"done_reason":"length"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error when response is truncated at context limit, got nil") + } + if !strings.Contains(err.Error(), "truncated") { + t.Errorf("expected 'truncated' error, got %v", err) + } +} diff --git a/internal/server/actors.go b/internal/server/actors.go index 724285c..205c8fa 100644 --- a/internal/server/actors.go +++ b/internal/server/actors.go @@ -7,16 +7,16 @@ import ( "sync" ) -func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]ActorProfile, error) { +func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]*ActorProfile, error) { var ( mu sync.Mutex - results []ActorProfile + results []*ActorProfile wg sync.WaitGroup errMu sync.Mutex errs []error success int ) - call := func(fn func(context.Context, string, int) ([]ActorProfile, error)) { + call := func(fn func(context.Context, string, int) ([]*ActorProfile, error)) { defer wg.Done() r, err := fn(ctx, query, limit) if err != nil { @@ -44,9 +44,9 @@ func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ( // dedupeActors removes near-duplicate profiles across platforms. Two results are considered // the same person when they share a normalized handle (the '@' and any instance/host stripped, // lower-cased) or a normalized display name. The first occurrence wins. -func dedupeActors(in []ActorProfile) []ActorProfile { +func dedupeActors(in []*ActorProfile) []*ActorProfile { seen := make(map[string]struct{}, len(in)) - out := make([]ActorProfile, 0, len(in)) + out := make([]*ActorProfile, 0, len(in)) for _, p := range in { key := normalizeHandle(p.Handle) if key == "" { diff --git a/internal/server/analytics.go b/internal/server/analytics.go index ea0faf7..cb38bcf 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -152,14 +152,19 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to return nil, fmt.Errorf("invalid time range: %w", err) } - platforms := []string{} + var platforms []string if args.Platform != "" { platforms = []string{args.Platform} } else { - // Twitter is included by default - the API costs it incurs are passed - // on to the caller through the paid tool pricing, so aggregating over - // it does not run the server at a loss. - platforms = []string{"Twitter", "Bluesky", "ActivityPub", "Nostr"} + // Twitter is included by default only when the API is enabled: the API + // costs it incurs are passed on to the caller through the upto pricing + // of these tools (see registerTools), so aggregating over it does not + // run the server at a loss. When no bearer token is configured there is + // no Twitter to aggregate over. + platforms = []string{"Bluesky", "ActivityPub", "Nostr"} + if s.twitterEnabled { + platforms = append(platforms, "Twitter") + } } timeRangeHash := hashTimeRange(start, end, limit) @@ -175,12 +180,13 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if cached, ok := s.getAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash); ok { var cachedResult json.RawMessage - json.Unmarshal(cached, &cachedResult) + _ = json.Unmarshal(cached, &cachedResult) return s.toToolResult(cachedResult) } // Fetch posts from all platforms var allPosts []clustering.Item + twitterPosts := 0 for _, p := range platforms { plat, ok := platformFromString(p) if !ok { @@ -192,6 +198,9 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to continue } allPosts = append(allPosts, posts...) + if plat == Twitter { + twitterPosts += len(posts) + } } if len(allPosts) == 0 { @@ -203,14 +212,40 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if err != nil { return nil, err } + result = setAnalyticsTwitterPostCount(result, twitterPosts) // Cache the result under the matching scope key. ttl := s.getCacheTTL(toolName) - s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) + _ = s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) return s.toToolResult(result) } +// setAnalyticsTwitterPostCount records how many posts in an analytics result +// were sourced from the Twitter API, so upto-scheme settlement can pass the +// realized Twitter costs on to the caller. Unknown result types are returned +// unchanged. +func setAnalyticsTwitterPostCount(result any, count int) any { + switch v := result.(type) { + case EngagementMetricsResponse: + v.TwitterPostCount = count + return v + case PostingPatternsResponse: + v.TwitterPostCount = count + return v + case TopicModelingResponse: + v.TwitterPostCount = count + return v + case SentimentTrendsResponse: + v.TwitterPostCount = count + return v + case EngagementDriversResponse: + v.TwitterPostCount = count + return v + } + return result +} + func (s *Server) fetchAnalyticsPosts(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]clustering.Item, error) { posts, err := s.fetchPostsInTimeForAnalytics(ctx, platform, author, start, end, limit) if err != nil { @@ -598,20 +633,16 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T return []TopicCluster{}, map[string]string{}, nil } - // Use embeddings from ollama for clustering items := posts - // We need embeddings for clustering - compute them embeddings, err := s.ollama.EmbedTexts(ctx, items) if err != nil { return nil, nil, fmt.Errorf("failed to compute embeddings: %w", err) } - // Update items with embeddings for i := range items { items[i].Embedding = embeddings[i] } - // Determine optimal number of topics topicCount := maxTopicCount if len(items) < maxTopicCount { topicCount = len(items) @@ -620,7 +651,6 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T topicCount = minTopicCount } - // Run KMeans results, err := clustering.KMeans(items, topicCount, 50, 42) if err != nil { return nil, nil, fmt.Errorf("kmeans failed: %w", err) @@ -634,7 +664,6 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T continue } - // Collect sample texts for labeling var samples []string for _, item := range cluster.Items { if len(samples) >= 3 { @@ -782,8 +811,7 @@ func (s *Server) computeSentimentTrends(ctx context.Context, args AnalyticsReque } func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsRequest, posts []clustering.Item) (any, error) { - // Topic modeling first, getting post->topic membership so we can correlate - // each topic with the engagement of its member posts. + // Topic modeling first, getting post->topic membership for correlation. topics, membership, err := s.clusterPosts(ctx, posts) if err != nil { return nil, err @@ -801,8 +829,7 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq topicEngagement[label] = append(topicEngagement[label], total) } - // Baseline is the mean engagement across all posts, used as the comparison - // point for relevance ranking. + // Baseline is the mean engagement across all posts, for comparison. var allEngagement []float64 for _, post := range posts { allEngagement = append(allEngagement, float64(post.Likes+post.Reposts+post.Replies+post.Quotes)) @@ -813,9 +840,6 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq for _, topic := range topics { values := topicEngagement[topic.Label] topicMean := mean(values) - // Correlation is the normalized difference between the topic's mean - // engagement and the overall baseline: positive when the topic draws - // above-average engagement, negative when below. Bounded to [-1, 1]. correlation := 0.0 if denom := topicMean + baseline; denom > 0 { correlation = (topicMean - baseline) / denom @@ -911,7 +935,7 @@ Texts: Neutral float64 `json:"neutral"` Emotions map[string]int `json:"emotions"` } - json.Unmarshal([]byte(resp), &result) + _ = json.Unmarshal([]byte(resp), &result) // Normalize sum := result.Positive + result.Negative + result.Neutral diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index d96a439..e74eef6 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -9,6 +9,7 @@ import ( "github.com/nacorid/naco-api/internal/activitypub" "github.com/nacorid/naco-api/internal/clustering" + "github.com/nacorid/naco-api/internal/twitter" ) var errSome = fmt.Errorf("some error") @@ -16,8 +17,8 @@ var errSome = fmt.Errorf("some error") func analyticsAPPosts() *fakeAP { base := time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC) return &fakeAP{posts: []*activitypub.Post{ - {ID: "p1", Author: "@a@mastodon.social", Content: "first post", Text: "first post", CreatedAt: base}, - {ID: "p2", Author: "@a@mastodon.social", Content: "second post", Text: "second post", CreatedAt: base.Add(time.Hour)}, + {ID: "p1", Author: "@a@mastodon.social", Content: "first post", CreatedAt: base}, + {ID: "p2", Author: "@a@mastodon.social", Content: "second post", CreatedAt: base.Add(time.Hour)}, }} } @@ -141,6 +142,54 @@ func TestAnalyticsPartialPlatformFailureTolerated(t *testing.T) { } } +// TestAnalyticsTwitterPostCount verifies the response reports how many posts +// were sourced from the Twitter API, so upto-scheme settlement can charge the +// realized Twitter API costs. +func TestAnalyticsTwitterPostCount(t *testing.T) { + tw := &fakeTwitter{posts: []*twitter.Post{ + {ID: "t1", Content: "first tweet", CreatedAt: time.Date(2024, 1, 11, 12, 0, 0, 0, time.UTC)}, + {ID: "t2", Content: "second tweet", CreatedAt: time.Date(2024, 1, 12, 12, 0, 0, 0, time.UTC)}, + {ID: "t3", Content: "third tweet", CreatedAt: time.Date(2024, 1, 13, 12, 0, 0, 0, time.UTC)}, + }} + st := newFakeStore() + s := newServerWith(&fakeBluesky{}, analyticsAPPosts(), &fakeNostr{}, tw, st, &fakeOllama{}) + + res, err := callTool(s, "getEngagementMetrics", AnalyticsRequest{ + Author: "@alice", + StartTime: "2024-01-01T00:00:00Z", + EndTime: "2024-12-31T00:00:00Z", + Platform: "Twitter", + }) + if err != nil { + t.Fatalf("getEngagementMetrics error: %v", err) + } + var out EngagementMetricsResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 3 { + t.Fatalf("expected TwitterPostCount 3, got %d", out.TwitterPostCount) + } + + // A non-Twitter platform reports 0 and the default aggregate excludes + // Twitter when the API is not enabled (twitterEnabled=false in tests). + res, err = callTool(s, "getEngagementMetrics", AnalyticsRequest{ + Author: "@a@mastodon.social", + StartTime: "2024-01-01T00:00:00Z", + EndTime: "2024-12-31T00:00:00Z", + Platform: "ActivityPub", + }) + if err != nil { + t.Fatalf("getEngagementMetrics error: %v", err) + } + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 0 { + t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out.TwitterPostCount) + } +} + // ---- pure compute functions ---- func TestComputePostingPatterns(t *testing.T) { diff --git a/internal/server/auth.go b/internal/server/auth.go index f822e48..07c8c24 100644 --- a/internal/server/auth.go +++ b/internal/server/auth.go @@ -2,7 +2,7 @@ package server import ( "context" - "crypto/ecdsa" + "crypto" "crypto/ed25519" "crypto/rand" "crypto/x509" @@ -20,7 +20,7 @@ import ( type CDPAuth struct { keyName string - privateKey interface{} // *ecdsa.PrivateKey or ed25519.PrivateKey + privateKey crypto.Signer signingMethod jwt.SigningMethod } @@ -36,13 +36,12 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { // Convert to real newlines before decoding; idempotent if already multiline. privateKeyPEM = strings.ReplaceAll(privateKeyPEM, "\\n", "\n") - // Check if it's a base64-encoded Ed25519 key (64 bytes) - if isValidEd25519Key(privateKeyPEM) { - decoded, _ := base64.StdEncoding.DecodeString(privateKeyPEM) - privateKey := ed25519.PrivateKey(decoded) + // Check if it's a base64-encoded Ed25519 key (64 bytes). + decoded, err := base64.StdEncoding.DecodeString(privateKeyPEM) + if err == nil && len(decoded) == 64 { return &CDPAuth{ keyName: keyName, - privateKey: privateKey, + privateKey: ed25519.PrivateKey(decoded), signingMethod: jwt.SigningMethodEdDSA, }, nil } @@ -66,15 +65,6 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { }, nil } -// isValidEd25519Key checks if a string is a valid base64-encoded Ed25519 private key (64 bytes). -func isValidEd25519Key(str string) bool { - decoded, err := base64.StdEncoding.DecodeString(str) - if err != nil { - return false - } - return len(decoded) == 64 -} - func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { nonceBytes := make([]byte, 16) if _, err := rand.Read(nonceBytes); err != nil { @@ -101,10 +91,8 @@ func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { var err error switch a.signingMethod { - case jwt.SigningMethodES256: - signedToken, err = token.SignedString(a.privateKey.(*ecdsa.PrivateKey)) - case jwt.SigningMethodEdDSA: - signedToken, err = token.SignedString(a.privateKey.(ed25519.PrivateKey)) + case jwt.SigningMethodES256, jwt.SigningMethodEdDSA: + signedToken, err = token.SignedString(a.privateKey) default: return "", fmt.Errorf("unsupported signing method: %v", a.signingMethod) } diff --git a/internal/server/backends.go b/internal/server/backends.go index 8bd39a6..3182a3b 100644 --- a/internal/server/backends.go +++ b/internal/server/backends.go @@ -26,15 +26,15 @@ type blueskyBackend interface { GetFeed(ctx context.Context, feedURI string, limit int) (*bsky.FeedGetFeed_Output, error) ResolveUser(ctx context.Context, user string) (syntax.DID, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type apBackend interface { GetPost(ctx context.Context, postID string) (*activitypub.Post, error) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*activitypub.Post, error) - GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) + GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type nostrBackend interface { @@ -42,7 +42,7 @@ type nostrBackend interface { GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*nostr.Post, error) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*nostr.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type twitterBackend interface { diff --git a/internal/server/cache.go b/internal/server/cache.go index a9b5869..09b7f2a 100644 --- a/internal/server/cache.go +++ b/internal/server/cache.go @@ -6,6 +6,7 @@ import ( ) const postCacheTTL = 5 * time.Minute +const postCacheMaxItems = 10000 type ttlCacheEntry struct { value any @@ -13,15 +14,17 @@ type ttlCacheEntry struct { } type ttlCache struct { - mu sync.Mutex - items map[string]ttlCacheEntry - ttl time.Duration + mu sync.Mutex + items map[string]ttlCacheEntry + ttl time.Duration + maxItems int } func newTTLCache(ttl time.Duration) *ttlCache { return &ttlCache{ - items: make(map[string]ttlCacheEntry), - ttl: ttl, + items: make(map[string]ttlCacheEntry), + ttl: ttl, + maxItems: postCacheMaxItems, } } @@ -42,6 +45,22 @@ func (c *ttlCache) get(key string) (any, bool) { func (c *ttlCache) set(key string, value any) { c.mu.Lock() defer c.mu.Unlock() + if len(c.items) >= c.maxItems { + // Evict the oldest expired entry, or if none expired, the first one. + var oldestKey string + var oldestTime time.Time + first := true + for k, e := range c.items { + if first || e.exp.Before(oldestTime) { + oldestKey = k + oldestTime = e.exp + first = false + } + } + if oldestKey != "" { + delete(c.items, oldestKey) + } + } c.items[key] = ttlCacheEntry{value: value, exp: time.Now().Add(c.ttl)} } @@ -64,6 +83,6 @@ func cachedFetch[T any](c *ttlCache, key string, fetch func() (T, error)) (T, er } type latestCacheValue struct { - posts []GetPostResponsePost + posts []*GetPostResponsePost aid string } diff --git a/internal/server/cache_version.go b/internal/server/cache_version.go index 25fc192..40ea07e 100644 --- a/internal/server/cache_version.go +++ b/internal/server/cache_version.go @@ -8,6 +8,7 @@ const ( // IsProfileStale returns true if the profile's versions are older than the current versions. // This is used to mark profiles stale for lazy recompute on the next request. +// Currently all three version fields are 1 since this is the first version of the combination. func IsProfileStale(model, schema, algorithm int) bool { return model != CurrentModelVersion || schema != CurrentSchemaVersion || diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 9e5a1df..1ff56b1 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -27,7 +27,7 @@ type fakeBluesky struct { postView *bsky.FeedDefs_PostView feedViews []*bsky.FeedDefs_FeedViewPost profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile feedOut *bsky.FeedGetFeed_Output err error calls int @@ -76,7 +76,7 @@ func (f *fakeBluesky) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -89,7 +89,7 @@ type fakeAP struct { post *activitypub.Post posts []*activitypub.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int lastLimit int @@ -109,15 +109,13 @@ func (f *fakeAP) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } return f.posts, nil } -func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) { +func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) { f.calls++ if f.err != nil { return nil, f.err } - out := make([]activitypub.Post, 0, len(f.posts)) - for _, p := range f.posts { - out = append(out, *p) - } + out := make([]*activitypub.Post, 0, len(f.posts)) + out = append(out, f.posts...) return out, nil } func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -127,7 +125,7 @@ func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, e } return f.profile, nil } -func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -140,7 +138,7 @@ type fakeNostr struct { post *nostr.Post posts []*nostr.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int lastLimit int @@ -174,7 +172,7 @@ func (f *fakeNostr) GetActor(ctx context.Context, author string) (*actor.Profile } return f.profile, nil } -func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -187,7 +185,7 @@ type fakeTwitter struct { post *twitter.Post posts []*twitter.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int } @@ -220,7 +218,7 @@ func (f *fakeTwitter) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ if f.err != nil { return nil, f.err @@ -394,10 +392,10 @@ func TestGetActorDispatch(t *testing.T) { } func TestSearchActorsAllSucceed(t *testing.T) { - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice", Limit: 10}) if err != nil { @@ -425,8 +423,8 @@ func TestSearchActorsAllFail(t *testing.T) { func TestSearchActorsPartialSuccess(t *testing.T) { bsky := &fakeBluesky{err: fmt.Errorf("boom")} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} tw := &fakeTwitter{err: fmt.Errorf("boom")} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice"}) @@ -740,9 +738,9 @@ func TestGetActorTwitter(t *testing.T) { // ---- searchActors edge cases ---- func TestSearchActorsLimitClamp(t *testing.T) { - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, &fakeTwitter{}) // Limit > 50 should be clamped to 50 in the backend call. @@ -765,10 +763,10 @@ func TestSearchActorsLimitClamp(t *testing.T) { func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { // Twitter is not supported for search in the current implementation - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "test", Limit: 10}) if err != nil { @@ -786,8 +784,8 @@ func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { func TestSearchActorsNormalizationAndDedup(t *testing.T) { // Test that handles normalize correctly and deduplicate - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} s := newTestServer(bsky, ap, &fakeNostr{}, &fakeTwitter{}) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "user", Limit: 10}) if err != nil { @@ -825,7 +823,7 @@ func TestPostCacheDifferentKeys(t *testing.T) { func TestCacheVersionStaleness(t *testing.T) { st := newFakeStore() - // Add a stale profile (older than 7 days) + // Add a stale profile (older than 30 days) st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", Platform: "ActivityPub", @@ -833,9 +831,9 @@ func TestCacheVersionStaleness(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } - ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}}} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "@a@mastodon.social"}) if err != nil { diff --git a/internal/server/platform.go b/internal/server/platform.go index 6dcb2bf..a0330cd 100644 --- a/internal/server/platform.go +++ b/internal/server/platform.go @@ -26,7 +26,7 @@ func determinePlatform(author string) (platform, error) { if len(author) == 64 { isHex := true for _, c := range author { - if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { isHex = false break } diff --git a/internal/server/posts.go b/internal/server/posts.go index eed3144..cd2bd28 100644 --- a/internal/server/posts.go +++ b/internal/server/posts.go @@ -16,34 +16,34 @@ func (s *Server) fetchSinglePost(ctx context.Context, platform platform, author, return nil, err } r := bskyPostViewToResponse(res) - return &r, nil + return r, nil case ActivityPub: p, err := s.apClient.GetPost(ctx, recordKey) if err != nil { return nil, err } - r := apPostToResponse(*p) - return &r, nil + r := apPostToResponse(p) + return r, nil case Nostr: p, err := s.nostrClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := nostrPostToResponse(p) - return &r, nil + return r, nil case Twitter: p, err := s.twitterClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := twitterPostToResponse(p) - return &r, nil + return r, nil } return nil, fmt.Errorf("unsupported platform %q", platform) }) } -func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]GetPostResponsePost, string, error) { +func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]*GetPostResponsePost, string, error) { key := fmt.Sprintf("latest:%s|%s|%d", platform, author, limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -84,7 +84,7 @@ func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]GetPostResponsePost, string, error) { +func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]*GetPostResponsePost, string, error) { key := fmt.Sprintf("time:%s|%s|%s|%s|%d", platform, author, start.Format(time.RFC3339), end.Format(time.RFC3339), limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -125,9 +125,9 @@ func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]GetPostResponsePost, error) { +func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]*GetPostResponsePost, error) { key := fmt.Sprintf("feed:%s|%s", platform, uri) - return cachedFetch(s.postCache, key, func() ([]GetPostResponsePost, error) { + return cachedFetch(s.postCache, key, func() ([]*GetPostResponsePost, error) { if platform == Bluesky { res, err := s.bskyClient.GetFeed(ctx, uri, 50) if err != nil { diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index 181c9fc..f35b394 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -10,7 +10,7 @@ import ( "github.com/nacorid/naco-api/internal/twitter" ) -func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { +func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) *GetPostResponsePost { author := "" if p.Author != nil { author = p.Author.Did @@ -34,7 +34,7 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { if p.ReplyCount != nil { replies = *p.ReplyCount } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: author, Content: content, Likes: likes, @@ -45,12 +45,12 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { } } -func apPostToResponse(p activitypub.Post) GetPostResponsePost { +func apPostToResponse(p *activitypub.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.FavouritesCount), @@ -60,24 +60,24 @@ func apPostToResponse(p activitypub.Post) GetPostResponsePost { } } -func nostrPostToResponse(p *nostr.Post) GetPostResponsePost { +func nostrPostToResponse(p *nostr.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, CreatedAt: created, } } -func twitterPostToResponse(p *twitter.Post) GetPostResponsePost { +func twitterPostToResponse(p *twitter.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.Likes), @@ -90,8 +90,8 @@ func twitterPostToResponse(p *twitter.Post) GetPostResponsePost { // ---- Batch converters for GetPostResponsePost ---- -func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]GetPostResponsePost, string) { - posts := make([]GetPostResponsePost, 0, len(views)) +func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]*GetPostResponsePost, string) { + posts := make([]*GetPostResponsePost, 0, len(views)) var aid string for _, p := range views { if p.Post == nil { @@ -105,18 +105,8 @@ func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]GetPost return posts, aid } -func apPostsPtrToResponses(posts []*activitypub.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) - var aid string - for _, p := range posts { - out = append(out, apPostToResponse(*p)) - aid = p.Author - } - return out, aid -} - -func apPostsToResponses(posts []activitypub.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func apPostsPtrToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, apPostToResponse(p)) @@ -125,8 +115,18 @@ func apPostsToResponses(posts []activitypub.Post) ([]GetPostResponsePost, string return out, aid } -func nostrPostsToResponses(posts []*nostr.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func apPostsToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) + var aid string + for _, p := range posts { + out = append(out, apPostToResponse(p)) + aid = p.Author + } + return out, aid +} + +func nostrPostsToResponses(posts []*nostr.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, nostrPostToResponse(p)) @@ -135,8 +135,8 @@ func nostrPostsToResponses(posts []*nostr.Post) ([]GetPostResponsePost, string) return out, aid } -func twitterPostsToResponses(posts []*twitter.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func twitterPostsToResponses(posts []*twitter.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, twitterPostToResponse(p)) @@ -154,6 +154,9 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I if !ok { continue } + if fp.Text == "" { + continue + } out = append(out, clustering.Item{ID: p.Post.Uri, Text: fp.Text}) } return out @@ -162,8 +165,8 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I func apPostsPtrToItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Text != "" { - out = append(out, clustering.Item{ID: p.ID, Text: p.Text}) + if p.Content != "" { + out = append(out, clustering.Item{ID: p.ID, Text: p.Content}) } } return out @@ -214,12 +217,12 @@ func blueskyFeedViewsToAnalyticsItems(views []*bsky.FeedDefs_FeedViewPost) []clu return out } -func apPostsToAnalyticsItems(posts []activitypub.Post) []clustering.Item { +func apPostsToAnalyticsItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Text != "" { + if p.Content != "" { out = append(out, clustering.Item{ - ID: p.ID, Text: p.Text, Timestamp: p.CreatedAt, + ID: p.ID, Text: p.Content, Timestamp: p.CreatedAt, Likes: int64(p.FavouritesCount), Reposts: int64(p.ReblogsCount), Replies: int64(p.RepliesCount), }) } diff --git a/internal/server/prewarm.go b/internal/server/prewarm.go index 55caf69..2fd9041 100644 --- a/internal/server/prewarm.go +++ b/internal/server/prewarm.go @@ -83,19 +83,11 @@ func (s *Server) discoverInterestingAuthors(ctx context.Context) []string { } func (s *Server) prewarmOne(ctx context.Context, author string) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() if _, err := s.requestStyleProfile(ctx, author); err != nil { log.WarnWithContext(ctx, "prewarm: failed to build profile", "author", author, "error", err) return } log.InfoWithContext(ctx, "prewarmed style profile", "author", author) } - -func (s *Server) startPostPruner() { - ticker := time.NewTicker(24 * time.Hour) - defer ticker.Stop() - for range ticker.C { - if err := s.db.PruneStylePosts(context.Background()); err != nil { - log.Error("failed to prune style posts", "error", err) - } - } -} diff --git a/internal/server/searchactors_test.go b/internal/server/searchactors_test.go index dcab2f3..61fa8c6 100644 --- a/internal/server/searchactors_test.go +++ b/internal/server/searchactors_test.go @@ -29,7 +29,7 @@ func TestNormalizeHandle(t *testing.T) { } func TestDedupeActors(t *testing.T) { - in := []ActorProfile{ + in := []*ActorProfile{ {Handle: "@alice", DisplayName: "Alice"}, {Handle: "alice@mastodon.social", DisplayName: "Alice"}, {Handle: "", DisplayName: ""}, @@ -45,7 +45,7 @@ func TestDedupeActors(t *testing.T) { func TestDedupeActorsByDisplayNameOnly(t *testing.T) { // When handles are empty but display names collide, dedupe by display name. - in := []ActorProfile{ + in := []*ActorProfile{ {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Other"}, @@ -59,10 +59,10 @@ func TestDedupeActorsByDisplayNameOnly(t *testing.T) { func TestSearchActorsTwitterExcluded(t *testing.T) { // searchActorsAll only queries Bluesky, ActivityPub and Nostr. A Twitter // result must never appear because the backend is not consulted. - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "q"}) diff --git a/internal/server/server.go b/internal/server/server.go index ef4508b..4cccf25 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -48,6 +48,12 @@ type Server struct { analyticsTTLTopics time.Duration analyticsTTLSentiment time.Duration analyticsTTLDrivers time.Duration + + // twitterEnabled reports whether the Twitter API is available (a bearer + // token is configured). It flips the Twitter-capable tools to upto + // variable pricing so the realized Twitter API costs are passed on to + // callers (see registerTools). + twitterEnabled bool } func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky *bluesky.BlueskyClient, n *nostr.Client, t *twitter.Client, o *ollama.OllamaClient, cfg *config.Config) *Server { @@ -78,6 +84,7 @@ func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky analyticsTTLTopics: cfg.AnalyticsTTLTopics, analyticsTTLSentiment: cfg.AnalyticsTTLSentiment, analyticsTTLDrivers: cfg.AnalyticsTTLDrivers, + twitterEnabled: cfg.TwitterBearerToken != "", } // Combine address and port for TCP (not unix socket) if port not already present @@ -104,6 +111,10 @@ func (s *Server) MetricsText() string { } func (s *Server) Start() error { - go s.startPostPruner() + //go s.startPostPruner() return s.mcpHandler.Start(s.listenAddress, s.permissions) } + +func (s *Server) Shutdown(ctx context.Context) error { + return s.mcpHandler.Shutdown(ctx) +} diff --git a/internal/server/server_init.go b/internal/server/server_init.go index 88f801b..832d103 100644 --- a/internal/server/server_init.go +++ b/internal/server/server_init.go @@ -65,7 +65,7 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } srv, err := x402.NewX402Server( - "x402 enabled Bluesky lookingglass for autonomous Agents", + "x402 enabled social media lookingglass for autonomous agents", "0.2.0", cfg.X402FacilitatorURL, auth, @@ -85,7 +85,9 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "amount", requirements.Amount, "asset", requirements.Asset, ) - persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer) + if err := persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer); err != nil { + log.WarnWithContext(r.Context(), "failed to persist HTTP settlement", "error", err) + } }, networkStrings, ) @@ -103,9 +105,11 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "asset", settleCtx.Requirements.GetAsset(), "scheme", settleCtx.Requirements.GetScheme(), ) - persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, + if err := persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, settleCtx.Requirements.GetAmount(), settleCtx.Requirements.GetAsset(), - string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer) + string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer); err != nil { + log.WarnWithContext(settleCtx.Ctx, "failed to persist settlement", "error", err) + } // Webhook is best-effort and non-blocking. go sendSettlementWebhook(settleCtx.Ctx, cfg, metrics, settleCtx.Result.Transaction) @@ -116,8 +120,8 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } // persistSettlement is a thin wrapper around settleHook to keep call sites uniform. -func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) { - _ = settleHook(ctx, db, transaction, amount, asset, network, to, from) +func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) error { + return settleHook(ctx, db, transaction, amount, asset, network, to, from) } // sendSettlementWebhook builds and sends the Matrix webhook payload for a settled transaction. @@ -132,8 +136,7 @@ func sendSettlementWebhook(ctx context.Context, cfg *config.Config, metrics *Met "x402_transaction_hash": txHash, }, "metrics": map[string]any{ - "function_runtime_ms": 0, - "platform": "unknown", + "platform": "unknown", }, "timestamp": time.Now().UTC().Format(time.RFC3339), "server_id": cfg.MatrixServerID, diff --git a/internal/server/settlement.go b/internal/server/settlement.go index 03c1a34..4acb2a0 100644 --- a/internal/server/settlement.go +++ b/internal/server/settlement.go @@ -36,7 +36,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass } if err != nil { - log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "value", ret, "error", err) + log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "error", err) return err } @@ -44,7 +44,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass return nil } -func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string) []x402v2.PaymentRequirements { +func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string, timeout int) []x402v2.PaymentRequirements { requirements := make([]x402v2.PaymentRequirements, 0, len(s.activeNetworks)) for _, info := range s.activeNetworks { @@ -57,7 +57,7 @@ func (s *Server) buildRequirements(amount string, extra map[string]any, scheme s Asset: info.Asset, Amount: amount, PayTo: s.x402Wallet, - MaxTimeoutSeconds: 600, + MaxTimeoutSeconds: timeout, Extra: mergedExtra, }) } diff --git a/internal/server/storefake_test.go b/internal/server/storefake_test.go index 066ca5e..4ab3b24 100644 --- a/internal/server/storefake_test.go +++ b/internal/server/storefake_test.go @@ -94,7 +94,7 @@ func (f *fakeStore) GetAnalyticsPostsInRange(ctx context.Context, arg store.GetA Quotes: p.Quotes, }) } - sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.After(rows[j].PostTimestamp) }) + sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.Before(rows[j].PostTimestamp) }) if int32(len(rows)) > arg.Limit { rows = rows[:arg.Limit] } diff --git a/internal/server/style.go b/internal/server/style.go index 5f9e217..8015d50 100644 --- a/internal/server/style.go +++ b/internal/server/style.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "strings" + "github.com/modelcontextprotocol/go-sdk/mcp" log "github.com/nacorid/logger" "github.com/nacorid/naco-api/internal/clustering" @@ -52,7 +54,7 @@ func (s *Server) getStyleProfile(ctx context.Context, request *mcp.CallToolReque } // requestStyleProfile returns a style profile for author. It computes and persists the -// profile only when a request is made AND the cached profile is older than a week (or +// profile only when a request is made AND the cached profile is older than a month (or // missing); otherwise it serves the cached copy. The prewarmer calls this same path, so a // pre-fetch is treated as a system-initiated request and obeys the identical update policy. func (s *Server) requestStyleProfile(ctx context.Context, author string) (*GetStyleProfileResponse, error) { @@ -86,7 +88,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor Platform: platform.String(), }) if err == nil { - isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*7*time.Hour + isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*30*time.Hour if isRecent { embedding, perr := utils.ParseEmbedding(styleProfile.Embedding) if perr != nil { @@ -110,7 +112,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor // enrichCrossPlatform looks for author on every platform other than sourcePlatform, in the // background. It uses the same identifier on each platform (matching only where that handle -// exists; mismatches fail gracefully and are skipped) and obeys the same 1-week update gate +// exists; mismatches fail gracefully and are skipped) and obeys the same 1-month update gate // as a live request. Concurrency is bounded to the other three platforms. func (s *Server) enrichCrossPlatform(ctx context.Context, sourcePlatform platform, author string) { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) @@ -191,7 +193,14 @@ func (s *Server) getSimilarProfiles(ctx context.Context, request *mcp.CallToolRe }) } - respBytes, err := json.Marshal(FindSimilarUsersResponse{Matches: matches}) + twitterPostCount := 0 + if profile.SourcePlatform == "Twitter" { + twitterPostCount = profile.PostCount + } + respBytes, err := json.Marshal(FindSimilarUsersResponse{ + Matches: matches, + TwitterPostCount: twitterPostCount, + }) if err != nil { return nil, err } @@ -211,6 +220,21 @@ func (s *Server) buildStyleProfile(ctx context.Context, platform platform, autho return nil, fmt.Errorf("no posts found for author %s", author) } + // Drop posts without any text (e.g. Bluesky image/reply-only posts). Embedding them + // yields empty vectors, which would poison the persona-average and the persisted + // profile. + filtered := posts[:0] + for _, p := range posts { + if strings.TrimSpace(p.Text) == "" { + continue + } + filtered = append(filtered, p) + } + if len(filtered) == 0 { + return nil, fmt.Errorf("no posts found for author %s", author) + } + posts = filtered + // Delete old posts for non-Twitter platforms before rebuild if platform != Twitter { if err := s.db.DeleteStylePosts(ctx, store.DeleteStylePostsParams{ diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index ffc8079..9051f18 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -3,6 +3,7 @@ package server import ( "database/sql" "fmt" + "reflect" "strings" "testing" "time" @@ -16,8 +17,8 @@ import ( func TestGetStyleProfileBuildsFresh(t *testing.T) { ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "alpha post one", Text: "alpha post one", CreatedAt: time.Now()}, - {Author: "@a@mastodon.social", Content: "alpha post two", Text: "alpha post two", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post one", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post two", CreatedAt: time.Now()}, }} st := newFakeStore() oll := &fakeOllama{embeddings: [][]float64{{0.1, 0.2}, {0.3, 0.4}}} @@ -46,7 +47,7 @@ func TestGetStyleProfileBuildsFresh(t *testing.T) { } func TestGetStyleProfileServesFreshCache(t *testing.T) { - // A profile modified less than a week ago is served without hitting the + // A profile modified less than the 30-day stale threshold is served without hitting the // backend or ollama. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ @@ -82,7 +83,7 @@ func TestGetStyleProfileServesFreshCache(t *testing.T) { } func TestGetStyleProfileRebuildsStale(t *testing.T) { - // A stale (older than a week) cached profile triggers a rebuild. + // A stale (older than a month) cached profile triggers a rebuild. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", @@ -91,10 +92,10 @@ func TestGetStyleProfileRebuildsStale(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}, }} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) @@ -134,7 +135,7 @@ func TestGetStyleProfileNoPostsFound(t *testing.T) { } func TestGetStyleProfileOllamaEmbedFailure(t *testing.T) { - ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", Text: "post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", CreatedAt: time.Now()}}} oll := &fakeOllama{embedErr: fmt.Errorf("embedding failed")} st := newFakeStore() s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, oll) @@ -165,6 +166,61 @@ func TestGetStyleProfileBlueskyResolution(t *testing.T) { } } +func TestGetStyleProfileFirstEmbeddingEmpty(t *testing.T) { + // Regression: an empty embedding returned by Ollama for a non-first post used to + // panic AverageVector when the first vector was also empty (accumulator sized to + // length 0, later non-empty vector indexed it). Empty vectors are skipped. + bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ + {Post: bskyPostView("first post")}, + {Post: bskyPostView("second post")}, + }} + oll := &fakeOllama{embeddings: [][]float64{{}, {0.1, 0.2}}} + st := newFakeStore() + s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) + + res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) + if err != nil { + t.Fatalf("getStyleProfile error: %v", err) + } + var out GetStyleProfileResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(out.PersonaEmbedding, []float64{0.1, 0.2}) { + t.Fatalf("expected averaged embedding {0.1,0.2}, got %v", out.PersonaEmbedding) + } + if out.PostCount != 2 { + t.Fatalf("expected post count 2, got %d", out.PostCount) + } +} + +func TestGetStyleProfileSkipsEmptyTextPosts(t *testing.T) { + // A Bluesky post with no text (image/reply-only) must not reach Ollama: its empty + // embedding would poison the persona average. It is dropped before embedding. + bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ + {Post: bskyPostView("")}, + {Post: bskyPostView("real post")}, + }} + oll := &fakeOllama{} + st := newFakeStore() + s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) + + res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) + if err != nil { + t.Fatalf("getStyleProfile error: %v", err) + } + var out GetStyleProfileResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.PostCount != 1 { + t.Fatalf("expected filtered post count 1, got %d", out.PostCount) + } + if len(out.PersonaEmbedding) != 2 { + t.Fatalf("expected 2-dim embedding, got %v", out.PersonaEmbedding) + } +} + // ---- findSimilarUsers ---- func TestFindSimilarUsers(t *testing.T) { @@ -248,6 +304,61 @@ func TestFindSimilarUsersEmptyMatches(t *testing.T) { } } +// TestFindSimilarUsersTwitterPostCount verifies the response reports how many +// posts went into the query author's style profile when that author is on +// Twitter, so upto-scheme settlement can charge the realized Twitter API costs. +func TestFindSimilarUsersTwitterPostCount(t *testing.T) { + st := newFakeStore() + st.similarRows = []store.FindSimilarUsersRow{ + {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, + } + // Serve a fresh cached Twitter profile with 42 analyzed posts. + st.styleProfiles["@alice|Twitter"] = store.UserStyleProfile{ + UserID: "@alice", + Platform: "Twitter", + Embedding: "[0.1,0.2]", + PostCount: 42, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + } + s := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) + res, err := callTool(s, "findSimilarUsers", FindSimilarUsersRequest{Author: "@alice"}) + if err != nil { + t.Fatalf("findSimilarUsers error: %v", err) + } + var out FindSimilarUsersResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 42 { + t.Fatalf("expected TwitterPostCount 42, got %d", out.TwitterPostCount) + } + + // A non-Twitter query author reports 0. + st2 := newFakeStore() + st2.similarRows = []store.FindSimilarUsersRow{ + {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, + } + st2.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ + UserID: "@a@mastodon.social", + Platform: "ActivityPub", + Embedding: "[0.1,0.2]", + PostCount: 9, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + } + s2 := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st2, &fakeOllama{}) + res2, err := callTool(s2, "findSimilarUsers", FindSimilarUsersRequest{Author: "@a@mastodon.social"}) + if err != nil { + t.Fatalf("findSimilarUsers error: %v", err) + } + var out2 FindSimilarUsersResponse + if err := jsonUnmarshal(t, textOf(t, res2), &out2); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out2.TwitterPostCount != 0 { + t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out2.TwitterPostCount) + } +} + func TestFindSimilarUsersDBError(t *testing.T) { st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ diff --git a/internal/server/tools.go b/internal/server/tools.go index 510aef4..bb91c3d 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -7,6 +7,7 @@ import ( "time" "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/nacorid/naco-api/internal/x402" ) // registerTools registers all MCP tools with their x402 payment requirements. @@ -36,14 +37,26 @@ func (s *Server) registerTools() { } type payableDef struct { - Name string - Desc string - Input any - Output any - Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) - Price string - Scheme string - Extra map[string]any + Name string + Desc string + Input any + Output any + Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) + Price string + MaxTimeoutSeconds int + // UsesTwitter marks tools that may read posts (or set up lookups) + // through the Twitter API and return only transformative/aggregated + // results, as permitted by the X API policy. With Twitter enabled these + // tools charge upto-scheme: base price plus the realized Twitter API + // costs. Tools that resell raw post/profile/feed data (getPost, + // getLatestPosts, ...) never set this. + UsesTwitter bool + // TwitterPostCap is the post count used to compute the advertised upto + // maximum (see x402.MaxTwitterCostMicro). It only bounds the largest + // settlement the tool can demand; the actual charge is settled from the + // response. + TwitterPostCap int64 + Extra map[string]any } payable := []payableDef{ @@ -93,8 +106,16 @@ func (s *Server) registerTools() { ), }, { + // base price is $0.25 (x402.BaseProfileCostMicro). With Twitter + // enabled the tool is advertised as upto at the maximum possible + // cost - $0.25 base + $0.01 lookup + $0.005/post at the 100-post + // cap = $0.76 (x402.MaxTwitterCostMicro(BaseProfileCostMicro, + // MaxProfileCountedPosts)) - and the actual charge is settled from + // the posts read via x402.NewStyleProfileCostResolver. Name: "getStyleProfile", Desc: "Get an actor's style Profile", - Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.36", Scheme: "upto", + Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.25", + MaxTimeoutSeconds: 600, + UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -109,7 +130,8 @@ func (s *Server) registerTools() { }, { Name: "findSimilarUsers", Desc: "Find authors with a similar writing style to the given author, across all platforms", - Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.05", + Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.10", + UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -147,6 +169,7 @@ func (s *Server) registerTools() { { Name: "getEngagementMetrics", Desc: "Get time-series engagement metrics (likes, reposts, replies, quotes) per post", Input: AnalyticsRequest{}, Output: EngagementMetricsResponse{}, Handler: s.getEngagementMetrics, Price: "0.15", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "metrics": Bodyfield{Type: "array", Description: "Engagement metrics per post"}, @@ -155,6 +178,7 @@ func (s *Server) registerTools() { { Name: "getPostingPatterns", Desc: "Get posting heatmap and patterns (hour-of-day x day-of-week)", Input: AnalyticsRequest{}, Output: PostingPatternsResponse{}, Handler: s.getPostingPatterns, Price: "0.10", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "heatmap": Bodyfield{Type: "array", Description: "24x7 matrix: hour (0-23) x day of week (0-6 = Sun-Sat)"}, @@ -168,6 +192,7 @@ func (s *Server) registerTools() { { Name: "getTopicModeling", Desc: "Get LLM-based topic extraction and clustering from posts", Input: AnalyticsRequest{}, Output: TopicModelingResponse{}, Handler: s.getTopicModeling, Price: "0.20", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "topics": Bodyfield{Type: "array", Description: "Identified topic clusters"}, @@ -176,6 +201,7 @@ func (s *Server) registerTools() { { Name: "getSentimentTrends", Desc: "Get sentiment analysis and emotional arcs over time", Input: AnalyticsRequest{}, Output: SentimentTrendsResponse{}, Handler: s.getSentimentTrends, Price: "0.15", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "timeSeries": Bodyfield{Type: "array", Description: "Sentiment over time"}, @@ -187,6 +213,7 @@ func (s *Server) registerTools() { { Name: "getEngagementDrivers", Desc: "Get topics/keywords that correlate with high engagement", Input: AnalyticsRequest{}, Output: EngagementDriversResponse{}, Handler: s.getEngagementDrivers, Price: "0.20", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "drivers": Bodyfield{Type: "array", Description: "Topics/keywords driving engagement"}, @@ -195,14 +222,29 @@ func (s *Server) registerTools() { } for _, d := range payable { - scheme := d.Scheme - if scheme == "" { - scheme = "exact" + // Twitter-capable tools are advertised as upto (variable-priced) only + // when the Twitter API is actually enabled: the upto maximum adds the + // realized Twitter lookup and per-post costs, and the actual charge is + // settled from the response. Without Twitter, every tool is exact at + // its flat base price. + scheme := "exact" + price := d.Price + var resolver x402.SettleAmountResolver + if d.UsesTwitter && s.twitterEnabled { + scheme = "upto" + baseMicro := x402.MicroFromPriceString(d.Price) + price = x402.FormatPriceMicro(x402.MaxTwitterCostMicro(baseMicro, d.TwitterPostCap)) + if d.Name == "getStyleProfile" { + resolver = x402.NewStyleProfileCostResolver() + } else { + resolver = x402.NewTwitterCountCostResolver(baseMicro) + } } s.mcpHandler.AddPayableTool( mcp.Tool{Name: d.Name, Description: d.Desc, InputSchema: d.Input, OutputSchema: d.Output}, d.Handler, - s.buildRequirements(d.Price, d.Extra, scheme)..., + resolver, + s.buildRequirements(price, d.Extra, scheme, d.MaxTimeoutSeconds)..., ) } @@ -333,7 +375,7 @@ func (s *Server) getPostsInTime(ctx context.Context, request *mcp.CallToolReques if authorID == "" { authorID = args.Author } - resp := &GetPostsInTimeResponse{ + resp := GetPostsInTimeResponse{ Author: authorID, Posts: posts, } diff --git a/internal/server/types.go b/internal/server/types.go index b1915b7..7d833c6 100644 --- a/internal/server/types.go +++ b/internal/server/types.go @@ -26,8 +26,8 @@ type GetLatestPostsRequest struct { } type GetLatestPostsResponse struct { - Author string `json:"actor" jsonschema_description:"Did of the actor"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` + Author string `json:"actor" jsonschema_description:"Did of the actor"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` } type GetPostsInTimeRequest struct { @@ -38,8 +38,8 @@ type GetPostsInTimeRequest struct { } type GetPostsInTimeResponse struct { - Author string `json:"author" jsonschema_description:"Did of the author"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` + Author string `json:"author" jsonschema_description:"Did of the author"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` } type GetPostRequest struct { @@ -63,8 +63,8 @@ type GetFeedRequest struct { } type GetFeedResponse struct { - URI string `json:"uri" jsonschema_description:"The URI of the feed"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` + URI string `json:"uri" jsonschema_description:"The URI of the feed"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` } type GetStyleProfileRequest struct { @@ -87,6 +87,10 @@ type FindSimilarUsersRequest struct { type FindSimilarUsersResponse struct { Matches []SimilarUser `json:"matches" jsonschema_description:"Ranked list of similar profiles by persona embedding similarity"` + // TwitterPostCount reports how many posts were sourced from the Twitter API + // while building the query author's style profile. It is 0 when the query + // author is not on Twitter. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts used for the query author's profile"` } type ActorProfile = actor.Profile @@ -105,7 +109,7 @@ type SearchActorsRequest struct { } type SearchActorsResponse struct { - Results []ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` + Results []*ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` } type SimilarUser struct { @@ -165,6 +169,10 @@ type AnalyticsRequest struct { type EngagementMetricsResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Metrics []EngagementMetricPoint `json:"metrics" jsonschema_description:"Engagement metrics per post"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementMetricPoint struct { @@ -185,11 +193,19 @@ type PostingPatternsResponse struct { AvgPostsPerDay float64 `json:"avgPostsPerDay" jsonschema_description:"Average posts per day"` TotalPosts int `json:"totalPosts" jsonschema_description:"Total posts in time range"` PostingDays int `json:"postingDays" jsonschema_description:"Number of days with at least one post"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicModelingResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Topics []TopicCluster `json:"topics" jsonschema_description:"Identified topic clusters"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicCluster struct { @@ -207,6 +223,10 @@ type SentimentTrendsResponse struct { OverallSentiment SentimentSummary `json:"overallSentiment" jsonschema_description:"Overall sentiment summary"` DominantEmotions []EmotionScore `json:"dominantEmotions" jsonschema_description:"Most frequent emotions"` SentimentShifts []SentimentShift `json:"sentimentShifts" jsonschema_description:"Significant sentiment changes"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type SentimentPoint struct { @@ -238,6 +258,10 @@ type SentimentShift struct { type EngagementDriversResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Drivers []EngagementDriver `json:"drivers" jsonschema_description:"Topics/keywords driving engagement"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementDriver struct { diff --git a/internal/server/webhook.go b/internal/server/webhook.go index 75708c1..adda05e 100644 --- a/internal/server/webhook.go +++ b/internal/server/webhook.go @@ -47,7 +47,12 @@ func (wh *WebhookSender) Send(ctx context.Context, jsonBytes []byte) { for attempt := 1; attempt <= 5; attempt++ { delay := exponentialBackoff(attempt) if attempt > 1 { - time.Sleep(delay) + select { + case <-time.After(delay): + case <-ctx.Done(): + log.WarnWithContext(ctx, "webhook cancelled during backoff", "attempt", attempt, "error", ctx.Err()) + return + } } err := wh.sendAttempt(ctx, jsonBytes, attempt) diff --git a/internal/store/integration_test.go b/internal/store/integration_test.go index 2adc0b4..5c9aeaf 100644 --- a/internal/store/integration_test.go +++ b/internal/store/integration_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "os" + "reflect" "strings" "testing" "time" @@ -12,93 +13,140 @@ import ( _ "github.com/lib/pq" ) -// TestStoreRoundTrip exercises the sqlc queries against a real Postgres+pgvector database. -// It is skipped unless TEST_DATABASE_URL is set (e.g. a local instance with the vector -// extension available), so it never fails in environments without a database. -func TestStoreRoundTrip(t *testing.T) { +// setupTestDB handles connection pooling, migrations, and cleanup per test run. +func setupTestDB(t *testing.T) (*sql.DB, *Queries) { + t.Helper() + dsn := os.Getenv("TEST_DATABASE_URL") if dsn == "" { t.Skip("TEST_DATABASE_URL not set; skipping store integration test") } + ctx := context.Background() conn, err := sql.Open("postgres", dsn) if err != nil { - t.Fatal(err) + t.Fatalf("failed to open db: %v", err) } - defer conn.Close() + if err := conn.PingContext(ctx); err != nil { + _ = conn.Close() t.Fatalf("cannot reach test database: %v", err) } + + t.Cleanup(func() { + _ = conn.Close() + }) + if err := Migrate(ctx, conn, "sql/schema"); err != nil { t.Fatalf("migrate: %v", err) } - q := New(conn) + + return conn, New(conn) +} + +func makeEmbedding(dim int, val string) string { + return "[" + strings.TrimSuffix(strings.Repeat(val+",", dim), ",") + "]" +} + +func TestStoreRoundTrip(t *testing.T) { + conn, q := setupTestDB(t) + ctx := context.Background() uid := fmt.Sprintf("test_%d", time.Now().UnixNano()) - emb := "[" + strings.TrimSuffix(strings.Repeat("0.0,", 1024), ",") + "]" + platform := "Bluesky" + emb1 := makeEmbedding(1024, "0.1") - if err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ + // Clean up after the test so leftover records do not pollute subsequent runs + t.Cleanup(func() { + _, _ = conn.ExecContext(context.Background(), "DELETE FROM style_profiles WHERE user_id = $1", uid) + }) + + // 1. Test INSERT path of Upsert + err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ UserID: uid, - Platform: "Bluesky", - Embedding: emb, + Platform: platform, + Embedding: emb1, TopKeywords: []string{"alpha", "beta"}, - StyleSummary: "summary", + StyleSummary: "original summary", PostCount: 5, - }); err != nil { - t.Fatalf("upsert: %v", err) + }) + if err != nil { + t.Fatalf("initial upsert failed: %v", err) } - p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: "Bluesky"}) + // 2. Test GET and verify fields + p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) if err != nil { - t.Fatalf("get: %v", err) - } - if p.UserID != uid || p.Platform != "Bluesky" || p.PostCount != 5 { - t.Fatalf("unexpected profile: %+v", p) + t.Fatalf("get failed: %v", err) } - rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{QueryEmbedding: emb, LimitVal: 10}) - if err != nil { - t.Fatalf("find similar: %v", err) + if p.UserID != uid || p.Platform != platform || p.PostCount != 5 { + t.Errorf("mismatched basic fields: %+v", p) } + if p.StyleSummary != "original summary" { + t.Errorf("got summary %q, want %q", p.StyleSummary, "original summary") + } + if !reflect.DeepEqual(p.TopKeywords, []string{"alpha", "beta"}) { + t.Errorf("got keywords %v, want %v", p.TopKeywords, []string{"alpha", "beta"}) + } + + // 3. Test UPDATE path of Upsert (Verify conflict resolution works) + emb2 := makeEmbedding(1024, "0.2") + err = q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ + UserID: uid, + Platform: platform, + Embedding: emb2, + TopKeywords: []string{"gamma"}, + StyleSummary: "updated summary", + PostCount: 10, + }) + if err != nil { + t.Fatalf("secondary upsert (update) failed: %v", err) + } + + updated, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) + if err != nil { + t.Fatalf("get updated profile failed: %v", err) + } + if updated.PostCount != 10 || updated.StyleSummary != "updated summary" { + t.Errorf("upsert failed to update existing record: %+v", updated) + } + + // 4. Test pgvector similarity search + rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{ + QueryEmbedding: emb2, + LimitVal: 10, + }) + if err != nil { + t.Fatalf("find similar failed: %v", err) + } + found := false for _, r := range rows { if r.UserID == uid { found = true + break } } if !found { - t.Fatalf("expected newly inserted profile to be returned by similarity search") + t.Errorf("expected user %s in similarity results, but was not found", uid) } } -// TestStoreSaveTransaction exercises SaveTransaction against a real Postgres database. -// It is skipped unless TEST_DATABASE_URL is set. func TestStoreSaveTransaction(t *testing.T) { - dsn := os.Getenv("TEST_DATABASE_URL") - if dsn == "" { - t.Skip("TEST_DATABASE_URL not set; skipping store integration test") - } + conn, q := setupTestDB(t) ctx := context.Background() - conn, err := sql.Open("postgres", dsn) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - if err := conn.PingContext(ctx); err != nil { - t.Fatalf("cannot reach test database: %v", err) - } - if err := Migrate(ctx, conn, "sql/schema"); err != nil { - t.Fatalf("migrate: %v", err) - } - q := New(conn) - txHash := fmt.Sprintf("0x_%d", time.Now().Unix()) + txHash := fmt.Sprintf("0x_%d", time.Now().UnixNano()) amount := "10.0" asset := "ETH" network := "eip155:1" - now := time.Now().Unix() - recipient := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000), "0") - sender := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000+0xab), "0") + recipient := "0x1234567890abcdef" + sender := "0xabcdef1234567890" + + t.Cleanup(func() { + _, _ = conn.ExecContext(context.Background(), "DELETE FROM transactions WHERE x402_transaction_hash = $1", txHash) + }) ret, err := q.SaveTransaction(ctx, SaveTransactionParams{ X402TransactionHash: txHash, @@ -113,22 +161,34 @@ func TestStoreSaveTransaction(t *testing.T) { } if len(ret) != 1 { - t.Fatalf("expected 1 result, got %d", len(ret)) + t.Fatalf("expected 1 result row returned, got %d", len(ret)) } - id := ret[0].ID - if id <= 0 { - t.Fatalf("expected positive ID, got %d", id) + saved := ret[0] + if saved.ID <= 0 { + t.Errorf("expected positive ID, got %d", saved.ID) + } + if !saved.CreatedAt.Valid { + t.Error("expected CreatedAt to be set") } - // Verify the returned row has the expected fields - if !ret[0].CreatedAt.Valid { - t.Fatal("expected CreatedAt to be set") + // Since the sqlc query only returns `id` and `created_at`, + // verify directly that PostgreSQL stored the remaining columns accurately: + var ( + gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender string + ) + err = conn.QueryRowContext(ctx, ` + SELECT x402_transaction_hash, amount, asset, network, recipient, sender + FROM transactions + WHERE id = $1 + `, saved.ID).Scan(&gotHash, &gotAmount, &gotAsset, &gotNetwork, &gotRecipient, &gotSender) + if err != nil { + t.Fatalf("failed to query inserted transaction from db: %v", err) } - // Verify we can retrieve the transaction back - // Note: we'd need a SELECT query for full verification, but the insert - // returning id and created_at is the critical path tested here - - t.Logf("SaveTransaction successful: id=%d, hash=%s", id, txHash) + if gotHash != txHash || gotAmount != amount || gotAsset != asset || + gotNetwork != network || gotRecipient != recipient || gotSender != sender { + t.Errorf("stored values do not match input params: got (%s, %s, %s, %s, %s, %s)", + gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender) + } } diff --git a/internal/twitter/client.go b/internal/twitter/client.go index 08e97e2..f849796 100644 --- a/internal/twitter/client.go +++ b/internal/twitter/client.go @@ -25,6 +25,12 @@ func parseTwitterTime(s string) time.Time { const apiBase = "https://api.x.com/2" +// httpTimeout bounds a single HTTP request so a stalled X backend cannot block +// a request indefinitely. +const httpTimeout = 60 * time.Second + +const userAgent = "naco-api/1.0" + type Client struct { httpClient *http.Client bearerToken string @@ -32,7 +38,7 @@ type Client struct { func NewTwitterClient(bearerToken string) *Client { return &Client{ - httpClient: http.DefaultClient, + httpClient: &http.Client{Timeout: httpTimeout}, bearerToken: bearerToken, } } @@ -105,95 +111,150 @@ type singleTweetResponse struct { } `json:"errors"` } -// doGet issues a GET and retries with exponential backoff when Twitter returns HTTP 429 -// (Too Many Requests). It honors the x-rate-limit-* headers and caps retries so a -// sustained rate limit still surfaces as a real error to the caller (and thus isn't billed). +const ( + maxRetries = 4 + maxRateLimitWait = 30 * time.Second // Never stall a request longer than 30s +) + +var defaultBackoff = 500 * time.Millisecond + func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { - const maxRetries = 4 - backoff := 500 * time.Millisecond + backoff := defaultBackoff + for attempt := 0; ; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, fmt.Errorf("failed to build request: %w", err) } req.Header.Set("Authorization", "Bearer "+c.bearerToken) + req.Header.Set("User-Agent", userAgent) resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to do request: %w", err) + // Transport-level failure (connection refused/reset, timeout). + if attempt >= maxRetries { + return nil, fmt.Errorf("twitter request failed after %d attempts: %w", maxRetries+1, err) + } + log.WarnWithContext(ctx, "twitter transport error, waiting to retry", "attempt", attempt+1, "error", err) + if werr := waitForRetry(ctx, backoff); werr != nil { + return nil, werr + } + backoff *= 2 + continue } - body, err := io.ReadAll(resp.Body) - resp.Body.Close() + + // Handle 429 Too Many Requests + if resp.StatusCode == http.StatusTooManyRequests { + // Drain & close body before retrying. + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + if attempt >= maxRetries { + return nil, fmt.Errorf("twitter rate limit exceeded, max retries reached") + } + + wait := parseRateLimitWait(resp.Header, backoff) + + // Protect against 15-minute thread freezes + if wait > maxRateLimitWait { + return nil, fmt.Errorf("twitter rate limited: reset wait (%s) exceeds maximum allowed (%s)", wait, maxRateLimitWait) + } + + log.WarnWithContext(ctx, "twitter rate limited, waiting to retry", "attempt", attempt+1, "wait", wait.String()) + if werr := waitForRetry(ctx, wait); werr != nil { + return nil, werr + } + backoff *= 2 + continue + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) + _ = resp.Body.Close() if err != nil { return nil, fmt.Errorf("failed to read response: %w", err) } - if resp.StatusCode == http.StatusTooManyRequests { - if attempt >= maxRetries { - return nil, fmt.Errorf("twitter rate limited after retries: %s", resp.Status) - } - // Honor x-rate-limit-* headers first - remaining := resp.Header.Get("x-rate-limit-remaining") - reset := resp.Header.Get("x-rate-limit-reset") - if ra := remaining; ra != "" { - if rem, perr := strconv.Atoi(ra); perr == nil { - if rem <= 0 && reset != "" { - if rst, perr2 := strconv.Atoi(reset); perr2 == nil { - // Wait until the rate limit resets (Unix timestamp) - now := time.Now().Unix() - rst64 := int64(rst) - wait := time.Duration(rst64-now) * time.Second - if wait > 0 { - log.WarnWithContext(ctx, "twitter rate limited, waiting until reset", "reset", reset, "wait", wait.String()) - select { - case <-time.After(wait): - case <-ctx.Done(): - return nil, ctx.Err() - } - backoff = 500 * time.Millisecond - continue - } - } - } - } - } - // Fall back to Retry-After if headers not useful - if ra := resp.Header.Get("Retry-After"); ra != "" { - if secs, perr := strconv.Atoi(ra); perr == nil && secs > 0 { - backoff = time.Duration(secs) * time.Second - } - } - log.WarnWithContext(ctx, "twitter rate limited, backing off", "attempt", attempt+1, "wait", backoff.String()) - select { - case <-time.After(backoff): - case <-ctx.Done(): - return nil, ctx.Err() + // Retry transient server-side failures (5xx) with backoff. 4xx/3xx fail + // fast below. + if resp.StatusCode >= http.StatusInternalServerError && attempt < maxRetries { + log.WarnWithContext(ctx, "twitter server error, waiting to retry", "attempt", attempt+1, "status", resp.Status) + if werr := waitForRetry(ctx, backoff); werr != nil { + return nil, werr } backoff *= 2 continue } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("twitter error: %s: %s", resp.Status, string(body)) + errMsg := strings.TrimSpace(string(body)) + if len(errMsg) > 512 { + errMsg = errMsg[:512] + "... (truncated)" + } + return nil, fmt.Errorf("twitter error (%s): %s", resp.Status, errMsg) } + return body, nil } } -func (c *Client) resolveUser(ctx context.Context, username string) (string, error) { +// waitForRetry sleeps for wait, aborting early if the context is canceled. +// A timer is used (not time.After) so the timer can be stopped to free its +// resource when the caller abandons the wait. +func waitForRetry(ctx context.Context, wait time.Duration) error { + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// parseRateLimitWait extracts x-rate-limit-* or Retry-After, falling back to backoff. +func parseRateLimitWait(header http.Header, fallback time.Duration) time.Duration { + // 1. Check x-rate-limit-reset (Unix timestamp) + if remStr := header.Get("x-rate-limit-remaining"); remStr != "" { + if rem, err := strconv.Atoi(remStr); err == nil && rem <= 0 { + if rstStr := header.Get("x-rate-limit-reset"); rstStr != "" { + if rst, err := strconv.ParseInt(rstStr, 10, 64); err == nil { + // Add 1s clock skew buffer + wait := time.Until(time.Unix(rst, 0)) + (1 * time.Second) + if wait > 0 { + return wait + } + } + } + } + } + + // 2. Check standard Retry-After (seconds) + if ra := header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + return time.Duration(secs)*time.Second + (1 * time.Second) + } + } + + return fallback +} + +func (c *Client) resolveUser(ctx context.Context, username string) (id, handle string, err error) { endpoint := fmt.Sprintf("%s/users/by/username/%s", apiBase, url.PathEscape(username)) body, err := c.doGet(ctx, endpoint) if err != nil { - return "", err + return "", "", err } var ur userResponse if err := json.Unmarshal(body, &ur); err != nil { - return "", fmt.Errorf("failed to unmarshal user response: %w", err) + return "", "", fmt.Errorf("failed to unmarshal user response: %w", err) } if len(ur.Errors) > 0 { - return "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) + return "", "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) } - return ur.Data.ID, nil + if ur.Data.ID == "" { + return "", "", fmt.Errorf("twitter user %q not found", username) + } + return ur.Data.ID, ur.Data.Username, nil } func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { @@ -202,7 +263,7 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } username := strings.TrimPrefix(actor, "@") - userID, err := c.resolveUser(ctx, username) + userID, handle, err := c.resolveUser(ctx, username) if err != nil { return nil, err } @@ -210,40 +271,10 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( if limit > 100 { limit = 100 } - if limit < 5 { - limit = 5 - } endpoint := fmt.Sprintf("%s/users/%s/tweets?max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", apiBase, url.PathEscape(userID), limit) - body, err := c.doGet(ctx, endpoint) - if err != nil { - return nil, err - } - - var tr tweetsResponse - if err := json.Unmarshal(body, &tr); err != nil { - return nil, fmt.Errorf("failed to unmarshal tweets response: %w", err) - } - if len(tr.Errors) > 0 { - return nil, fmt.Errorf("twitter error: %s: %s", tr.Errors[0].Title, tr.Errors[0].Detail) - } - - posts := make([]*Post, 0, len(tr.Data)) - for _, t := range tr.Data { - created := parseTwitterTime(t.CreatedAt) - posts = append(posts, &Post{ - ID: t.ID, - Author: userID, - Content: t.Text, - CreatedAt: created, - Likes: t.PublicMetrics.LikeCount, - Reposts: t.PublicMetrics.RetweetCount, - Quotes: t.PublicMetrics.QuoteCount, - Replies: t.PublicMetrics.ReplyCount, - }) - } - return posts, nil + return c.fetchTweets(ctx, handle, endpoint) } func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { @@ -252,19 +283,22 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e } username := strings.TrimPrefix(actor, "@") - userID, err := c.resolveUser(ctx, username) + userID, handle, err := c.resolveUser(ctx, username) if err != nil { return nil, err } if limit > 100 { limit = 100 } - if limit < 5 { - limit = 5 - } endpoint := fmt.Sprintf("%s/users/%s/tweets?start_time=%s&end_time=%s&max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", apiBase, url.PathEscape(userID), start.Format(time.RFC3339), end.Format(time.RFC3339), limit) + return c.fetchTweets(ctx, handle, endpoint) +} + +// fetchTweets performs the request and converts the tweet payload into Posts, +// skipping entries without an ID so downstream dedupe keys are never empty. +func (c *Client) fetchTweets(ctx context.Context, username, endpoint string) ([]*Post, error) { body, err := c.doGet(ctx, endpoint) if err != nil { return nil, err @@ -280,12 +314,14 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e posts := make([]*Post, 0, len(tr.Data)) for _, t := range tr.Data { - created := parseTwitterTime(t.CreatedAt) + if t.ID == "" { + continue + } posts = append(posts, &Post{ ID: t.ID, - Author: userID, + Author: username, Content: t.Text, - CreatedAt: created, + CreatedAt: parseTwitterTime(t.CreatedAt), Likes: t.PublicMetrics.LikeCount, Reposts: t.PublicMetrics.RetweetCount, Quotes: t.PublicMetrics.QuoteCount, @@ -316,32 +352,18 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { if tr.Data.ID == "" { return nil, fmt.Errorf("tweet %q not found", id) } - author := id + author := "unknown" if len(tr.Includes.Users) > 0 { author = tr.Includes.Users[0].Username } - created, _ := time.Parse(time.RFC3339, tr.Data.CreatedAt) return &Post{ ID: tr.Data.ID, Author: author, Content: tr.Data.Text, - CreatedAt: created, + CreatedAt: parseTwitterTime(tr.Data.CreatedAt), Likes: tr.Data.PublicMetrics.LikeCount, Reposts: tr.Data.PublicMetrics.RetweetCount, Quotes: tr.Data.PublicMetrics.QuoteCount, Replies: tr.Data.PublicMetrics.ReplyCount, }, nil } - -type userFields struct { - ID string `json:"id"` - Name string `json:"name"` - Username string `json:"username"` - Description string `json:"description"` - ProfileImageURL string `json:"profile_image_url"` - PublicMetrics struct { - FollowersCount int `json:"followers_count"` - FollowingCount int `json:"following_count"` - TweetCount int `json:"tweet_count"` - } `json:"public_metrics"` -} diff --git a/internal/twitter/client_test.go b/internal/twitter/client_test.go index 9dca301..87a6024 100644 --- a/internal/twitter/client_test.go +++ b/internal/twitter/client_test.go @@ -2,14 +2,22 @@ package twitter import ( "context" + "errors" "fmt" + "io" "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" ) +// Speed up test suite: override defaultBackoff to 1ms so retries are instantaneous. +func init() { + defaultBackoff = 1 * time.Millisecond +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } @@ -46,7 +54,8 @@ func TestGetLatestNPosts(t *testing.T) { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -54,7 +63,8 @@ func TestGetLatestNPosts(t *testing.T) { return } tweetsURL = r.URL.String() - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -67,7 +77,7 @@ func TestGetLatestNPosts(t *testing.T) { if len(posts) != 1 { t.Fatalf("expected 1 post, got %d", len(posts)) } - if posts[0].Content != "hello world" || posts[0].Author != "12345" { + if posts[0].Content != "hello world" || posts[0].Author != "testuser" { t.Fatalf("unexpected post: %+v", posts[0]) } if !strings.Contains(tweetsURL, "max_results=10") { @@ -75,16 +85,18 @@ func TestGetLatestNPosts(t *testing.T) { } } -func TestGetLatestNPostsLimitClamp(t *testing.T) { +func TestGetLatestNPostsLimitNoLowerFloor(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"1","username":"u"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) }) mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { - if got := r.URL.Query().Get("max_results"); got != "5" { - t.Fatalf("expected clamped max_results=5 for limit=3, got %q", got) + if got := r.URL.Query().Get("max_results"); got != "3" { + t.Fatalf("expected max_results=3 for limit=3, got %q", got) } - fmt.Fprint(w, `{"data":[]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -110,14 +122,16 @@ func TestGetLatestNPostsHTTPError(t *testing.T) { func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/tweets/999", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -138,7 +152,8 @@ func TestGetPost(t *testing.T) { func TestGetPostsInTimeframe(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -148,7 +163,8 @@ func TestGetPostsInTimeframe(t *testing.T) { if r.URL.Query().Get("start_time") == "" || r.URL.Query().Get("end_time") == "" { t.Fatalf("expected start_time and end_time query params") } - fmt.Fprint(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -174,54 +190,19 @@ func TestRateLimitRetryWithResetHeader(t *testing.T) { mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ if callCount == 1 { - // First call: rate limited with reset header + // First call: rate limited with reset header slightly in the past/present w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(100*time.Millisecond).Unix())) + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Unix())) w.WriteHeader(http.StatusTooManyRequests) return } // Second call: success - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - c := newTestClient(server.URL) - // Use a short context timeout to avoid waiting too long - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - posts, err := c.GetLatestNPosts(ctx, "@testuser", 10) - if err != nil { - t.Fatalf("GetLatestNPosts error after retry: %v", err) - } - if callCount != 2 { - t.Fatalf("expected 2 calls (1 retry), got %d", callCount) - } - if len(posts) != 1 { - t.Fatalf("expected 1 post after retry, got %d", len(posts)) - } -} - -func TestRateLimitRetryWithRetryAfterHeader(t *testing.T) { - callCount := 0 - mux := http.NewServeMux() - mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - callCount++ - if callCount == 1 { - // First call: rate limited with Retry-After header - w.Header().Set("Retry-After", "1") // 1 second - w.WriteHeader(http.StatusTooManyRequests) - return - } - // Second call: success - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) - }) - mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -247,29 +228,25 @@ func TestRateLimitExhaustedAfterMaxRetries(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ - // Always rate limited - no reset header, no Retry-After, so exponential backoff is used w.Header().Set("x-rate-limit-remaining", "0") - // Don't set x-rate-limit-reset or Retry-After to force exponential backoff w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Use a timeout long enough for all retries (4 retries with exponential backoff: 500ms, 1s, 2s, 4s = ~7.5s) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error after max retries exhausted") } - // Should have tried maxRetries (4) + 1 initial = 5 times if callCount < 5 { t.Fatalf("expected at least 5 attempts (1 initial + 4 retries), got %d", callCount) } - if !strings.Contains(err.Error(), "rate limited after retries") { - t.Fatalf("expected 'rate limited after retries' error, got %v", err) + if !strings.Contains(err.Error(), "rate limit") { + t.Fatalf("expected rate limit error, got %v", err) } } @@ -277,7 +254,8 @@ func TestUserLookupNotFound(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - fmt.Fprint(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -296,7 +274,8 @@ func TestUserLookupUnauthorized(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - fmt.Fprint(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -314,65 +293,153 @@ func TestUserLookupUnauthorized(t *testing.T) { func TestUserLookupRateLimited(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "1") w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Need enough time for max retries (4 retries * 1 second = 4 seconds) + initial - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error for rate limited user lookup") } - if !strings.Contains(err.Error(), "twitter error") && !strings.Contains(err.Error(), "rate limited after retries") { - t.Fatalf("expected twitter error or rate limited after retries, got %v", err) + if !strings.Contains(err.Error(), "rate limit") { + t.Fatalf("expected rate limit error, got %v", err) } } func TestContextCancellationDuringRetry(t *testing.T) { - callCount := 0 + reqReceived := make(chan struct{}) + var once sync.Once + mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - callCount++ - // First call: rate limited with long reset time - if callCount == 1 { - w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) - w.WriteHeader(http.StatusTooManyRequests) - return - } + w.Header().Set("x-rate-limit-remaining", "0") + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) + w.WriteHeader(http.StatusTooManyRequests) + once.Do(func() { + close(reqReceived) // Signal test that the 429 response was returned + }) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Cancel context immediately after first call ctx, cancel := context.WithCancel(context.Background()) - // Start the request in a goroutine + done := make(chan error, 1) go func() { _, err := c.GetLatestNPosts(ctx, "@testuser", 10) done <- err }() - // Give it a moment to make the first call, then cancel - time.Sleep(100 * time.Millisecond) - cancel() + // Deterministic wait: block until the mock server receives the request, then cancel immediately + select { + case <-reqReceived: + cancel() + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for mock request to be hit") + } select { case err := <-done: if err == nil { t.Fatal("expected error from context cancellation") } - if !strings.Contains(err.Error(), "context canceled") && !strings.Contains(err.Error(), "context cancelled") { - t.Fatalf("expected context cancellation error, got %v", err) + if !errors.Is(err, context.Canceled) && !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("expected context.Canceled, got %v", err) } case <-time.After(2 * time.Second): - t.Fatal("test timed out waiting for cancellation") + t.Fatal("test timed out waiting for goroutine to abort") + } +} + +func TestResolveUserEmptyData(t *testing.T) { + // 200 with an empty data object (banned/suspended or privacy-restricted + // account) must error rather than build a bogus /users//tweets request. + var tweetsCalled bool + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{}}`) + }) + mux.HandleFunc("/2/users//tweets", func(w http.ResponseWriter, r *http.Request) { + tweetsCalled = true + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + _, err := c.GetLatestNPosts(context.Background(), "@ghost", 10) + if err == nil { + t.Fatal("expected error for empty user data, got nil") + } + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected 'not found' error, got %v", err) + } + if tweetsCalled { + t.Fatal("tweets endpoint should not be called when user lookup returns empty data") + } +} + +func TestGetLatestNPostsSkipsEmptyID(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) + }) + mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"","text":"bad"},{"id":"2","text":"good","created_at":"2024-01-01T00:00:00Z"}]}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + posts, err := c.GetLatestNPosts(context.Background(), "u", 10) + if err != nil { + t.Fatalf("error: %v", err) + } + if len(posts) != 1 { + t.Fatalf("expected 1 post (empty-ID tweet skipped), got %d", len(posts)) + } + if posts[0].ID != "2" || posts[0].Content != "good" { + t.Fatalf("unexpected post: %+v", posts[0]) + } +} + +func TestServerErrorRetried(t *testing.T) { + callCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + callCount++ + if callCount == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `{"title":"Internal Server Error","detail":"transient"}`) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) + }) + mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"ok","created_at":"2024-01-01T00:00:00Z"}]}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + posts, err := c.GetLatestNPosts(context.Background(), "u", 10) + if err != nil { + t.Fatalf("GetLatestNPosts error after 5xx retry: %v", err) + } + if callCount != 2 { + t.Fatalf("expected 2 calls (1 failure + 1 retry), got %d", callCount) + } + if len(posts) != 1 || posts[0].Content != "ok" { + t.Fatalf("unexpected posts after retry: %+v", posts) } } diff --git a/internal/utils/styleProfiles.go b/internal/utils/styleProfiles.go index 7a6a587..cb475a4 100644 --- a/internal/utils/styleProfiles.go +++ b/internal/utils/styleProfiles.go @@ -56,17 +56,34 @@ func FormatVector(v []float64) string { return "[" + strings.Join(parts, ",") + "]" } +// AverageVector returns the element-wise mean of vs. Zero-length vectors (e.g. an +// embedding model returning an empty vector for a post without text) are skipped, and +// the accumulator is sized from the first non-empty vector so a leading empty vector +// cannot panic later in the loop. Returns nil when no usable vector is present. func AverageVector(vs [][]float64) []float64 { - dim := len(vs[0]) - avg := make([]float64, dim) - + dim := 0 + var valid [][]float64 for _, v := range vs { - for i, val := range v { - avg[i] += val + if len(v) == 0 { + continue + } + if dim == 0 { + dim = len(v) + } + valid = append(valid, v) + } + if len(valid) == 0 { + return nil + } + + avg := make([]float64, dim) + for _, v := range valid { + for i := 0; i < dim && i < len(v); i++ { + avg[i] += v[i] } } for i := range avg { - avg[i] /= float64(len(vs)) + avg[i] /= float64(len(valid)) } return avg } diff --git a/internal/utils/styleProfiles_test.go b/internal/utils/styleProfiles_test.go new file mode 100644 index 0000000..4692b95 --- /dev/null +++ b/internal/utils/styleProfiles_test.go @@ -0,0 +1,47 @@ +package utils + +import ( + "math" + "testing" +) + +func assertVectorsEqual(t *testing.T, got, want []float64) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("length %d, want %d", len(got), len(want)) + } + for i := range want { + if math.Abs(got[i]-want[i]) > 1e-9 { + t.Fatalf("index %d = %v, want %v", i, got, want) + } + } +} + +func TestAverageVector(t *testing.T) { + tests := []struct { + name string + in [][]float64 + want []float64 + }{ + {name: "empty input", in: nil, want: nil}, + {name: "empty slice", in: [][]float64{}, want: nil}, + {name: "all empty vectors", in: [][]float64{{}, {}, {}}, want: nil}, + {name: "single vector", in: [][]float64{{0.4, -0.2}}, want: []float64{0.4, -0.2}}, + {name: "multiple vectors", in: [][]float64{{0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "first vector empty", in: [][]float64{{}, {0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "trailing empty vectors", in: [][]float64{{0.1, 0.2}, {}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "only first valid", in: [][]float64{{}, {}, {0.1, 0.2}}, want: []float64{0.1, 0.2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := AverageVector(tt.in) + if tt.want == nil { + if got != nil { + t.Fatalf("AverageVector(%v) = %v, want nil", tt.in, got) + } + return + } + assertVectorsEqual(t, got, tt.want) + }) + } +} diff --git a/internal/x402/mcpwrapper.go b/internal/x402/mcpwrapper.go new file mode 100644 index 0000000..8d69b01 --- /dev/null +++ b/internal/x402/mcpwrapper.go @@ -0,0 +1,156 @@ +package x402 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/modelcontextprotocol/go-sdk/mcp" + log "github.com/nacorid/logger" + x402 "github.com/x402-foundation/x402/go/v2" + "github.com/x402-foundation/x402/go/v2/types" +) + +// Payment meta keys exchanged over MCP, mirroring the x402 SDK contract. +const ( + PaymentMetaKey = "x402/payment" + PaymentResponseMetaKey = "x402/payment-response" +) + +// SettleAmountResolver computes the actual settlement amount for a completed +// tool call. It returns ("", false) to settle the amount advertised in the +// payment requirements as-is. +type SettleAmountResolver func(result *mcp.CallToolResult) (string, bool) + +// PaymentWrapperConfig configures a PaymentWrapper. Unlike the SDK's wrapper, +// it accepts a SettleAmountResolver so upto-scheme tools can charge the actual +// computed cost after the handler runs (the same behavior the REST path gets +// via nethttp.SetSettlementOverrides). +type PaymentWrapperConfig struct { + Accepts []x402.PaymentRequirements + Resource *types.ResourceInfo + Extensions map[string]interface{} + // SettleAmountResolver is invoked after a successful handler call. When it + // returns a non-empty amount, that amount is charged instead of the + // advertised maximum (which must therefore be <= the accept amount). + SettleAmountResolver SettleAmountResolver +} + +// PaymentWrapper gates an MCP tool behind x402 payments and settles them. +type PaymentWrapper struct { + server *x402.X402ResourceServer + config PaymentWrapperConfig +} + +// NewPaymentWrapper creates a PaymentWrapper for the given resource server. +func NewPaymentWrapper(server *x402.X402ResourceServer, config PaymentWrapperConfig) *PaymentWrapper { + if len(config.Accepts) == 0 { + panic("PaymentWrapperConfig.Accepts must have at least one payment requirement") + } + return &PaymentWrapper{server: server, config: config} +} + +// Wrap returns a ToolHandler that requires a valid x402 payment before invoking +// the wrapped handler and settles it afterwards. +func (w *PaymentWrapper) Wrap(handler mcp.ToolHandler) mcp.ToolHandler { + return func(ctx context.Context, request *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + paymentData := extractPaymentFromRequest(request) + if paymentData == nil { + return w.paymentRequiredResult("Payment Required"), nil + } + + payloadBytes, err := json.Marshal(paymentData) + if err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil + } + var payload types.PaymentPayload + if err := json.Unmarshal(payloadBytes, &payload); err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil + } + + matched := w.server.FindMatchingRequirements(w.config.Accepts, payload) + if matched == nil { + return w.paymentRequiredResult("No matching payment requirements found"), nil + } + requirements := *matched + + verifyResp, err := w.server.VerifyPayment(ctx, payload, requirements) + if err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Payment verification error: %v", err)), nil + } + if !verifyResp.IsValid { + return w.paymentRequiredResult(fmt.Sprintf("Payment verification failed: %s", verifyResp.InvalidReason)), nil + } + + result, err := handler(ctx, request) + if err != nil { + return nil, err + } + if result.IsError { + return result, nil + } + + var overrides *x402.SettlementOverrides + if w.config.SettleAmountResolver != nil { + if amount, ok := w.config.SettleAmountResolver(result); ok && amount != "" { + overrides = &x402.SettlementOverrides{Amount: amount} + log.Debug("MCP settlement override", "amount", amount) + } + } + + settleResp, err := w.server.SettlePayment(ctx, payload, requirements, overrides) + if err != nil { + return w.settlementFailedResult(fmt.Sprintf("Settlement error: %v", err)), nil + } + if !settleResp.Success { + return w.settlementFailedResult(fmt.Sprintf("Settlement failed: %s", settleResp.ErrorReason)), nil + } + + if result.Meta == nil { + result.Meta = mcp.Meta{} + } + result.Meta[PaymentResponseMetaKey] = settleResp + return result, nil + } +} + +func extractPaymentFromRequest(request *mcp.CallToolRequest) interface{} { + if request == nil || request.Params.Meta == nil { + return nil + } + return request.Params.Meta[PaymentMetaKey] +} + +func (w *PaymentWrapper) paymentRequiredResult(errorMessage string) *mcp.CallToolResult { + var resource *types.ResourceInfo + if w.config.Resource != nil { + resource = &types.ResourceInfo{ + URL: w.config.Resource.URL, + Description: w.config.Resource.Description, + } + } + payment := types.PaymentRequired{ + X402Version: 2, + Error: errorMessage, + Resource: resource, + Accepts: w.config.Accepts, + Extensions: w.config.Extensions, + } + b, err := json.Marshal(payment) + if err != nil { + panic(err) + } + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}, + } +} + +func (w *PaymentWrapper) settlementFailedResult(errorMessage string) *mcp.CallToolResult { + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{ + &mcp.TextContent{Text: fmt.Sprintf("{\"x402Version\":2,\"error\":%q}", errorMessage)}, + }, + } +} diff --git a/internal/x402/x402.go b/internal/x402/x402.go index e1cb2a6..28eeb5c 100644 --- a/internal/x402/x402.go +++ b/internal/x402/x402.go @@ -7,10 +7,13 @@ import ( "fmt" "io" "log/slog" + "math" "net" "net/http" "os" + "os/user" "reflect" + "slices" "strconv" "strings" "sync" @@ -22,9 +25,9 @@ import ( x402 "github.com/x402-foundation/x402/go/v2" x402http "github.com/x402-foundation/x402/go/v2/http" nethttp "github.com/x402-foundation/x402/go/v2/http/nethttp" - x402mcp "github.com/x402-foundation/x402/go/v2/mcp" evm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/exact/server" uptoevm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/upto/server" + "github.com/x402-foundation/x402/go/v2/types" ) type X402Middlewares struct { @@ -55,6 +58,14 @@ type X402Server struct { resourceInitMu sync.Mutex resourceInitDone bool + + // settleResolvers maps payable tool name -> SettleAmountResolver used by + // both the MCP wrapper and the REST override path to charge the actual + // computed cost (instead of the advertised upto maximum). + settleResolvers map[string]SettleAmountResolver + + httpServer *http.Server + mu sync.Mutex } const VERSION1 = "v1/" @@ -108,6 +119,7 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au tools: make([]ToolInfo, 0), globalReqLog: globalLoggingMw, networks: networks, + settleResolvers: make(map[string]SettleAmountResolver), } // Warm the facilitator cache now so AddPayableTool can build Accepts with @@ -128,7 +140,7 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au s.mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) + _, _ = w.Write([]byte("ok")) }) s.mux.HandleFunc("GET /openapi.json", s.serveOpenAPI) @@ -151,9 +163,9 @@ const facilitatorRetryTimeout = 10 * time.Second // facilitator outage at boot (or afterwards) no longer requires a restart. func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { s.resourceInitMu.Lock() - done := s.resourceInitDone - s.resourceInitMu.Unlock() - if done { + defer s.resourceInitMu.Unlock() + + if s.resourceInitDone { return } @@ -164,9 +176,7 @@ func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { return } - s.resourceInitMu.Lock() s.resourceInitDone = true - s.resourceInitMu.Unlock() } func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { @@ -190,11 +200,12 @@ func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { func (s *X402Server) AddPayableTool( tool mcp.Tool, handler mcp.ToolHandler, + resolver SettleAmountResolver, requirements ...x402.PaymentRequirements, ) { if len(requirements) == 0 { - log.Fatalf("tool %s requires at least one payment requirement", tool.Name) + panic(fmt.Sprintf("tool %s requires at least one payment requirement", tool.Name)) } // Convert each raw dollar price into the asset's smallest-unit integer amount using the @@ -212,7 +223,7 @@ func (s *X402Server) AddPayableTool( Extra: req.Extra, }) if err != nil { - log.Fatalf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err) + panic(fmt.Sprintf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err)) } mcpAccepts = append(mcpAccepts, built...) } @@ -220,9 +231,14 @@ func (s *X402Server) AddPayableTool( tool.InputSchema = toJSONSchema(tool.InputSchema) tool.OutputSchema = toJSONSchema(tool.OutputSchema) - wrapper := x402mcp.NewPaymentWrapper(s.x402ResourceServer, x402mcp.PaymentWrapperConfig{ - Accepts: mcpAccepts, - Resource: &x402mcp.ResourceInfo{ + if resolver != nil { + s.settleResolvers[tool.Name] = resolver + } + + wrapper := NewPaymentWrapper(s.x402ResourceServer, PaymentWrapperConfig{ + Accepts: mcpAccepts, + SettleAmountResolver: resolver, + Resource: &types.ResourceInfo{ URL: "mcp://tool/" + tool.Name, Description: tool.Description, }, @@ -240,27 +256,16 @@ func (s *X402Server) AddPayableTool( var options x402http.PaymentOptions for _, req := range requirements { options = append(options, x402http.PaymentOption{ - Scheme: req.Scheme, - PayTo: req.PayTo, - Price: req.Amount, - Network: x402.Network(req.Network), + Scheme: req.Scheme, + PayTo: req.PayTo, + Price: req.Amount, + Network: x402.Network(req.Network), + MaxTimeoutSeconds: req.MaxTimeoutSeconds, + Extra: req.Extra, }) } - schemes := make([]nethttp.SchemeConfig, 0, len(s.networks)*2) - for _, net := range s.networks { - // For getStyleProfile, only register upto scheme (variable pricing) - if tool.Name == "getStyleProfile" { - schemes = append(schemes, - nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, - ) - } else { - schemes = append(schemes, - nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}, - nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, - ) - } - } + schemes := buildSchemeConfigs(s.networks, requirements...) settlementHandler := func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) { requirements, _ := nethttp.RequirementsFromContext(r.Context()) @@ -355,8 +360,32 @@ func (s *X402Server) AddPayableTool( s.mux.Handle("POST /"+VERSION1+tool.Name, s.middlewareChain(paymentMiddleware(restHandler))) } +// buildSchemeConfigs maps the schemes actually declared in a tool's payment +// requirements to the EVM scheme servers a route needs to verify and settle +// them, for every configured network. Only the schemes present in the +// requirements are registered: an exact-only tool does not expose an upto +// scheme, and vice versa. This keeps upto (variable-pricing) authorization from +// being offered for tools that cannot vary their price. +func buildSchemeConfigs(networks []string, requirements ...x402.PaymentRequirements) []nethttp.SchemeConfig { + present := make(map[string]struct{}) + for _, req := range requirements { + present[req.Scheme] = struct{}{} + } + schemes := make([]nethttp.SchemeConfig, 0, len(networks)*len(present)) + for _, net := range networks { + if _, ok := present["exact"]; ok { + schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}) + } + if _, ok := present["upto"]; ok { + schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}) + } + } + return schemes +} + func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, toolName string, handler mcp.ToolHandler) { - bodyBytes, err := io.ReadAll(r.Body) + defer r.Body.Close() + bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 5*1024*1024)) if err != nil { http.Error(w, "Failed to read body", http.StatusBadRequest) return @@ -378,30 +407,31 @@ func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, tool } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) if len(resp.Content) == 1 { if textContent, ok := resp.Content[0].(*mcp.TextContent); ok { var js json.RawMessage if json.Unmarshal([]byte(textContent.Text), &js) == nil { - // For upto scheme tools, compute actual cost and apply settlement override - if toolName == "getStyleProfile" { - actualCost := computeTwitterStyleProfileCost(r, textContent.Text) - log.DebugWithContext(r.Context(), "Setting settlement override", - "tool", toolName, - "actualCost", actualCost, - ) - nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ - Amount: actualCost, - }) + if resolver := s.settleResolvers[toolName]; resolver != nil { + if actualCost, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{textContent}}); ok && actualCost != "" { + log.DebugWithContext(r.Context(), "Setting settlement override", + "tool", toolName, + "actualCost", actualCost, + ) + nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ + Amount: actualCost, + }) + } } - w.Write([]byte(textContent.Text)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(textContent.Text)) return } } } - json.NewEncoder(w).Encode(resp) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) } func (s *X402Server) OnAfterSettle(hook func(ctx x402.SettleResultContext) error) { @@ -492,6 +522,23 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to set socket permissions: %w", err) } + grp, err := user.LookupGroup("mcpserver") + if err != nil { + return fmt.Errorf("failed to lookup group mcpserver: %w", err) + } + gid, err := strconv.Atoi(grp.Gid) + if err != nil { + return fmt.Errorf("invalid gid %q: %w", grp.Gid, err) + } + + if err := os.Chown(socketPath, -1, gid); err != nil { + return fmt.Errorf("failed to set socket group ownership: %w", err) + } + + if err := os.Chmod(socketPath, permissions); err != nil { + return fmt.Errorf("failed to set socket permissions: %w", err) + } + log.Info("Starting X402 MCP Server on unix socket", "socketPath", socketPath) } else { log.Info("Starting X402 MCP Server on TCP", "addr", addr) @@ -501,15 +548,33 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to listen on tcp: %w", err) } } - defer listener.Close() srv := &http.Server{ - Handler: s.globalReqLog(s.mux), - ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), + Handler: s.globalReqLog(s.mux), + ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), + ReadTimeout: 15 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, } + + s.mu.Lock() + s.httpServer = srv + s.mu.Unlock() + return srv.Serve(listener) } +func (s *X402Server) Shutdown(ctx context.Context) error { + s.mu.Lock() + srv := s.httpServer + s.mu.Unlock() + + if srv == nil { + return nil + } + return srv.Shutdown(ctx) +} + func toJSONSchema(val any) json.RawMessage { if val == nil { return json.RawMessage(`{"type": "object"}`) @@ -570,8 +635,8 @@ func parseSchema(schemaAny any) any { func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { - for i := len(middlewares) - 1; i >= 0; i-- { - next = middlewares[i](next) + for _, middleware := range slices.Backward(middlewares) { + next = middleware(next) } return next } @@ -583,38 +648,60 @@ func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(h // Twitter API costs guarantees the call always clears the underlying API cost // and generates a profit - it can never break even or lose money. const ( - // baseProfileCostMicro is the base cost of the style-profile endpoint for + // BaseProfileCostMicro is the base cost of the style-profile endpoint for // the non-Twitter path. It doubles as the profit margin added to Twitter // requests, expressed in micro-dollars (USDC has 6 decimals). - baseProfileCostMicro int64 = 100_000 // $0.10 - // twitterUserLookupMicro is the per-request Twitter user lookup API cost. - twitterUserLookupMicro int64 = 10_000 // $0.01 - // twitterPostReadMicro is the per-post Twitter read API cost. - twitterPostReadMicro int64 = 5_000 // $0.005 per post + BaseProfileCostMicro int64 = 250_000 // $0.25 + // TwitterUserLookupMicro is the per-request Twitter user lookup API cost. + TwitterUserLookupMicro int64 = 10_000 // $0.01 + // TwitterPostReadMicro is the per-post Twitter read API cost. + TwitterPostReadMicro int64 = 5_000 // $0.005 per post + // MaxProfileCountedPosts caps the post count a style profile can ever + // report. It is NOT applied inside computeTwitterStyleProfileCost (charges + // follow the posts actually read, and the server fetches at most this + // many); it only bounds the largest settlement the endpoint can demand. + MaxProfileCountedPosts int64 = 100 + // MaxAnalyticsCountedPosts caps the number of posts an analytics tool can + // include from any single platform (mirrors maxAnalyticsLimit in the server + // package). It bounds the largest settlement an upto analytics request can + // demand. + MaxAnalyticsCountedPosts int64 = 200 ) +// maxProfileCostMicro is the largest price getStyleProfile can ever command: +// the flat base cost plus the Twitter lookup and per-post costs at the +// post-count cap. The price advertised for getStyleProfile in tools.go must +// equal formatDollarAmount(maxProfileCostMicro) ("$0.76"). +const maxProfileCostMicro = BaseProfileCostMicro + TwitterUserLookupMicro + MaxProfileCountedPosts*TwitterPostReadMicro + // computeTwitterStyleProfileCost calculates the actual charge for a Twitter // getStyleProfile request based on the number of posts fetched and the user -// lookup, and returns it as a dollar price string (e.g. "$0.16") suitable for a +// lookup, and returns it as a dollar price string (e.g. "$0.31") suitable for a // x402 settlement override. // -// Pricing: guaranteed base cost of $0.10 (the endpoint cost for other +// Pricing: guaranteed base cost of $0.25 (the endpoint cost for other // platforms), plus user lookup $0.01, plus $0.005 per post read. For example, -// 10 posts = $0.16. Non-Twitter platforms are charged the flat $0.10 base. +// 10 posts = $0.31. Non-Twitter platforms are charged the flat $0.25 base. +// +// The price always reflects the posts actually read - it is never clamped to a +// cap (the server only ever fetches at most maxProfileCountedPosts, so the +// derived max always stays at or below formatDollarAmount(maxProfileCostMicro)). +// A PostCount of 0 means no posts were read (server-side defaults are applied +// before the profile is built, so responses carry the real count). func computeTwitterStyleProfileCost(_ *http.Request, responseText string) string { // Parse the response to get PostCount and SourcePlatform var resp GetStyleProfileResponse if err := json.Unmarshal([]byte(responseText), &resp); err != nil { - return formatDollarAmount(baseProfileCostMicro) // fallback: base cost + return formatDollarAmount(BaseProfileCostMicro) // fallback: base cost } - costMicro := baseProfileCostMicro + costMicro := BaseProfileCostMicro // Twitter additionally passes on the user lookup and per-post API costs. if resp.SourcePlatform == "Twitter" { postCount := max(resp.PostCount, 0) - costMicro += twitterUserLookupMicro - costMicro += int64(postCount) * twitterPostReadMicro + costMicro += TwitterUserLookupMicro + costMicro += int64(postCount) * TwitterPostReadMicro } return formatDollarAmount(costMicro) @@ -638,6 +725,74 @@ func formatDollarAmount(micro int64) string { return "$" + s } +// FormatPriceMicro renders an amount in micro-dollars as a plain, un-prefixed +// numeric price string ("0.76") as expected by x402 PaymentRequirements.Amount. +func FormatPriceMicro(micro int64) string { + return strings.TrimPrefix(formatDollarAmount(micro), "$") +} + +// MicroFromPriceString parses a plain dollar price string ("0.76") into +// micro-dollars so pricing arithmetic can be done without float rounding. +func MicroFromPriceString(price string) int64 { + micro, err := strconv.ParseFloat(price, 64) + if err != nil { + return 0 + } + return int64(math.Round(micro * 1_000_000)) +} + +// MaxTwitterCostMicro is the largest cost a Twitter-capable tool can command +// when it reaches its post-count cap: the flat base cost plus the Twitter +// lookup and per-post costs at the cap. It is used to advertise the upto +// maximum for tools whose actual cost is settled from the response. +func MaxTwitterCostMicro(baseMicro, postCap int64) int64 { + return baseMicro + TwitterUserLookupMicro + postCap*TwitterPostReadMicro +} + +// NewStyleProfileCostResolver returns a SettleAmountResolver that bills a +// getStyleProfile request by the posts actually read (see +// computeTwitterStyleProfileCost), rather than the advertised upto maximum. +func NewStyleProfileCostResolver() SettleAmountResolver { + return func(result *mcp.CallToolResult) (string, bool) { + if len(result.Content) != 1 { + return "", false + } + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + return "", false + } + return computeTwitterStyleProfileCost(nil, textContent.Text), true + } +} + +// NewTwitterCountCostResolver returns a SettleAmountResolver for tools whose +// response carries a `twitterPostCount` JSON field. It settles the flat +// baseCostMicro, plus the Twitter lookup and per-post costs when Twitter posts +// were actually used (twitterPostCount > 0). When no Twitter posts went into +// the response the caller is charged only the base cost. +func NewTwitterCountCostResolver(baseCostMicro int64) SettleAmountResolver { + return func(result *mcp.CallToolResult) (string, bool) { + if len(result.Content) != 1 { + return "", false + } + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + return "", false + } + var payload struct { + TwitterPostCount int `json:"twitterPostCount"` + } + if json.Unmarshal([]byte(textContent.Text), &payload) != nil { + return "", false + } + costMicro := baseCostMicro + if payload.TwitterPostCount > 0 { + costMicro += TwitterUserLookupMicro + int64(payload.TwitterPostCount)*TwitterPostReadMicro + } + return formatDollarAmount(costMicro), true + } +} + // GetStyleProfileResponse is a local copy of the response schema for cost calculation type GetStyleProfileResponse struct { Author string `json:"author"` @@ -660,6 +815,13 @@ func (c *verifyErrorCapture) WriteHeader(code int) { c.ResponseWriter.WriteHeader(code) } +func (c *verifyErrorCapture) Write(b []byte) (int, error) { + if c.statusCode == 0 { + c.statusCode = http.StatusOK + } + return c.ResponseWriter.Write(b) +} + func decodePaymentRequiredHeader(header string) (x402.PaymentRequired, error) { data, err := base64.StdEncoding.DecodeString(header) if err != nil { diff --git a/internal/x402/x402_test.go b/internal/x402/x402_test.go index 2598be8..f26e9ca 100644 --- a/internal/x402/x402_test.go +++ b/internal/x402/x402_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "sync/atomic" "testing" @@ -51,6 +52,7 @@ func TestX402Server_HTTPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -123,6 +125,7 @@ func TestX402Server_MCPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -136,14 +139,14 @@ func TestX402Server_MCPBoundaries(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer serverSession.Close() + defer func() { _ = serverSession.Close() }() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer clientSession.Close() + defer func() { _ = clientSession.Close() }() toolsResponse, err := clientSession.ListTools(ctx, nil) if err != nil { @@ -206,6 +209,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -218,10 +222,11 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, + nil, x402.PaymentRequirements{ Scheme: "upto", Network: "eip155:84532", - Amount: "0.36", + Amount: "0.51", PayTo: "0x0000000000000000000000000000000000000000", }, ) @@ -230,14 +235,14 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer serverSession.Close() + defer func() { _ = serverSession.Close() }() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer clientSession.Close() + defer func() { _ = clientSession.Close() }() cases := []struct { tool string @@ -245,7 +250,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { want string }{ {tool: "getExact", scheme: "exact", want: "10000"}, - {tool: "getUpto", scheme: "upto", want: "360000"}, + {tool: "getUpto", scheme: "upto", want: "510000"}, } for _, tc := range cases { @@ -439,11 +444,13 @@ func TestComputeTwitterStyleProfileCost(t *testing.T) { postCount int want string }{ - {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.10"}, - {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.16"}, - {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.11"}, - {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.36"}, - {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.125"}, + {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.25"}, + {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.31"}, + {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.26"}, + {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.51"}, + {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.275"}, + {name: "posts at the max counted cap", platform: "Twitter", postCount: 100, want: "$0.76"}, + {name: "beyond cap is not clamped", platform: "Twitter", postCount: 200, want: "$1.26"}, } for _, tc := range cases { @@ -469,15 +476,15 @@ func TestComputeTwitterStyleProfileCostAlwaysProfitable(t *testing.T) { r, _ := http.NewRequest("POST", "/v1/getStyleProfile", nil) got := computeTwitterStyleProfileCost(r, string(payload)) - // Cut the "$" prefix and compare numerically to baseProfileCostMicro. + // Cut the "$" prefix and compare numerically to BaseProfileCostMicro. dollars := strings.TrimPrefix(got, "$") parsed, err := strconv.ParseFloat(dollars, 64) if err != nil { t.Fatalf("failed to parse %q: %v", got, err) } micro := int64(parsed * 1_000_000) - if micro <= baseProfileCostMicro { - t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, baseProfileCostMicro) + if micro <= BaseProfileCostMicro { + t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, BaseProfileCostMicro) } } } @@ -615,6 +622,7 @@ func TestAddPayableTool_EmptyRequirements(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, // No requirements provided ) } @@ -651,6 +659,7 @@ func TestX402Server_MultipleTools(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"tool": "` + name + `"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -691,10 +700,11 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { postCount int want string }{ - {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.10"}, // not "Twitter" so flat base - {name: "empty platform", platform: "", postCount: 10, want: "$0.10"}, - {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.11"}, // should clamp to 0 - {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.11"}, + {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.25"}, // not "Twitter" so flat base + {name: "empty platform", platform: "", postCount: 10, want: "$0.25"}, + {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.26"}, // should clamp to 0 + {name: "few posts", platform: "Twitter", postCount: 1, want: "$0.265"}, + {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.26"}, } for _, tc := range cases { @@ -710,3 +720,309 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { }) } } + +// TestStyleProfileMaxPriceStaysAdvertised guards the advertised getStyleProfile +// price in internal/server/tools.go against silent drift from the x402 pricing +// model. internal/server shares the x402 constants (BaseProfileCostMicro / +// MaxProfileCountedPosts) to compute the upto maximum, so this only needs to +// catch a hand-edited Price string going out of sync. +func TestStyleProfileMaxPriceStaysAdvertised(t *testing.T) { + if got := formatDollarAmount(maxProfileCostMicro); got != "$0.76" { + t.Fatalf("maxProfileCostMicro = %d (%s), want $0.76; update getStyleProfile base Price in internal/server/tools.go if the pricing model changed", maxProfileCostMicro, got) + } +} + +// TestX402Server_MCPStyleProfileSettlementOverride drives the full MCP flow for +// the getStyleProfile upto tool and asserts the facilitator is told to settle +// the computed per-request cost (not the advertised $0.76 maximum). +func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { + ctx := context.Background() + + var mu sync.Mutex + var settleAmounts []string + mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == "GET" && r.URL.Path == "/supported": + _, _ = w.Write([]byte(`{ + "kinds": [ + {"x402Version": 2, "scheme": "upto", "network": "eip155:84532"} + ] + }`)) + case r.Method == "POST" && r.URL.Path == "/verify": + _, _ = w.Write([]byte(`{"isValid": true, "invalidReason": ""}`)) + case r.Method == "POST" && r.URL.Path == "/settle": + var body struct { + PaymentRequirements map[string]json.RawMessage `json:"paymentRequirements"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if raw := body.PaymentRequirements["amount"]; len(raw) > 0 { + var s string + var f float64 + mu.Lock() + if json.Unmarshal(raw, &s) == nil { + settleAmounts = append(settleAmounts, s) + } else if json.Unmarshal(raw, &f) == nil { + settleAmounts = append(settleAmounts, strconv.FormatFloat(f, 'f', -1, 64)) + } + mu.Unlock() + } + _, _ = w.Write([]byte(`{ + "success": true, + "transaction": "0x1234abcd", + "network": "eip155:84532", + "payer": "0xabcd1234" + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer mock.Close() + + serverTransport, clientTransport := mcp.NewInMemoryTransports() + + srv, err := NewX402Server( + "Test Agent API", + "1.0.0", + mock.URL, + &DummyAuthProvider{}, + X402Middlewares{}, + func(next http.Handler) http.Handler { return next }, + func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) {}, + []string{"eip155:84532"}, + ) + if err != nil { + t.Fatalf("failed to create server: %v", err) + } + + responses := []GetStyleProfileResponse{ + {Author: "@a", TopKeywords: []string{}, StyleSummary: "s", PostCount: 10, SourcePlatform: "Bluesky"}, + {Author: "@b", TopKeywords: []string{}, StyleSummary: "s", PostCount: 50, SourcePlatform: "Twitter"}, + } + var idx int + srv.AddPayableTool( + mcp.Tool{Name: "getStyleProfile", Description: "get a style profile"}, + func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + resp := responses[idx%len(responses)] + idx++ + b, _ := json.Marshal(resp) + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}}, nil + }, + NewStyleProfileCostResolver(), + x402.PaymentRequirements{ + Scheme: "upto", + Network: "eip155:84532", + Amount: "0.76", + PayTo: "0x0000000000000000000000000000000000000000", + }, + ) + + serverSession, err := srv.mcpServer.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatalf("failed to connect server session: %v", err) + } + defer func() { _ = serverSession.Close() }() + + client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) + clientSession, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatalf("failed to connect client session: %v", err) + } + defer func() { _ = clientSession.Close() }() + + paidCall := func() (*mcp.CallToolResult, error) { + challenge, err := clientSession.CallTool(ctx, &mcp.CallToolParams{Name: "getStyleProfile", Arguments: json.RawMessage(`{}`)}) + if err != nil { + return nil, fmt.Errorf("challenge call failed: %w", err) + } + if !challenge.IsError { + return nil, fmt.Errorf("expected a payment challenge, got %+v", challenge) + } + text, ok := challenge.Content[0].(*mcp.TextContent) + if !ok { + return nil, fmt.Errorf("unexpected challenge content type %T", challenge.Content[0]) + } + var pr types.PaymentRequired + if err := json.Unmarshal([]byte(text.Text), &pr); err != nil { + return nil, fmt.Errorf("failed to decode payment challenge: %w", err) + } + if len(pr.Accepts) == 0 { + return nil, fmt.Errorf("expected at least one accept, got %d", len(pr.Accepts)) + } + payload := types.PaymentPayload{ + X402Version: 2, + Payload: map[string]interface{}{"chainId": 84532}, + Accepted: pr.Accepts[0], + } + return clientSession.CallTool(ctx, &mcp.CallToolParams{ + Name: "getStyleProfile", + Arguments: json.RawMessage(`{}`), + Meta: mcp.Meta{PaymentMetaKey: payload}, + }) + } + + for i := 0; i < 2; i++ { + result, err := paidCall() + if err != nil { + t.Fatalf("paid call: %v", err) + } + if result.IsError { + text, _ := result.Content[0].(*mcp.TextContent) + t.Fatalf("expected successful paid call, got error result: %+v", text) + } + if resp := result.Meta[PaymentResponseMetaKey]; resp == nil { + t.Fatalf("expected %q in result meta", PaymentResponseMetaKey) + } + } + + mu.Lock() + got := append([]string(nil), settleAmounts...) + mu.Unlock() + // The "$0.25" / "$0.51" overrides resolve to 250000 / 510000 atomic units, + // with up to 1 micro tolerance for float rounding in the SDK's dollar parser. + wants := []int64{250000, 510000} + if len(got) != len(wants) { + t.Fatalf("settle amount count = %d (%v), want %d: %v", len(got), got, len(wants), wants) + } + for i, want := range wants { + v, err := strconv.ParseInt(got[i], 10, 64) + if err != nil { + t.Fatalf("settle amount[%d] = %q is not an integer: %v", i, got[i], err) + } + diff := v - want + if diff < 0 { + diff = -diff + } + if diff > 1 { + t.Errorf("settle amount[%d] = %d, want ~%d", i, v, want) + } + } +} + +func TestNewStyleProfileCostResolver(t *testing.T) { + resolver := NewStyleProfileCostResolver() + + cases := []struct { + name string + resp GetStyleProfileResponse + want string + }{ + {name: "bluesky base", resp: GetStyleProfileResponse{SourcePlatform: "Bluesky", PostCount: 10}, want: "$0.25"}, + {name: "twitter with posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 50}, want: "$0.51"}, + {name: "twitter no posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 0}, want: "$0.26"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + payload, _ := json.Marshal(tc.resp) + result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} + got, ok := resolver(result) + if !ok { + t.Fatalf("resolver returned ok=false") + } + if got != tc.want { + t.Fatalf("resolver = %q, want %q", got, tc.want) + } + }) + } + + // Non-text content resolves to "no override"; malformed JSON falls back to + // the flat base cost via computeTwitterStyleProfileCost. + if _, ok := resolver(&mcp.CallToolResult{}); ok { + t.Fatalf("expected no override for empty content") + } + got, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}) + if !ok { + t.Fatalf("expected a base-cost fallback for malformed JSON") + } + if got != "$0.25" { + t.Fatalf("malformed JSON resolver = %q, want base fallback $0.25", got) + } +} + +// TestNewTwitterCountCostResolver verifies the actual-cost settlement for +// tools whose responses carry a `twitterPostCount` field (analytics tools, +// findSimilarUsers): flat base price when no Twitter posts went into the +// response, and base + lookup + per-post when they did. +func TestNewTwitterCountCostResolver(t *testing.T) { + const baseMicro = 150_000 // getEngagementMetrics base + resolver := NewTwitterCountCostResolver(baseMicro) + + cases := []struct { + name string + twitterCount int + want string + }{ + {name: "no twitter posts", twitterCount: 0, want: "$0.15"}, + {name: "one twitter post", twitterCount: 1, want: "$0.165"}, + {name: "ten twitter posts", twitterCount: 10, want: "$0.21"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "metrics": []any{}, + "twitterPostCount": tc.twitterCount, + }) + result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} + got, ok := resolver(result) + if !ok { + t.Fatalf("resolver returned ok=false") + } + if got != tc.want { + t.Fatalf("resolver = %q, want %q", got, tc.want) + } + }) + } + + if _, ok := resolver(&mcp.CallToolResult{}); ok { + t.Fatalf("expected no override for empty content") + } + if _, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}); ok { + t.Fatalf("expected no override for malformed JSON") + } +} + +// TestBuildSchemeConfigs verifies scheme servers are derived from the payment +// requirements rather than the tool name, so upto is only registered for tools +// that actually declare it. +func TestBuildSchemeConfigs(t *testing.T) { + req := func(scheme string) x402.PaymentRequirements { + return x402.PaymentRequirements{Scheme: scheme, Network: "eip155:8453", Asset: "usdc", Amount: "1", PayTo: "to"} + } + + t.Run("exact only", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453"}, req("exact")) + if len(got) != 1 || got[0].Network != x402.Network("eip155:8453") { + t.Fatalf("expected 1 exact scheme for one network, got %+v", got) + } + }) + + t.Run("upto only", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453"}, req("upto")) + if len(got) != 1 { + t.Fatalf("expected 1 upto scheme, got %d", len(got)) + } + }) + + t.Run("both schemes per network", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453", "eip155:84532"}, req("exact"), req("upto")) + if len(got) != 4 { + t.Fatalf("expected 2 schemes x 2 networks = 4, got %d", len(got)) + } + }) +} + +func TestPricingHelpers(t *testing.T) { + if got := FormatPriceMicro(MaxTwitterCostMicro(BaseProfileCostMicro, MaxProfileCountedPosts)); got != "0.76" { + t.Fatalf("style profile upto max = %q, want 0.76", got) + } + if got := FormatPriceMicro(MaxTwitterCostMicro(150_000, MaxAnalyticsCountedPosts)); got != "1.16" { + t.Fatalf("engagement metrics upto max = %q, want 1.16", got) + } + if got := MicroFromPriceString("0.25"); got != BaseProfileCostMicro { + t.Fatalf("MicroFromPriceString(0.25) = %d, want %d", got, BaseProfileCostMicro) + } + if got := MicroFromPriceString("garbage"); got != 0 { + t.Fatalf("MicroFromPriceString(garbage) = %d, want 0", got) + } +} diff --git a/mcpServer.service b/mcpServer.service index d9e183e..d852b7b 100644 --- a/mcpServer.service +++ b/mcpServer.service @@ -7,7 +7,7 @@ User=mcpserver Group=mcpserver WorkingDirectory=/home/ubuntu/gitSources/mcpServer -ExecStart=/home/ubuntu/gitSources/mcpServer/bin/main +ExecStart=/home/ubuntu/gitSources/mcpServer/bin/mcpserver RuntimeDirectory=mcpserver RuntimeDirectoryMode=0750 diff --git a/sql/schema/003_add_cache_version.sql b/sql/schema/003_add_cache_version.sql index f5d2496..4fdbf1e 100644 --- a/sql/schema/003_add_cache_version.sql +++ b/sql/schema/003_add_cache_version.sql @@ -1,8 +1,8 @@ -- +goose up ALTER TABLE user_style_profiles - ADD COLUMN model_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN schema_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN algorithm_version INTEGER NOT NULL DEFAULT 1; + ADD COLUMN IF NOT EXISTS model_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS schema_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS algorithm_version INTEGER NOT NULL DEFAULT 1; CREATE INDEX IF NOT EXISTS idx_user_style_profiles_versions ON user_style_profiles (model_version, schema_version, algorithm_version); diff --git a/sql/schema/004_add_style_posts.sql b/sql/schema/004_add_style_posts.sql index cc020c1..ae5bf08 100644 --- a/sql/schema/004_add_style_posts.sql +++ b/sql/schema/004_add_style_posts.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS user_style_posts ( UNIQUE (user_id, platform, post_id) ); -CREATE INDEX idx_user_style_posts_user_platform +CREATE INDEX IF NOT EXISTS idx_user_style_posts_user_platform ON user_style_posts (user_id, platform); -- +goose down diff --git a/testClient/mcp/test-mcp-endpoints.go b/testClient/mcp/test-mcp-endpoints.go index b6668e8..79b35cb 100644 --- a/testClient/mcp/test-mcp-endpoints.go +++ b/testClient/mcp/test-mcp-endpoints.go @@ -87,7 +87,7 @@ func getEnv(key, fallback string) string { } func main() { - godotenv.Load() + _ = godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080") @@ -125,7 +125,7 @@ func main() { if err != nil { log.Fatal("Failed to connect MCP client:", err) } - defer session.Close() + defer func() { _ = session.Close() }() x402Mcp := x402mcp.NewX402MCPClientFromConfig(session, []x402mcp.SchemeRegistration{ {Network: x402.Network("eip155:84532"), Client: evm.NewExactEvmScheme(signer, nil)}, diff --git a/testClient/rest/test-rest-endpoints.go b/testClient/rest/test-rest-endpoints.go index f048e29..c9e9f7b 100644 --- a/testClient/rest/test-rest-endpoints.go +++ b/testClient/rest/test-rest-endpoints.go @@ -70,7 +70,7 @@ func callEndpoint(client *http.Client, method, endpoint, body string) error { if err != nil { return fmt.Errorf("%s HTTP request failed: %w", endpoint, err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() respBody, _ := io.ReadAll(resp.Body) fmt.Printf("%s Response Status: %s\n", endpoint, resp.Status) @@ -124,7 +124,7 @@ func getEnv(key, fallback string) string { } func main() { - godotenv.Load() + _ = godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080")