diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 6d5891e..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: "2" -linters: - settings: - errcheck: - exclude-functions: - - (*net/http.Response).Body.Close - - io.Closer.Close - exclusions: - generated: lax - presets: - - comments - - common-false-positives - - legacy - - std-error-handling - paths: - - third_party$ - - builtin$ - - examples$ -formatters: - exclusions: - generated: lax - paths: - - third_party$ - - builtin$ - - examples$ diff --git a/AGENTS.md b/AGENTS.md index 317bf8d..23f17b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,8 +36,7 @@ internal/server/ MCP handlers + x402 + analytics (~1800 lines in serv types.go request/response types, platform enum auth.go / webhook.go / cache*.go / *Middleware.go / networkInfo.go *_test.go split suites: mcp_test.go, analytics_test.go, validation_test.go, etc. -internal/activitypub/ AP client; throttle.go per-host token bucket (1/s) for background GTS warming -internal/{bluesky,nostr,twitter,ollama,config,utils,actor,x402} +internal/{bluesky,nostr,activitypub,twitter,ollama,config,utils,actor,x402} sql/schema/ 001..007 goose migrations sql/queries/ sqlc query files → internal/store/ config/models.env single source of truth for EMBEDDING_MODEL / STYLE_MODEL @@ -92,6 +91,5 @@ Loaded in order: `config/models.env` → `.env` (gitignored, via `godotenv`) → - Unix socket MCP + TCP readyz sidecar; `mcpServer.service` expects `bin/main` at repo root. - Platform inferred from author handle in `determinePlatform`/`fetch*` — not caller-supplied. - Bluesky auth refresh via `isAuthError`/`withRefresh` — 401/expired token retried once. -- Twitter client clamps `limit` (`>100` → 100), retries 429/5xx/transport errors with `x-rate-limit-*`/`Retry-After` backoff (bounded by `maxRateLimitWait`), and has a 60s per-request timeout. - - ActivityPub client: GTS Mastodon API only serves locally-federated statuses. Remote actors are resolved via `/accounts/search?resolve=true` (not `/accounts/lookup`, which skips WebFinger). When the local cache can't satisfy full `limit` (clamped to 200), `GetLatestNPosts`/`GetPostsInTimeframe` fall back to walking the remote outbox directly (webfinger → actor → `OrderedCollection`/`OrderedCollectionPage`), filtering `Create`/`Note`-like activities and dereferencing IRI-only objects; `GetPost` derefs status URLs over AP when GTS `/search` can't resolve them. The `read:search` scope is needed for status-URL resolution. Outbox-fetched posts are warmed into GTS in background via `GET /api/v2/search?resolve=true&type=statuses&limit=1` per post, throttled at 1 req/s per remote host via per-host token bucket (`internal/activitypub/throttle.go`) with cross-host concurrency, no follows. - - Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). +- Twitter client has per-limit clamp (`>100`/`>50` rejected) and `x-rate-limit-*`/`Retry-After` backoff. +- Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). diff --git a/cdpScripts/rescueFunds.py b/cdpScripts/rescueFunds.py new file mode 100644 index 0000000..18ef5fd --- /dev/null +++ b/cdpScripts/rescueFunds.py @@ -0,0 +1,25 @@ +from cdp import cdp_client +import asyncio +import inspect +from dotenv import load_dotenv + +load_dotenv() + +OLD = "0x3C17021dF1071b7F45cd3D1e4Db0bcBd03C6ECF4" +NETWORK = "base-sepolia" + + +async def main(): + cdp = cdp_client.CdpClient() + + try: + private_key = await cdp.evm.export_account( + name="api-naco-li" + ) + print(f"Private key: {private_key}") + + finally: + await cdp.close() + + +asyncio.run(main()) \ No newline at end of file diff --git a/cmd/check-facilitator/main.go b/cmd/check-facilitator/main.go index 717cf00..db8aa29 100644 --- a/cmd/check-facilitator/main.go +++ b/cmd/check-facilitator/main.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "strings" - "time" "github.com/nacorid/naco-api/internal/server" ) @@ -117,22 +116,18 @@ func main() { os.Exit(1) } - httpClient := &http.Client{ - Timeout: 30 * time.Second, - } - resp, err := httpClient.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Fprintf(os.Stderr, "Request failed: %v\n", err) os.Exit(1) } defer resp.Body.Close() - maxBytes := int64(5 * 1024 * 1024) - body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes)) - if err != nil { - fmt.Printf("failed reading response body: %v", err) - } - + body, _ := io.ReadAll(resp.Body) fmt.Printf("Status: %d\n", resp.StatusCode) - fmt.Printf("Response: %s\n", body) + fmt.Printf("Response: %s\n", string(body)) + + if resp.StatusCode != http.StatusOK { + os.Exit(1) + } } diff --git a/cmd/server/main.go b/cmd/server/main.go index b42c61f..300f84c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,9 +3,6 @@ package main import ( "context" "database/sql" - "errors" - "fmt" - "io" "net/http" "net/url" "os" @@ -23,45 +20,25 @@ import ( "github.com/nacorid/naco-api/internal/server" "github.com/nacorid/naco-api/internal/store" "github.com/nacorid/naco-api/internal/twitter" - "golang.org/x/sync/errgroup" ) func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - // backgroundCtx is for long-lived clients whose lifetime is the process, - // not a single request. It must not inherit signal cancellation so that a - // SIGTERM during startup does not poison an in-flight session creation. - backgroundCtx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() cfg := config.Load() - if err := log.Init(log.Config{ + log.Init(log.Config{ LogFilePath: cfg.LogFile, ConsoleLevel: log.LevelDebug, FileLevel: log.LevelTrace, - }); err != nil { - log.Fatalf("Failed to init logger: %v", err) - } + }) defer log.Close() - conn, err := sql.Open("postgres", cfg.DatabaseURL) if err != nil { - log.Fatalf("Failed to initialize database connection: %v", err) + log.Fatalf("Failed to connect to database: %v", err) } defer conn.Close() - - conn.SetMaxOpenConns(10) - conn.SetMaxIdleConns(10) - conn.SetConnMaxLifetime(5 * time.Minute) - - dbCtx, dbCancel := context.WithTimeout(ctx, 5*time.Second) - err = conn.PingContext(dbCtx) - dbCancel() - if err != nil { - log.Fatalf("Failed to ping database: %v", err) - } db := store.New(conn) if cfg.MigrateOnStart { @@ -71,7 +48,7 @@ func main() { log.Info("Database migrations applied", "dir", cfg.MigrationsDir) } - bskyClient, err := bluesky.NewBlueskyClient(backgroundCtx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) + bskyClient, err := bluesky.NewBlueskyClient(ctx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) if err != nil { log.Fatal("Failed to create bluesky client", "error", err) } @@ -83,9 +60,9 @@ func main() { ollamaHTTPClient := &http.Client{ Timeout: 600 * time.Second, } - ollamaClient := ollama.New(uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) + ollamaClient := ollama.New(ctx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) - apClient, err := activitypub.NewAPClient(cfg.ActivityPubHost, cfg.ActivityPubToken) + apClient, err := activitypub.NewAPClient(ctx, cfg.ActivityPubHost, cfg.ActivityPubToken) if err != nil { log.Fatal("Failed to create activitypub client", "error", err) } @@ -101,86 +78,48 @@ func main() { srv.StartPrewarm(ctx) - g, gCtx := errgroup.WithContext(ctx) - startReadyzServer(g, ctx, cfg, conn) - - g.Go(func() error { - log.Info("Server starting", "address", cfg.ListenAddress) - if err := srv.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("main server error: %w", err) - } - return nil - }) - - g.Go(func() error { - <-gCtx.Done() - log.Info("Shutting down main server...") - shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - return srv.Shutdown(shutdownCtx) - }) - - if err := g.Wait(); err != nil && !errors.Is(err, context.Canceled) { - log.Error("Application terminated with error", "error", err) - } - - log.Info("Application shutdown complete") -} - -func startReadyzServer(g *errgroup.Group, shutdownCtx context.Context, cfg *config.Config, conn *sql.DB) { - mux := http.NewServeMux() - - client := &http.Client{ - Timeout: 10 * time.Second, - } - - mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { - if err := conn.PingContext(r.Context()); err != nil { - http.Error(w, "database unavailable", http.StatusServiceUnavailable) - return - } - - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) - if err != nil { - http.Error(w, "invalid ollama configuration", http.StatusInternalServerError) - return - } - - resp, err := client.Do(req) - if err != nil || resp.StatusCode != http.StatusOK { - if resp != nil { - resp.Body.Close() + go func() { + mux := http.NewServeMux() + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + if err := conn.PingContext(r.Context()); err != nil { + http.Error(w, "database unavailable", http.StatusServiceUnavailable) + return + } + req, _ := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + if resp != nil { + resp.Body.Close() + } + http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) + return } - http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) - return - } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() - }() - - _, _ = w.Write([]byte("ok")) - }) - - readyServer := &http.Server{ - Addr: cfg.ReadyzAddress + ":" + cfg.ReadyzPort, - Handler: mux, - ReadTimeout: 5 * time.Second, - WriteTimeout: 5 * time.Second, - } - - g.Go(func() error { - log.Info("Readiness server starting", "address", readyServer.Addr) - if err := readyServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("readiness server failed: %w", err) + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + }) + mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + w.WriteHeader(http.StatusOK) + w.Write([]byte(srv.MetricsText())) + }) + addr := ":" + cfg.ReadyzPort + log.Info("Readiness server starting", "address", addr) + if err := http.ListenAndServe(addr, mux); err != nil { + log.Error("Readiness server error", "error", err) } - return nil - }) + }() - g.Go(func() error { - <-shutdownCtx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - return readyServer.Shutdown(shutdownCtx) - }) + go func() { + log.Info("Server starting", "address", cfg.ListenAddress) + if err := srv.Start(); err != http.ErrServerClosed { + log.Fatal("HTTP server error", "error", err) + } + }() + + stopChan := make(chan os.Signal, 1) + signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM) + <-stopChan + + log.Info("Shutting down server...") } diff --git a/go.mod b/go.mod index 13124e1..5a95b31 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 github.com/modelcontextprotocol/go-sdk v1.6.1 - github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 + github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 github.com/ollama/ollama v0.12.10 github.com/x402-foundation/x402/go/v2 v2.19.0 golang.org/x/time v0.14.0 @@ -28,7 +28,7 @@ require ( github.com/klauspost/compress v1.18.1 // indirect github.com/supranational/blst v0.3.16 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - golang.org/x/sync v0.19.0 + golang.org/x/sync v0.19.0 // indirect ) require ( diff --git a/go.sum b/go.sum index d43b247..9ef062a 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7B github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 h1:o7CvgRLUikfsbyuZIJTTrfRac5rMXhca0b61VLBWSIA= -github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= +github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 h1:nZwGInvJNQzPSFYpstVR/TLumqBEZt0+yUXchdyBZiI= +github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/ollama/ollama v0.12.10 h1:Dd0/SeCc+nv+FffxmWuQTGiRreib7Gt3nBhIIFuKwZA= github.com/ollama/ollama v0.12.10/go.mod h1:RUSmYywUWx/YZMaHrqtnT1ZChu+iSz/7jx2aO9+Mgfg= diff --git a/internal/activitypub/client.go b/internal/activitypub/client.go index c3684dd..cc45f07 100644 --- a/internal/activitypub/client.go +++ b/internal/activitypub/client.go @@ -7,374 +7,189 @@ import ( "io" "net/http" "net/url" - "strings" - "sync" "time" "github.com/nacorid/naco-api/internal/actor" ) -const maxMastodonPageSize = 40 -const maxTimeframePages = 100 -const maxResponseBody = 1 << 20 -const maxLimit = 200 - type APClient struct { client *http.Client host string bearerToken string - outbox *outboxClient - limiter *hostLimiter } -func NewAPClient(host, apikey string) (*APClient, error) { +func NewAPClient(ctx context.Context, host, apikey string) (*APClient, error) { if host == "" || apikey == "" { return nil, fmt.Errorf("host and apikey must not be empty") } - return &APClient{ - client: &http.Client{Timeout: 30 * time.Second}, - host: strings.TrimRight(host, "/"), - bearerToken: "Bearer " + apikey, - outbox: newOutboxClient(), - limiter: newHostLimiter(), + client: http.DefaultClient, + host: host, + bearerToken: fmt.Sprintf("Bearer %s", apikey), }, nil } -func (ap *APClient) get(ctx context.Context, endpoint string, target any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) +func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { + url := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, postID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) - } - - if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { - return fmt.Errorf("failed to decode response: %w", err) - } - - return nil -} - -func (ap *APClient) accountSearch(ctx context.Context, query string) (*Account, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=1", ap.host, url.QueryEscape(query)) - var accounts []*Account - if err := ap.get(ctx, endpoint, &accounts); err != nil { - return nil, fmt.Errorf("failed to search accounts: %w", err) - } - for _, acc := range accounts { - if acc != nil { - return acc, nil - } - } - return nil, nil -} - -func (ap *APClient) resolveAccount(ctx context.Context, actorHandle string) (*Account, error) { - handle := strings.TrimPrefix(actorHandle, "@") - - acc, err := ap.accountSearch(ctx, handle) - if err != nil { - return nil, err - } - if acc != nil { - return acc, nil - } - - if !strings.Contains(handle, "@") { - var byID Account - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(handle)) - if err := ap.get(ctx, endpoint, &byID); err != nil { - return nil, fmt.Errorf("account %q not found by search or id: %w", actorHandle, err) - } - if byID.ID == "" { - return nil, fmt.Errorf("account %q not found", actorHandle) - } - return &byID, nil - } - return nil, fmt.Errorf("account %q not found", actorHandle) -} - -func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { - status, err := ap.resolveStatus(ctx, postID) - if err != nil { - return nil, err - } - return status.toPost(), nil -} - -func (ap *APClient) resolveStatus(ctx context.Context, postID string) (*internalPost, error) { - if strings.HasPrefix(postID, "http://") || strings.HasPrefix(postID, "https://") || strings.HasPrefix(postID, "//") { - if status, err := ap.searchStatus(ctx, postID); err == nil && status != nil { - return status, nil - } - note, derr := ap.outbox.resolveNote(ctx, postID) - if derr != nil { - return nil, fmt.Errorf("status %q not resolvable: %w", postID, derr) - } - author := extractHandleFromURL(postID) - return noteToInternalPost(note, author), nil - } - - endpoint := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, url.PathEscape(postID)) - var status internalPost - if err := ap.get(ctx, endpoint, &status); err != nil { return nil, fmt.Errorf("failed to get post: %w", err) } - return &status, nil -} - -func (ap *APClient) searchStatus(ctx context.Context, postID string) (*internalPost, error) { - endpoint := fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(postID)) - var res searchResult - if err := ap.get(ctx, endpoint, &res); err != nil { - return nil, fmt.Errorf("failed to resolve status %q: %w", postID, err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get post: %s", resp.Status) } - for _, s := range res.Statuses { - if s != nil && (s.URL == postID || s.URI == postID) { - return s, nil - } - } - return nil, nil -} - -func (ap *APClient) GetLatestNPosts(ctx context.Context, actorID string, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil - } - if limit > maxLimit { - limit = maxLimit - } - - acc, err := ap.resolveAccount(ctx, actorID) + body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read response body: %w", err) } - - cached, cacheErr := ap.cacheLatestPosts(ctx, acc, limit) - if cacheErr != nil { - return nil, cacheErr + var internal = &internalPost{} + if err = json.Unmarshal(body, internal); err != nil { + return nil, fmt.Errorf("failed to unmarshal json response: %w", err) } - if len(cached) >= limit { - return cachePostsToPosts(cached), nil - } - - if !isRemoteAccount(acc) || acc.URL == "" { - return cachePostsToPosts(cached), nil - } - - outboxPosts, apErr := ap.outbox.getLatest(ctx, acc.URL, acc.Acct, limit-len(cached)) - if apErr != nil { - return nil, fmt.Errorf("outbox fallback failed: %w", apErr) - } - merged := mergePostsBySource(cached, outboxPosts) - if len(outboxPosts) > 0 { - urls := make([]string, 0, len(outboxPosts)) - for _, p := range outboxPosts { - if p != nil && p.ID != "" { - urls = append(urls, p.ID) - } - } - go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) - } - return merged, nil + return internal.toPost(), nil } -func (ap *APClient) cacheLatestPosts(ctx context.Context, acc *Account, limit int) ([]*internalPost, error) { - var ( - internal []*internalPost - maxID string - ) +func (ap *APClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { + url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, actor, limit) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", ap.bearerToken) + resp, err := ap.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get latest posts: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get latest posts: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + var internal []*internalPost + if err = json.Unmarshal(body, &internal); err != nil { + return nil, fmt.Errorf("failed to unmarshal json response: %w", err) + } + posts := make([]*Post, len(internal)) + for i, p := range internal { + posts[i] = p.toPost() + } + return posts, nil +} - for len(internal) < limit { +func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]Post, error) { + var posts []Post + var maxID string + + for { if ctx.Err() != nil { return nil, ctx.Err() } - - pageSize := limit - len(internal) - if pageSize > maxMastodonPageSize { - pageSize = maxMastodonPageSize + if len(posts) >= limit { + break } - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), pageSize) + url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=50", ap.host, actor) if maxID != "" { - endpoint += "&max_id=" + url.QueryEscape(maxID) + url += fmt.Sprintf("&max_id=%s", maxID) } - var page []*internalPost - if err := ap.get(ctx, endpoint, &page); err != nil { - return nil, err + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) } - if len(page) == 0 { + req.Header.Set("Authorization", ap.bearerToken) + + resp, err := ap.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get posts: %w", err) + } + // !!IMPORTANT!! Manually close body instead of deferring, due to loop + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("failed to get posts: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + var internal []*internalPost + if err := json.Unmarshal(body, &internal); err != nil { + return nil, fmt.Errorf("failed to unmarshal response body: %w", err) + } + + if len(internal) == 0 { break } - newMaxID := page[len(page)-1].ID - if newMaxID == "" { - break - } - if maxID != "" && newMaxID == maxID { - break - } - - internal = append(internal, page...) - maxID = newMaxID - } - if len(internal) > limit { - internal = internal[:limit] - } - return internal, nil -} - -func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, start, end time.Time, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil - } - if limit > maxLimit { - limit = maxLimit - } - - acc, err := ap.resolveAccount(ctx, actorID) - if err != nil { - return nil, err - } - - cached, cacheErr := ap.cachePostsInTimeframe(ctx, acc, start, end, limit) - if cacheErr != nil { - return nil, cacheErr - } - if len(cached) >= limit { - return cachePostsToPosts(cached), nil - } - - if !isRemoteAccount(acc) || acc.URL == "" { - return cachePostsToPosts(cached), nil - } - - outboxPosts, apErr := ap.outbox.getInTimeframe(ctx, acc.URL, acc.Acct, start, end, limit-len(cached)) - if apErr != nil { - return nil, fmt.Errorf("outbox fallback failed: %w", apErr) - } - merged := mergePostsBySource(cached, outboxPosts) - if len(outboxPosts) > 0 { - urls := make([]string, 0, len(outboxPosts)) - for _, p := range outboxPosts { - if p != nil && p.ID != "" { - urls = append(urls, p.ID) - } - } - go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) - } - return merged, nil -} - -func (ap *APClient) cachePostsInTimeframe(ctx context.Context, acc *Account, start, end time.Time, limit int) ([]*internalPost, error) { - var ( - internal []*internalPost - maxID string - ) - - for pages := 0; pages < maxTimeframePages; pages++ { - if ctx.Err() != nil { - return nil, ctx.Err() - } - if len(internal) >= limit { - break - } - - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), maxMastodonPageSize) - if maxID != "" { - endpoint += "&max_id=" + url.QueryEscape(maxID) - } - - var page []*internalPost - if err := ap.get(ctx, endpoint, &page); err != nil { - return nil, err - } - if len(page) == 0 { - break - } - - newMaxID := page[len(page)-1].ID - if newMaxID == "" { - break - } - if maxID != "" && newMaxID == maxID { - break - } - - for _, i := range page { + for _, i := range internal { if i.CreatedAt.After(end) { continue } + if i.CreatedAt.Before(start) { - return internal, nil + return posts, nil } - internal = append(internal, i) - if len(internal) >= limit { - return internal, nil + + posts = append(posts, *i.toPost()) + + if len(posts) >= limit { + return posts, nil } } - maxID = newMaxID + + maxID = internal[len(internal)-1].ID } - return internal, nil + return posts, nil } -func isRemoteAccount(acc *Account) bool { - return acc != nil && strings.Contains(acc.Acct, "@") -} - -func cachePostsToPosts(cached []*internalPost) []*Post { - if len(cached) == 0 { - return nil - } - posts := make([]*Post, 0, len(cached)) - for _, c := range cached { - if c == nil { - continue - } - posts = append(posts, c.toPost()) - } - return posts -} - -func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { - acc, err := ap.resolveAccount(ctx, actorHandle) +func (ap *APClient) GetActor(ctx context.Context, actor string) (*actor.Profile, error) { + // Mastodon exposes actor lookup by acct; fall back to a direct id lookup. + prof, err := ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actor))) if err != nil { - return nil, err - } - return ap.toProfile(acc), nil -} - -func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=%d", ap.host, url.QueryEscape(query), limit) - var accounts []*Account - if err := ap.get(ctx, endpoint, &accounts); err != nil { - return nil, fmt.Errorf("failed to search actors: %w", err) - } - - results := make([]*actor.Profile, 0, len(accounts)) - for _, acc := range accounts { - if acc == nil { - continue + prof, err = ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actor))) + if err != nil { + return nil, err } - results = append(results, ap.toProfile(acc)) } - return results, nil + return prof, nil } -func (ap *APClient) toProfile(acc *Account) *actor.Profile { +func (ap *APClient) fetchAccount(ctx context.Context, endpoint string) (*actor.Profile, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", ap.bearerToken) + resp, err := ap.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get actor: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get actor: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + var acc Account + if err := json.Unmarshal(body, &acc); err != nil { + return nil, fmt.Errorf("failed to unmarshal account: %w", err) + } return &actor.Profile{ Platform: "ActivityPub", Handle: acc.Acct, @@ -385,74 +200,45 @@ func (ap *APClient) toProfile(acc *Account) *actor.Profile { FollowingCount: acc.FollowingCount, PostCount: acc.StatusesCount, URL: acc.URL, - } + }, nil } -func extractHandleFromURL(postURL string) string { - u, err := url.Parse(postURL) +func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&limit=%d", ap.host, url.QueryEscape(query), limit) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return "" + return nil, fmt.Errorf("failed to create request: %w", err) } - parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") - for _, p := range parts { - if strings.HasPrefix(p, "@") { - return strings.TrimPrefix(p, "@") - } + req.Header.Set("Authorization", ap.bearerToken) + resp, err := ap.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to search actors: %w", err) } - return "" -} - -func (ap *APClient) warmGTSAsync(ctx context.Context, urls []string) { - if len(urls) == 0 { - return - } - if ap.limiter == nil { - ap.limiter = newHostLimiter() - } - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - - seen := make(map[string]struct{}, len(urls)) - uniq := make([]string, 0, len(urls)) - for _, u := range urls { - if _, ok := seen[u]; ok { - continue - } - seen[u] = struct{}{} - uniq = append(uniq, u) - } - if len(uniq) > 50 { - uniq = uniq[:50] - } - - byHost := make(map[string][]string) - for _, u := range uniq { - parsed, err := url.Parse(u) - if err != nil || parsed.Host == "" { - continue - } - byHost[parsed.Host] = append(byHost[parsed.Host], u) - } - if len(byHost) == 0 { - return - } - - var wg sync.WaitGroup - for host, us := range byHost { - wg.Add(1) - go func(host string, us []string) { - defer wg.Done() - for _, u := range us { - if err := ap.limiter.Wait(ctx, host); err != nil { - return - } - if ctx.Err() != nil { - return - } - var res searchResult - _ = ap.get(ctx, fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(u)), &res) - } - }(host, us) - } - wg.Wait() + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to search actors: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + var accounts []Account + if err := json.Unmarshal(body, &accounts); err != nil { + return nil, fmt.Errorf("failed to unmarshal accounts: %w", err) + } + results := make([]actor.Profile, 0, len(accounts)) + for _, acc := range accounts { + results = append(results, actor.Profile{ + Platform: "ActivityPub", + Handle: acc.Acct, + DisplayName: acc.DisplayName, + AvatarURL: acc.Avatar, + Bio: acc.Note, + FollowersCount: acc.FollowersCount, + FollowingCount: acc.FollowingCount, + PostCount: acc.StatusesCount, + URL: acc.URL, + }) + } + return results, nil } diff --git a/internal/activitypub/client_test.go b/internal/activitypub/client_test.go index 04fbffd..2fdfa11 100644 --- a/internal/activitypub/client_test.go +++ b/internal/activitypub/client_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "strconv" "strings" "testing" "time" @@ -17,7 +16,7 @@ type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func newTestClient(serverURL string) *APClient { - ap, err := NewAPClient("https://example.com", "tok") + ap, err := NewAPClient(context.Background(), "https://example.com", "tok") if err != nil { panic(err) } @@ -36,29 +35,22 @@ func newTestClient(serverURL string) *APClient { } func TestNewAPClientEmpty(t *testing.T) { - if _, err := NewAPClient("", "tok"); err == nil { + if _, err := NewAPClient(context.Background(), "", "tok"); err == nil { t.Fatal("expected error for empty host") } - if _, err := NewAPClient("host", ""); err == nil { + if _, err := NewAPClient(context.Background(), "host", ""); err == nil { t.Fatal("expected error for empty apikey") } } func TestGetLatestNPosts(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer tok" { w.WriteHeader(http.StatusUnauthorized) return } - if r.URL.Query().Get("max_id") != "" { - _, _ = fmt.Fprint(w, `[]`) - return - } - _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) + fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -76,92 +68,10 @@ func TestGetLatestNPosts(t *testing.T) { } } -func TestGetLatestNPostsPaginates(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Query().Get("max_id") { - case "40": - ids := make([]map[string]any, 0, 10) - for id := 41; id <= 50; id++ { - ids = append(ids, map[string]any{ - "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", - "account": map[string]any{"acct": "alice"}, - }) - } - _ = json.NewEncoder(w).Encode(ids) - case "": - ids := make([]map[string]any, 0, 40) - for id := 1; id <= 40; id++ { - ids = append(ids, map[string]any{ - "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", - "account": map[string]any{"acct": "alice"}, - }) - } - _ = json.NewEncoder(w).Encode(ids) - default: - _ = json.NewEncoder(w).Encode([]map[string]any{}) - } - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - posts, err := ap.GetLatestNPosts(context.Background(), "alice", 50) - if err != nil { - t.Fatalf("GetLatestNPosts error: %v", err) - } - if len(posts) != 50 { - t.Fatalf("expected 50 posts across 2 pages, got %d", len(posts)) - } - if posts[49].Content != "post 50" { - t.Fatalf("expected last post 'post 50', got %q", posts[49].Content) - } -} - -func TestGetLatestNPostsZeroLimit(t *testing.T) { - ap := newTestClient("http://example.com") - posts, err := ap.GetLatestNPosts(context.Background(), "alice", 0) - if err != nil { - t.Fatalf("GetLatestNPosts with zero limit error: %v", err) - } - if len(posts) != 0 { - t.Fatalf("expected no posts for zero limit, got %d", len(posts)) - } -} - -func TestGetLatestNPostsLimitClamp(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - limit := r.URL.Query().Get("limit") - l, _ := strconv.Atoi(limit) - if l > maxLimit { - t.Fatalf("expected limit <= %d, got %s", maxLimit, limit) - } - _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - posts, err := ap.GetLatestNPosts(context.Background(), "alice", 500) - if err != nil { - t.Fatalf("GetLatestNPosts error: %v", err) - } - if len(posts) == 0 { - t.Fatal("expected at least one post") - } -} - func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/statuses/42", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) + fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -176,215 +86,34 @@ func TestGetPost(t *testing.T) { } } -func TestGetPostURLResolves(t *testing.T) { - url := "https://inst.social/@alice/statuses/42" - mux := http.NewServeMux() - mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != url { - w.WriteHeader(http.StatusBadRequest) - return - } - if r.URL.Query().Get("type") != "statuses" || r.URL.Query().Get("resolve") != "true" { - w.WriteHeader(http.StatusBadRequest) - return - } - _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"42","uri":"`+url+`","url":"`+url+`","content":"resolved post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob@inst.social"}}]}`) - }) - server := httptest.NewServer(mux) +func TestGetLatestNPostsHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) defer server.Close() ap := newTestClient(server.URL) - post, err := ap.GetPost(context.Background(), url) - if err != nil { - t.Fatalf("GetPost error: %v", err) - } - if post.Content != "resolved post" || post.Author != "bob@inst.social" { - t.Fatalf("unexpected post: %+v", post) - } -} - -func TestGetPostURLNotFound(t *testing.T) { - derefTarget := httptest.NewServer(http.NotFoundHandler()) - defer derefTarget.Close() - - mux := http.NewServeMux() - mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[]}`) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - if _, err := ap.GetPost(context.Background(), derefTarget.URL+"/missed"); err == nil { - t.Fatal("expected error when search and dereference both return nothing") - } -} - -func TestGetPostURLSearchReturnsWrongPost(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"wrong","uri":"https://other.com/1","content":"wrong post"}]}`) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - if _, err := ap.GetPost(context.Background(), "https://inst.social/@alice/statuses/42"); err == nil { - t.Fatal("expected error when search returns no matching post") - } -} - -func TestGetPostURLDerefFallback(t *testing.T) { - remote, _ := newRemoteServer() - defer remote.Close() - - mux := http.NewServeMux() - mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - post, err := ap.GetPost(context.Background(), remote.URL+"/notes/2") - if err != nil { - t.Fatalf("GetPost error: %v", err) - } - if post.Content != "deref note 2" { - t.Fatalf("expected dereferenced note, got %+v", post) - } -} - -func TestGetLatestNPostsRemoteFallsBackToOutbox(t *testing.T) { - remote, remoteBase := newRemoteServer() - defer remote.Close() - - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("max_id") != "" { - _, _ = fmt.Fprint(w, `[]`) - return - } - _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached","uri":`+`"`+remoteBase+`/notes/9","created_at":"2024-03-10T00:00:00Z","account":{"acct":"alice@example.social"}}]`) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 5) - if err != nil { - t.Fatalf("GetLatestNPosts error: %v", err) - } - if len(posts) != 5 { - t.Fatalf("expected 5 merged posts, got %d: %+v", len(posts), posts) - } - if posts[0].Content != "cached" { - t.Fatalf("expected newest cached first, got %q", posts[0].Content) - } - if posts[4].Content != "note 4" { - t.Fatalf("expected oldest outbox post last, got %q", posts[4].Content) - } -} - -func TestGetLatestNPostsRemoteEmptyCacheFallsBack(t *testing.T) { - remote, remoteBase := newRemoteServer() - defer remote.Close() - - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("max_id") != "" { - _, _ = fmt.Fprint(w, `[]`) - return - } - _, _ = fmt.Fprint(w, `[]`) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 4) - if err != nil { - t.Fatalf("GetLatestNPosts error: %v", err) - } - if len(posts) != 4 { - t.Fatalf("expected all 4 posts from outbox, got %d", len(posts)) - } - if posts[0].Content != "bare note 3" { - t.Fatalf("expected outbox post newest-first, got %q", posts[0].Content) - } -} - -func TestGetPostsInTimeframeRemoteFallsBackToOutbox(t *testing.T) { - remote, remoteBase := newRemoteServer() - defer remote.Close() - - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("max_id") != "" { - _, _ = fmt.Fprint(w, `[]`) - return - } - _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached in window","uri":`+`"`+remoteBase+`/notes/5","created_at":"2024-02-20T00:00:00Z","account":{"acct":"alice@example.social"}}]`) - }) - gts := httptest.NewServer(mux) - defer gts.Close() - - ap, err := NewAPClient(gts.URL, "tok") - if err != nil { - t.Fatal(err) - } - start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) - posts, err := ap.GetPostsInTimeframe(context.Background(), "@alice@example.social", start, end, 3) - if err != nil { - t.Fatalf("GetPostsInTimeframe error: %v", err) - } - if len(posts) != 3 { - t.Fatalf("expected 3 in-window posts, got %d: %+v", len(posts), posts) - } - if posts[0].Content != "created note 1" || posts[1].Content != "cached in window" || posts[2].Content != "deref note 2" { - t.Fatalf("unexpected merged order: %+v", posts) + if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { + t.Fatal("expected error on non-200 status") } } func TestGetPostsInTimeframe(t *testing.T) { + // Feed is reverse-chronological: newer than `end`, two in-window, then older + // than `start`. The client must skip the too-new post, return both in-window + // posts, and stop when it crosses the `start` boundary. mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { feed := []map[string]any{} switch r.URL.Query().Get("max_id") { case "": + // Page 1: too-new post and one in-window post. feed = []map[string]any{ {"id": "5", "content": "too-new", "text": "too-new", "created_at": "2025-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "4", "content": "mid", "text": "mid", "created_at": "2024-06-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, } case "4": + // Page 2: in-window post then too-old post. feed = []map[string]any{ {"id": "3", "content": "early-mid", "text": "early-mid", "created_at": "2024-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "2", "content": "too-old", "text": "too-old", "created_at": "2023-01-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, @@ -413,31 +142,14 @@ func TestGetPostsInTimeframe(t *testing.T) { } } -func TestGetLatestNPostsHTTPError(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { - t.Fatal("expected error on non-200 status from timeline") - } -} - func TestGetActor(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != "alice" { + mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("acct") != "alice" { w.WriteHeader(http.StatusBadRequest) return } - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}]`) + fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -463,11 +175,11 @@ func TestGetActor(t *testing.T) { func TestGetActorIDFallback(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[]`) + mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) }) mux.HandleFunc("/api/v1/accounts/alice", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) + fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -482,29 +194,6 @@ func TestGetActorIDFallback(t *testing.T) { } } -func TestGetActorAtLocalHandle(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query().Get("q") - if q != "@alice" && q != "alice" { - w.WriteHeader(http.StatusBadRequest) - return - } - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - prof, err := ap.GetActor(context.Background(), "@alice") - if err != nil { - t.Fatalf("GetActor error: %v", err) - } - if prof.Handle != "alice" { - t.Fatalf("expected local handle, got %q", prof.Handle) - } -} - func TestSearchActors(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { @@ -512,11 +201,7 @@ func TestSearchActors(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - if r.URL.Query().Get("resolve") != "true" { - w.WriteHeader(http.StatusBadRequest) - return - } - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) + fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -533,65 +218,3 @@ func TestSearchActors(t *testing.T) { t.Fatalf("unexpected result: %+v", results[0]) } } - -func TestExtractHandleFromURL(t *testing.T) { - tests := []struct { - url string - expected string - }{ - {"https://inst.social/@alice/statuses/42", "alice"}, - {"http://example.com/@bob/posts/1", "bob"}, - {"https://no-at-here.com/notes/1", ""}, - {"https://host/@user/note", "user"}, - } - for _, tt := range tests { - got := extractHandleFromURL(tt.url) - if got != tt.expected { - t.Fatalf("extractHandleFromURL(%q) = %q, want %q", tt.url, got, tt.expected) - } - } -} - -func TestGetPostBareLocalHandle(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query().Get("q") - if q != "@alice" && q != "alice" { - w.WriteHeader(http.StatusBadRequest) - return - } - _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) - }) - mux.HandleFunc("/api/v1/statuses/alice", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `{"id":"1","content":"local post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - _, err := ap.GetPost(context.Background(), "alice") - if err != nil { - t.Fatalf("GetPost with local handle error: %v", err) - } -} - -func TestSearchActorsNilEntries(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, `[null,{"id":"1","acct":"alice@inst.social"}]`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - ap := newTestClient(server.URL) - results, err := ap.SearchActors(context.Background(), "alice", 10) - if err != nil { - t.Fatalf("SearchActors error: %v", err) - } - if len(results) != 1 { - t.Fatalf("expected 1 result after filtering nil, got %d", len(results)) - } - if results[0].Handle != "alice@inst.social" { - t.Fatalf("unexpected result: %+v", results[0]) - } -} diff --git a/internal/activitypub/outbox.go b/internal/activitypub/outbox.go deleted file mode 100644 index f7dc81f..0000000 --- a/internal/activitypub/outbox.go +++ /dev/null @@ -1,501 +0,0 @@ -package activitypub - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "sort" - "strings" - "time" -) - -const outboxMaxPages = 100 - -var webfingerScheme = "https" - -type outboxClient struct { - client *http.Client -} - -type apStringOrArray []string - -func (s *apStringOrArray) UnmarshalJSON(b []byte) error { - b = bytes.TrimSpace(b) - if len(b) == 0 || string(b) == "null" { - *s = nil - return nil - } - if b[0] == '"' { - var v string - if err := json.Unmarshal(b, &v); err != nil { - return err - } - *s = []string{v} - return nil - } - var arr []string - if err := json.Unmarshal(b, &arr); err != nil { - return err - } - *s = arr - return nil -} - -type apIRI string - -func (i *apIRI) UnmarshalJSON(b []byte) error { - b = bytes.TrimSpace(b) - if len(b) == 0 || string(b) == "null" { - *i = "" - return nil - } - if b[0] == '"' { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - *i = apIRI(s) - return nil - } - var o struct { - ID string `json:"id"` - } - if err := json.Unmarshal(b, &o); err != nil { - return err - } - *i = apIRI(o.ID) - return nil -} - -type apRef struct { - IRI string - Inline json.RawMessage -} - -func (r *apRef) UnmarshalJSON(b []byte) error { - b = bytes.TrimSpace(b) - if len(b) == 0 || string(b) == "null" { - r.IRI = "" - r.Inline = nil - return nil - } - if b[0] == '"' { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - r.IRI = s - return nil - } - r.Inline = append([]byte(nil), b...) - return nil -} - -type apActor struct { - ID string `json:"id"` - Type string `json:"type"` - PreferredUsername string `json:"preferredUsername"` - URL string `json:"url"` - Outbox apIRI `json:"outbox"` - Inbox apIRI `json:"inbox"` -} - -type apActivity struct { - ID string `json:"id"` - Type string `json:"type"` - Published string `json:"published"` - Content string `json:"content"` - URL apStringOrArray `json:"url"` - Object json.RawMessage `json:"object"` -} - -type apCollection struct { - Type string `json:"type"` - First apRef `json:"first"` -} - -type apPage struct { - Type string `json:"type"` - OrderedItems json.RawMessage `json:"orderedItems"` - Next apRef `json:"next"` -} - -type apNote struct { - ID string `json:"id"` - Type string `json:"type"` - Published string `json:"published"` - Content string `json:"content"` - URL apStringOrArray `json:"url"` -} - -type webfingerResponse struct { - Subject string `json:"subject"` - Links []struct { - Rel string `json:"rel"` - Type string `json:"type"` - Href string `json:"href"` - } `json:"links"` -} - -func newOutboxClient() *outboxClient { - return &outboxClient{client: &http.Client{Timeout: 30 * time.Second}} -} - -func (oc *outboxClient) getJSON(ctx context.Context, endpoint string, target any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Accept", `application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/json`) - - resp, err := oc.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) - } - - if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { - return fmt.Errorf("failed to decode response: %w", err) - } - return nil -} - -func (oc *outboxClient) webfinger(ctx context.Context, handle string) (string, error) { - full := strings.TrimPrefix(handle, "@") - at := strings.LastIndex(full, "@") - if at <= 0 || at == len(full)-1 { - return "", fmt.Errorf("invalid handle %q", handle) - } - domain := full[at+1:] - - endpoint := fmt.Sprintf("%s://%s/.well-known/webfinger?resource=%s", webfingerScheme, domain, url.QueryEscape("acct:"+full)) - var wf webfingerResponse - if err := oc.getJSON(ctx, endpoint, &wf); err != nil { - return "", fmt.Errorf("webfinger lookup for %q failed: %w", full, err) - } - - var htmlSelf string - for _, l := range wf.Links { - if l.Rel != "self" || l.Href == "" { - continue - } - if strings.Contains(l.Type, "activity+json") || strings.Contains(l.Type, "ld+json") { - return l.Href, nil - } - if htmlSelf == "" && strings.Contains(l.Type, "html") { - htmlSelf = l.Href - } - } - if htmlSelf != "" { - return htmlSelf, nil - } - return "", fmt.Errorf("webfinger for %q returned no self link", full) -} - -func (oc *outboxClient) getActor(ctx context.Context, actorURL string) (*apActor, error) { - var actor apActor - if err := oc.getJSON(ctx, actorURL, &actor); err != nil { - return nil, fmt.Errorf("failed to fetch actor %q: %w", actorURL, err) - } - if actor.ID == "" && actor.Outbox == "" { - return nil, fmt.Errorf("response from %q is not an actor document", actorURL) - } - return &actor, nil -} - -func (oc *outboxClient) resolveActor(ctx context.Context, actorURL, handle string) (*apActor, error) { - if actorURL == "" { - var err error - actorURL, err = oc.webfinger(ctx, handle) - if err != nil { - return nil, err - } - } - return oc.getActor(ctx, actorURL) -} - -func (oc *outboxClient) getLatest(ctx context.Context, actorURL, author string, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil - } - return oc.walk(ctx, actorURL, author, limit, func(*Post) bool { return true }) -} - -func (oc *outboxClient) getInTimeframe(ctx context.Context, actorURL, author string, start, end time.Time, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil - } - return oc.walk(ctx, actorURL, author, limit, func(p *Post) bool { - return !p.CreatedAt.Before(start) && !p.CreatedAt.After(end) - }) -} - -func (oc *outboxClient) walk(ctx context.Context, actorURL, author string, limit int, include func(*Post) bool) ([]*Post, error) { - actor, err := oc.resolveActor(ctx, actorURL, author) - if err != nil { - return nil, err - } - if actor.Outbox == "" { - return nil, fmt.Errorf("actor %q exposes no outbox", actorURL) - } - - var posts []*Post - seen := make(map[string]struct{}) - - cur, err := oc.firstPage(ctx, string(actor.Outbox)) - if err != nil { - return nil, err - } - - for pages := 0; pages < outboxMaxPages; pages++ { - if ctx.Err() != nil { - return nil, ctx.Err() - } - if len(posts) >= limit { - break - } - - page, ok, err := oc.getPage(ctx, cur) - if err != nil { - return nil, err - } - if !ok { - break - } - - var items []apActivity - if len(page.OrderedItems) > 0 { - if err := json.Unmarshal(page.OrderedItems, &items); err != nil { - return nil, fmt.Errorf("failed to decode outbox page items: %w", err) - } - } - - for _, item := range items { - p, valid, err := oc.itemToPost(ctx, item, author) - if err != nil { - return nil, err - } - if !valid || p == nil { - continue - } - if _, dup := seen[p.ID]; dup { - continue - } - seen[p.ID] = struct{}{} - if include(p) { - posts = append(posts, p) - if len(posts) >= limit { - return posts, nil - } - } - } - - cur = page.Next - if cur.IRI == "" && len(cur.Inline) == 0 { - break - } - } - - return posts, nil -} - -func (oc *outboxClient) firstPage(ctx context.Context, outbox string) (apRef, error) { - var coll apCollection - if err := oc.getJSON(ctx, outbox, &coll); err != nil { - return apRef{}, fmt.Errorf("failed to fetch outbox %q: %w", outbox, err) - } - if coll.Type != "" && coll.Type != "OrderedCollection" { - return apRef{}, fmt.Errorf("outbox %q has unexpected type %q", outbox, coll.Type) - } - if coll.First.IRI == "" && len(coll.First.Inline) == 0 { - return apRef{}, fmt.Errorf("outbox %q exposes no pages", outbox) - } - return coll.First, nil -} - -func (oc *outboxClient) getPage(ctx context.Context, ref apRef) (apPage, bool, error) { - var page apPage - switch { - case ref.IRI != "": - if err := oc.getJSON(ctx, ref.IRI, &page); err != nil { - return apPage{}, false, fmt.Errorf("failed to fetch outbox page %q: %w", ref.IRI, err) - } - case len(ref.Inline) > 0: - if err := json.Unmarshal(ref.Inline, &page); err != nil { - return apPage{}, false, fmt.Errorf("failed to decode inline outbox page: %w", err) - } - default: - return apPage{}, false, nil - } - return page, true, nil -} - -func (oc *outboxClient) itemToPost(ctx context.Context, item apActivity, author string) (*Post, bool, error) { - var note *apNote - var err error - - switch item.Type { - case "Create", "Update": - note, err = oc.noteFromObject(ctx, item.Object) - case "Note", "Article", "Question", "Page", "": - note = &apNote{ - ID: item.ID, - Type: item.Type, - Published: item.Published, - Content: item.Content, - URL: item.URL, - } - default: - return nil, false, nil - } - if err != nil { - return nil, false, err - } - if note == nil || !isPostNote(note.Type) { - return nil, false, nil - } - - id := note.ID - if id == "" && len(note.URL) > 0 { - id = note.URL[0] - } - if id == "" { - return nil, false, nil - } - - created, ok := parseActivityTime(note.Published) - if !ok || created.IsZero() { - return nil, false, nil - } - - return &Post{ - ID: id, - Author: author, - Content: note.Content, - CreatedAt: created, - }, true, nil -} - -func (oc *outboxClient) noteFromObject(ctx context.Context, raw json.RawMessage) (*apNote, error) { - raw = bytes.TrimSpace(raw) - if len(raw) == 0 || string(raw) == "null" { - return nil, nil - } - if raw[0] == '"' { - var id string - if err := json.Unmarshal(raw, &id); err != nil { - return nil, err - } - return oc.fetchNote(ctx, id) - } - var n apNote - if err := json.Unmarshal(raw, &n); err != nil { - return nil, err - } - return &n, nil -} - -func (oc *outboxClient) fetchNote(ctx context.Context, iri string) (*apNote, error) { - var n apNote - if err := oc.getJSON(ctx, iri, &n); err != nil { - return nil, fmt.Errorf("failed to dereference object %q: %w", iri, err) - } - return &n, nil -} - -func (oc *outboxClient) resolveNote(ctx context.Context, url string) (*apNote, error) { - var act apActivity - if err := oc.getJSON(ctx, url, &act); err != nil { - return nil, fmt.Errorf("failed to dereference status %q: %w", url, err) - } - switch act.Type { - case "Create": - return oc.noteFromObject(ctx, act.Object) - case "Note", "Article", "Question", "Page", "": - return &apNote{ - ID: act.ID, - Type: act.Type, - Published: act.Published, - Content: act.Content, - URL: act.URL, - }, nil - default: - return nil, fmt.Errorf("document at %q is not a post (type %q)", url, act.Type) - } -} - -func isPostNote(t string) bool { - switch t { - case "", "Note", "Article", "Question", "Page": - return true - } - return false -} - -func parseActivityTime(s string) (time.Time, bool) { - if s == "" { - return time.Time{}, false - } - for _, layout := range []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05"} { - if t, err := time.Parse(layout, s); err == nil { - return t, true - } - } - return time.Time{}, false -} - -func noteToInternalPost(n *apNote, author string) *internalPost { - p := &internalPost{ - ID: n.ID, - URI: n.ID, - Content: n.Content, - } - if len(n.URL) > 0 { - p.URL = n.URL[0] - } - if t, ok := parseActivityTime(n.Published); ok { - p.CreatedAt = t - } - if author != "" { - p.Account = Account{Acct: author} - } - return p -} - -func mergePostsBySource(cached []*internalPost, outbox []*Post) []*Post { - seen := make(map[string]struct{}, len(cached)) - res := make([]*Post, 0, len(cached)+len(outbox)) - for _, c := range cached { - key := c.URI - if key == "" { - key = c.ID - } - if key != "" { - seen[key] = struct{}{} - } - if c != nil { - res = append(res, c.toPost()) - } - } - for _, p := range outbox { - if p == nil || p.ID == "" { - continue - } - if _, dup := seen[p.ID]; dup { - continue - } - seen[p.ID] = struct{}{} - res = append(res, p) - } - sort.SliceStable(res, func(i, j int) bool { return res[i].CreatedAt.After(res[j].CreatedAt) }) - return res -} diff --git a/internal/activitypub/outbox_test.go b/internal/activitypub/outbox_test.go deleted file mode 100644 index fb8c733..0000000 --- a/internal/activitypub/outbox_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package activitypub - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -// newRemoteServer serves a synthetic ActivityPub instance: a webfinger -// endpoint, an actor with an outbox collection, two outbox pages (with mixed -// activity types and an IRI-referenced object) and a dereferenceable note. -// Returns the server and the actor's canonical URL. -func newRemoteServer() (*httptest.Server, string) { - base := "" - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body string - switch { - case strings.HasSuffix(r.URL.Path, "/.well-known/webfinger"): - body = `{"subject":"acct:alice@example.social","links":[` + - `{"rel":"self","type":"application/activity+json","href":"` + base + `/users/alice"},` + - `{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"` + base + `/@alice"}]}` - case r.URL.Path == "/users/alice": - body = `{"id":"` + base + `/users/alice","type":"Person","preferredUsername":"alice","url":"` + base + `/@alice","outbox":"` + base + `/users/alice/outbox"}` - case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "": - body = `{"type":"OrderedCollection","totalItems":5,"first":"` + base + `/users/alice/outbox?page=1"}` - case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "1": - body = `{"type":"OrderedCollectionPage","partOf":"` + base + `/users/alice/outbox","orderedItems":[` + - `{"type":"Create","published":"2024-03-01T00:00:00Z","object":{"type":"Note","id":"` + base + `/notes/1","published":"2024-03-01T00:00:00Z","content":"created note 1","url":"` + base + `/@alice/statuses/1"}},` + - `{"type":"Announce","object":"` + base + `/notes/9"},` + - `{"type":"Like","object":"` + base + `/notes/8"},` + - `{"type":"Create","object":"` + base + `/notes/2"},` + - `{"type":"Note","id":"` + base + `/notes/3","published":"2024-03-03T00:00:00Z","content":"bare note 3","url":["` + base + `/@alice/statuses/3"]}` + - `],"next":"` + base + `/users/alice/outbox?page=2"}` - case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "2": - body = `{"type":"OrderedCollectionPage","orderedItems":[` + - `{"type":"Create","object":{"type":"Note","id":"` + base + `/notes/4","published":"2024-01-15T00:00:00Z","content":"note 4"}}` + - `]}` - case r.URL.Path == "/notes/2": - body = `{"type":"Note","id":"` + base + `/notes/2","published":"2024-02-01T00:00:00Z","content":"deref note 2","url":"` + base + `/@alice/statuses/2"}` - default: - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/activity+json") - _, _ = fmt.Fprint(w, body) - })) - base = server.URL - return server, base + "/users/alice" -} - -func TestOutboxGetLatest(t *testing.T) { - server, actorURL := newRemoteServer() - defer server.Close() - - oc := newOutboxClient() - posts, err := oc.getLatest(context.Background(), actorURL, "alice@example.social", 10) - if err != nil { - t.Fatalf("getLatest error: %v", err) - } - if len(posts) != 4 { - t.Fatalf("expected 4 Create/Note posts (skipping Announce/Like), got %d: %+v", len(posts), posts) - } - want := []string{"created note 1", "deref note 2", "bare note 3", "note 4"} - for i, w := range want { - if posts[i].Content != w { - t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) - } - } - if posts[0].Author != "alice@example.social" { - t.Fatalf("Author = %q, want handle", posts[0].Author) - } -} - -func TestOutboxGetLatestLimit(t *testing.T) { - server, actorURL := newRemoteServer() - defer server.Close() - - oc := newOutboxClient() - posts, err := oc.getLatest(context.Background(), actorURL, "h", 2) - if err != nil { - t.Fatalf("getLatest error: %v", err) - } - if len(posts) != 2 { - t.Fatalf("expected 2 posts, got %d", len(posts)) - } - want := []string{"created note 1", "deref note 2"} - for i, w := range want { - if posts[i].Content != w { - t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) - } - } -} - -func TestOutboxGetInTimeframe(t *testing.T) { - server, actorURL := newRemoteServer() - defer server.Close() - - oc := newOutboxClient() - start := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) - posts, err := oc.getInTimeframe(context.Background(), actorURL, "h", start, end, 10) - if err != nil { - t.Fatalf("getInTimeframe error: %v", err) - } - if len(posts) != 2 { - t.Fatalf("expected 2 in-window posts (note 3 and 4 are outside), got %d", len(posts)) - } - if posts[0].Content != "created note 1" || posts[1].Content != "deref note 2" { - t.Fatalf("unexpected in-window posts: %+v", posts) - } -} - -func TestOutboxResolveViaWebfinger(t *testing.T) { - server, _ := newRemoteServer() - defer server.Close() - - scheme := webfingerScheme - webfingerScheme = "http" - defer func() { webfingerScheme = scheme }() - - handle := "@alice@" + strings.TrimPrefix(server.URL, "http://") - oc := newOutboxClient() - posts, err := oc.getLatest(context.Background(), "", handle, 10) - if err != nil { - t.Fatalf("getLatest via webfinger error: %v", err) - } - if len(posts) != 4 { - t.Fatalf("expected 4 posts via webfinger, got %d", len(posts)) - } -} - -func TestOutboxResolveNote(t *testing.T) { - server, _ := newRemoteServer() - defer server.Close() - - oc := newOutboxClient() - note, err := oc.resolveNote(context.Background(), server.URL+"/notes/2") - if err != nil { - t.Fatalf("resolveNote error: %v", err) - } - if note.Content != "deref note 2" { - t.Fatalf("unexpected note: %+v", note) - } -} - -func TestNoteToInternalPostSetsAuthor(t *testing.T) { - note := &apNote{ - ID: "https://example.social/notes/1", - Content: "hello", - Published: "2024-01-01T00:00:00Z", - } - p := noteToInternalPost(note, "alice@example.social") - if p.Account.Acct != "alice@example.social" { - t.Fatalf("expected Account.Acct = 'alice@example.social', got %q", p.Account.Acct) - } - if p.toPost().Author != "alice@example.social" { - t.Fatalf("expected toPost().Author = 'alice@example.social', got %q", p.toPost().Author) - } -} - -func TestNoteToInternalPostNoAuthor(t *testing.T) { - note := &apNote{ - ID: "https://example.social/notes/1", - Content: "hello", - Published: "2024-01-01T00:00:00Z", - } - p := noteToInternalPost(note, "") - if p.Account.Acct != "" { - t.Fatalf("expected empty Account.Acct when no author, got %q", p.Account.Acct) - } - if p.toPost().Author != "" { - t.Fatalf("expected empty author when none provided, got %q", p.toPost().Author) - } -} - -func TestMergePostsBySourceDedup(t *testing.T) { - created := time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC) - cached := []*internalPost{ - {ID: "local-1", URI: "https://example.social/notes/1", Content: "cached", CreatedAt: created}, - } - outbox := []*Post{ - {ID: "https://example.social/notes/1", Content: "cached dup", CreatedAt: created}, - {ID: "https://example.social/notes/2", Content: "outbox only", CreatedAt: created.Add(-time.Hour)}, - } - merged := mergePostsBySource(cached, outbox) - if len(merged) != 2 { - t.Fatalf("expected dedup to drop the duplicate, got %d", len(merged)) - } - if merged[0].Content != "cached" { - t.Fatalf("newest first: expected cached first, got %q", merged[0].Content) - } - if merged[1].Content != "outbox only" { - t.Fatalf("expected outbox only second, got %q", merged[1].Content) - } -} - -func TestParseActivityTime(t *testing.T) { - for _, s := range []string{"2024-01-01T00:00:00Z", "2024-01-01T00:00:00.000Z", "2024-01-01T00:00:00+02:00"} { - if _, ok := parseActivityTime(s); !ok { - t.Fatalf("parseActivityTime(%q) failed", s) - } - } - if _, ok := parseActivityTime("not a date"); ok { - t.Fatal("expected invalid timestamp to fail") - } -} - -func TestAPStringOrArray(t *testing.T) { - var sa apStringOrArray - if err := json.Unmarshal([]byte(`"https://a"`), &sa); err != nil || len(sa) != 1 || sa[0] != "https://a" { - t.Fatalf("string form failed: %v %v", sa, err) - } - if err := json.Unmarshal([]byte(`["https://a","https://b"]`), &sa); err != nil || len(sa) != 2 { - t.Fatalf("array form failed: %v %v", sa, err) - } - if err := json.Unmarshal([]byte(`null`), &sa); err != nil || len(sa) != 0 { - t.Fatalf("null form failed: %v %v", sa, err) - } -} diff --git a/internal/activitypub/throttle.go b/internal/activitypub/throttle.go deleted file mode 100644 index 19287d5..0000000 --- a/internal/activitypub/throttle.go +++ /dev/null @@ -1,58 +0,0 @@ -package activitypub - -import ( - "context" - "sync" - "time" -) - -type hostLimiter struct { - mu sync.Mutex - buckets map[string]*bucket -} - -type bucket struct { - mu sync.Mutex - last time.Time -} - -func newHostLimiter() *hostLimiter { - return &hostLimiter{buckets: make(map[string]*bucket)} -} - -func (h *hostLimiter) Wait(ctx context.Context, host string) error { - h.mu.Lock() - b, ok := h.buckets[host] - if !ok { - b = &bucket{} - h.buckets[host] = b - if len(h.buckets) > 1024 { - for k, v := range h.buckets { - if time.Since(v.last) > 10*time.Minute { - delete(h.buckets, k) - } - } - } - } - h.mu.Unlock() - - b.mu.Lock() - defer b.mu.Unlock() - - if b.last.IsZero() { - b.last = time.Now() - return nil - } - wait := time.Until(b.last.Add(time.Second)) - if wait <= 0 { - b.last = time.Now() - return nil - } - select { - case <-time.After(wait): - b.last = time.Now() - return nil - case <-ctx.Done(): - return ctx.Err() - } -} diff --git a/internal/activitypub/types.go b/internal/activitypub/types.go index 1d9fc27..269604d 100644 --- a/internal/activitypub/types.go +++ b/internal/activitypub/types.go @@ -6,6 +6,7 @@ type Post struct { ID string `json:"id"` Author string `json:"author"` Content string `json:"content"` + Text string `json:"text"` CreatedAt time.Time `json:"created_at"` FavouritesCount int `json:"favourites_count"` ReblogsCount int `json:"reblogs_count"` @@ -13,18 +14,16 @@ type Post struct { } func (i *internalPost) toPost() *Post { - p := &Post{ + return &Post{ ID: i.ID, + Author: i.Account.Acct, Content: i.Content, + Text: i.Text, CreatedAt: i.CreatedAt, FavouritesCount: i.FavouritesCount, ReblogsCount: i.ReblogsCount, RepliesCount: i.RepliesCount, } - if i.Account.Acct != "" { - p.Author = i.Account.Acct - } - return p } type internalPost struct { @@ -58,6 +57,7 @@ type internalPost struct { Sensitive bool `json:"sensitive"` SpoilerText string `json:"spoiler_text"` Tags []Tag `json:"tags"` + Text string `json:"text"` URI string `json:"uri"` URL string `json:"url"` Visibility string `json:"visibility"` @@ -270,10 +270,3 @@ type Tag struct { Name string `json:"name"` URL string `json:"url"` } - -// searchResult mirrors the /api/v2/search response: statuses are the api -// `status` objects served by the search endpoint. -type searchResult struct { - Accounts []*Account `json:"accounts"` - Statuses []*internalPost `json:"statuses"` -} diff --git a/internal/bluesky/client.go b/internal/bluesky/client.go index edb6359..2ddc4bf 100644 --- a/internal/bluesky/client.go +++ b/internal/bluesky/client.go @@ -6,7 +6,6 @@ import ( "fmt" "net/url" "strings" - "sync" "time" "github.com/bluesky-social/indigo/api/atproto" @@ -17,36 +16,8 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) -const ( - // maxBskyFeedLimit is the upper bound the Bluesky API accepts for ?limit=. - // Higher values are silently clamped server-side. - maxBskyFeedLimit = 100 - - // maxAuthorFeedPages caps the number of author-feed pages GetPostsInTimeframe - // scans. At 50 posts/page this allows up to ~5,000 statuses per window before - // giving up. - maxAuthorFeedPages = 100 - - // didCacheTTL bounds how long a handle->DID resolution is reused. Handles - // almost never re-map, but the TTL guarantees a long-running process still - // notices handle changes instead of pinning a stale DID forever. - didCacheTTL = 24 * time.Hour -) - type BlueskyClient struct { - client *xrpc.Client - authMu sync.RWMutex - refreshMu sync.Mutex - cacheMu sync.Mutex - didCache map[string]didCacheEntry -} - -// didCacheEntry holds a positive (successfully resolved) handle->DID mapping -// and when it expires. Failures are never cached, so transient resolution -// errors are retried on the next call. -type didCacheEntry struct { - did string - expiresAt time.Time + client *xrpc.Client } func NewBlueskyClient(ctx context.Context, handle, password, host string) (*BlueskyClient, error) { @@ -68,81 +39,37 @@ func NewBlueskyClient(ctx context.Context, handle, password, host string) (*Blue Handle: session.Handle, Did: session.Did, } - return &BlueskyClient{ - client: client, - didCache: make(map[string]didCacheEntry), - }, nil + return &BlueskyClient{client: client}, nil } -// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the -// session is refreshed once and fn is retried. -// -// refreshMu serializes refresh calls: Bluesky refresh tokens are single-use and -// rotated, so two concurrent refreshes would race and one should fail for no -// reason. After acquiring the lock the error is re-checked, because a previous -// goroutine may already have refreshed the session while this one waited. +// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the session is +// refreshed once and fn is retried. This makes long-running and background enrichment calls +// resilient to Bluesky access-token expiry without callers having to handle it. func (c *BlueskyClient) withRefresh(ctx context.Context, fn func() error) error { - if err := fn(); err == nil { - return nil - } else if !isAuthError(err) { - return err - } - - c.refreshMu.Lock() - defer c.refreshMu.Unlock() - - // Another goroutine may have refreshed while we waited for the lock. - err := fn() - if err == nil { + firstErr := fn() + if firstErr == nil { return nil } - if !isAuthError(err) { - return err + if !isAuthError(firstErr) { + return firstErr } - if rerr := c.refreshSession(ctx); rerr != nil { log.WarnWithContext(ctx, "bluesky session refresh failed", "error", rerr) - return err + return firstErr } return fn() } -// clientSnapshot returns a copy of the xrpc client that requests must use. It -// pins the current, immutable AuthInfo under the read lock: refreshSession -// replaces the Auth pointer under the exclusive lock rather than mutating the -// old value in place, so every goroutine sharing a pinned AuthInfo reads a -// complete, consistent token. xrpc.Client.Do reads client.Auth without any -// locking of its own, so no request may touch c.client.Auth directly. -func (c *BlueskyClient) clientSnapshot() *xrpc.Client { - c.authMu.RLock() - defer c.authMu.RUnlock() - cp := *c.client - return &cp -} - func (c *BlueskyClient) refreshSession(ctx context.Context) error { - refreshed, err := atproto.ServerRefreshSession(ctx, c.clientSnapshot()) + refreshed, err := atproto.ServerRefreshSession(ctx, c.client) if err != nil { return fmt.Errorf("failed to refresh bluesky session: %w", err) } - - c.authMu.Lock() - defer c.authMu.Unlock() - c.client.Auth = &xrpc.AuthInfo{ - AccessJwt: refreshed.AccessJwt, - RefreshJwt: refreshed.RefreshJwt, - Handle: c.client.Auth.Handle, - Did: c.client.Auth.Did, - } + c.client.Auth.AccessJwt = refreshed.AccessJwt + c.client.Auth.RefreshJwt = refreshed.RefreshJwt return nil } -// isAuthError reports whether err is an auth/expired-token XRPC error. The -// ErrStr switch covers the canonical cases; the message-substring fallback is a -// deliberate heuristic for third-party PDS instances that return nonstandard -// error strings. It can false-positive (an unrelated message that happens to -// contain "expired"/"unauthorized"/"token"), which only costs one gratuitous -// refresh + retry. func isAuthError(err error) bool { var xe *xrpc.XRPCError if !errors.As(err, &xe) { @@ -164,18 +91,14 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe return err } - uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) + var uri string if collection != "" { uri = fmt.Sprintf("at://%s/%s/%s", repo.String(), collection, recordKey) + } else { + uri = fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) } - // collection and recordKey are caller-supplied; reject a malformed - // at-uri before spending a network round-trip. - if _, err := syntax.ParseATURI(uri); err != nil { - return fmt.Errorf("invalid at uri %q: %w", uri, err) - } - log.DebugWithContext(ctx, "Fetching post", "URI", uri) - resp, err := bsky.FeedGetPosts(ctx, c.clientSnapshot(), []string{uri}) + resp, err := bsky.FeedGetPosts(ctx, c.client, []string{uri}) if err != nil { return fmt.Errorf("failed to get post: %w", err) } @@ -189,21 +112,13 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe } func (c *BlueskyClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*bsky.FeedDefs_FeedViewPost, error) { - if limit <= 0 { - return nil, nil - } - pageSize := limit - if pageSize > maxBskyFeedLimit { - pageSize = maxBskyFeedLimit - } - var result []*bsky.FeedDefs_FeedViewPost err := c.withRefresh(ctx, func() error { repo, err := c.resolveUser(ctx, actor) if err != nil { return err } - resp, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), "", "posts", false, int64(pageSize)) + resp, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), "", "posts", false, int64(limit)) if err != nil { return fmt.Errorf("failed to get latest posts: %w", err) } @@ -224,11 +139,8 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s var posts []*bsky.FeedDefs_FeedViewPost var cursor string - // The author feed is reverse-chronological, so the first post older - // than start terminates the scan; maxAuthorFeedPages is a belt-and- - // braces guard against a misbehaving server. - for pages := 0; pages < maxAuthorFeedPages; pages++ { - feed, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), cursor, "posts", false, 50) + for len(posts) < limit { + feed, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), cursor, "posts", false, 50) if err != nil { return err } @@ -249,18 +161,12 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s continue } - if createdAt.After(end) { - continue - } - if createdAt.Before(start) { - // Reverse-chronological: nothing older is in-window. - result = posts - return nil - } - posts = append(posts, post) - if len(posts) >= limit { - result = posts - return nil + if createdAt.After(start) && createdAt.Before(end) { + posts = append(posts, post) + if len(posts) >= limit { + result = posts + return nil + } } } @@ -280,17 +186,9 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) if err != nil { return nil, err } - pageSize := limit - if pageSize <= 0 { - pageSize = 50 - } - if pageSize > maxBskyFeedLimit { - pageSize = maxBskyFeedLimit - } - var result *bsky.FeedGetFeed_Output err = c.withRefresh(ctx, func() error { - resp, err := bsky.FeedGetFeed(ctx, c.clientSnapshot(), "", uri, int64(pageSize)) + resp, err := bsky.FeedGetFeed(ctx, c.client, "", uri, int64(50)) if err != nil { return fmt.Errorf("failed to get feed: %w", err) } @@ -300,33 +198,20 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) return result, err } -// resolveUser converts a handle or DID into a syntax.DID. Handle resolutions -// are cached for didCacheTTL; DIDs pass through unmodified. Only successful -// resolutions are cached. func (c *BlueskyClient) resolveUser(ctx context.Context, user string) (syntax.DID, error) { + var did syntax.DID if d, err := syntax.ParseDID(user); err == nil { - return d, nil + did = d + } else { + resp, err := atproto.IdentityResolveHandle(ctx, c.client, user) + if err != nil { + return "", fmt.Errorf("failed to resolve handle to DID: %w", err) + } + did, err = syntax.ParseDID(resp.Did) + if err != nil { + return "", fmt.Errorf("failed to parse resolved DID: %w", err) + } } - - c.cacheMu.Lock() - if e, ok := c.didCache[user]; ok && time.Now().Before(e.expiresAt) { - c.cacheMu.Unlock() - return syntax.DID(e.did), nil - } - c.cacheMu.Unlock() - - resp, err := atproto.IdentityResolveHandle(ctx, c.clientSnapshot(), user) - if err != nil { - return "", fmt.Errorf("failed to resolve handle to DID: %w", err) - } - did, err := syntax.ParseDID(resp.Did) - if err != nil { - return "", fmt.Errorf("failed to parse resolved DID: %w", err) - } - - c.cacheMu.Lock() - c.didCache[user] = didCacheEntry{did: did.String(), expiresAt: time.Now().Add(didCacheTTL)} - c.cacheMu.Unlock() return did, nil } @@ -339,7 +224,7 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro err := c.withRefresh(ctx, func() error { params := map[string]any{"actor": author} var prof bskyActorProfile - if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { + if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { return fmt.Errorf("failed to get actor profile: %w", err) } result = &actor.Profile{ @@ -351,30 +236,30 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro FollowersCount: prof.FollowersCount, FollowingCount: prof.FollowsCount, PostCount: prof.PostsCount, - URL: blueskyProfileURL(prof.Handle), + URL: prof.URL, } return nil }) return result, err } -func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { - var result []*actor.Profile +func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { + var result []actor.Profile err := c.withRefresh(ctx, func() error { params := map[string]any{"q": query, "limit": limit} var out bskyActorSearch - if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { + if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { return fmt.Errorf("failed to search actors: %w", err) } - results := make([]*actor.Profile, 0, len(out.Actors)) + results := make([]actor.Profile, 0, len(out.Actors)) for _, a := range out.Actors { - results = append(results, &actor.Profile{ + results = append(results, actor.Profile{ Platform: "Bluesky", Handle: a.Handle, DisplayName: a.DisplayName, AvatarURL: a.Avatar, Bio: a.Description, - URL: blueskyProfileURL(a.Handle), + URL: a.URL, }) } result = results @@ -383,15 +268,6 @@ func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit in return result, err } -// blueskyProfileURL derives the canonical public profile URL from the handle. -// app.bsky.actor.getProfile/searchActors do not return a url field. -func blueskyProfileURL(handle string) string { - if handle == "" { - return "" - } - return "https://bsky.app/profile/" + strings.TrimPrefix(handle, "@") -} - type bskyActorProfile struct { Did string `json:"did"` Handle string `json:"handle"` @@ -401,6 +277,7 @@ type bskyActorProfile struct { FollowersCount int `json:"followersCount"` FollowsCount int `json:"followsCount"` PostsCount int `json:"postsCount"` + URL string `json:"url"` } type bskyActorSearch struct { @@ -410,6 +287,7 @@ type bskyActorSearch struct { DisplayName string `json:"displayName"` Avatar string `json:"avatar"` Description string `json:"description"` + URL string `json:"url"` } `json:"actors"` } @@ -425,10 +303,6 @@ func (c *BlueskyClient) sanitizeFeedURI(ctx context.Context, input string) (stri return "", fmt.Errorf("invalid URI: %w", err) } - if u.Scheme != "http" && u.Scheme != "https" { - return "", fmt.Errorf("unrecognized URI form (scheme must be http/https): %s", input) - } - parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) >= 4 && parts[0] == "profile" && parts[2] == "feed" { didRaw := parts[1] diff --git a/internal/bluesky/client_test.go b/internal/bluesky/client_test.go index 599edbb..73e9977 100644 --- a/internal/bluesky/client_test.go +++ b/internal/bluesky/client_test.go @@ -7,8 +7,6 @@ import ( "net/http" "net/http/httptest" "strconv" - "sync" - "sync/atomic" "testing" "time" @@ -33,7 +31,7 @@ func setupMockServer() *httptest.Server { if input.Identifier == "fail.test" { w.WriteHeader(http.StatusUnauthorized) - _, _ = fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) + fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) return } @@ -45,7 +43,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:test", } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetAuthorFeed @@ -83,7 +81,7 @@ func setupMockServer() *httptest.Server { Feed: feed, } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetFeed @@ -109,7 +107,7 @@ func setupMockServer() *httptest.Server { }, } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetPosts (used by GetPost) @@ -142,7 +140,7 @@ func setupMockServer() *httptest.Server { Posts: posts, } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.getProfile (used by GetActor) @@ -162,9 +160,10 @@ func setupMockServer() *httptest.Server { "followersCount": 100, "followsCount": 50, "postsCount": 200, + "url": "https://bsky.app/profile/" + actor, } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.searchActors (used by SearchActors) @@ -189,12 +188,13 @@ func setupMockServer() *httptest.Server { "displayName": fmt.Sprintf("Test User %d", i), "avatar": "https://example.com/avatar.jpg", "description": fmt.Sprintf("Bio for user %d", i), + "url": fmt.Sprintf("https://bsky.app/profile/%s%d.test", query, i), }) } resp := map[string]any{"actors": actors} w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.identity.resolveHandle (used by ResolveUser) @@ -209,7 +209,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:resolved", } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.server.refreshSession (used by withRefresh) @@ -218,7 +218,7 @@ func setupMockServer() *httptest.Server { auth := r.Header.Get("Authorization") if auth == "" { w.WriteHeader(http.StatusUnauthorized) - _ = json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) + json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) return } @@ -227,7 +227,7 @@ func setupMockServer() *httptest.Server { RefreshJwt: "refreshed_refresh_jwt", } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) }) return httptest.NewServer(mux) @@ -292,7 +292,6 @@ func newTestClient(t *testing.T, serverURL string) *BlueskyClient { AccessJwt: "test_access_jwt", }, }, - didCache: make(map[string]didCacheEntry), } } @@ -303,7 +302,7 @@ func TestGetLatestNPosts(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() actor := "did:plc:testactor" - limit := 5 + var limit int = 5 output, err := client.GetLatestNPosts(ctx, actor, limit) if err != nil { @@ -330,7 +329,7 @@ func TestGetFeed(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() feedURI := "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.generator/with-friends" - limit := 10 + var limit int = 10 output, err := client.GetFeed(ctx, feedURI, limit) if err != nil { @@ -386,14 +385,13 @@ func TestGetPost(t *testing.T) { } func TestGetPostsInTimeframe(t *testing.T) { - // The mock author-feed is reverse-chronological: page 1 has posts newer than - // the window end (skipped, cursor advances), page 2 has the in-range posts - // followed by one older than start (which terminates the scan). + // The mock author-feed returns two pages: the first contains out-of-range posts, + // then in-range posts, verifying both date filtering and cursor pagination. requests := 0 mux := http.NewServeMux() mux.HandleFunc("/xrpc/com.atproto.server.createSession", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) + json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) }) mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { requests++ @@ -415,22 +413,22 @@ func TestGetPostsInTimeframe(t *testing.T) { var feed []*bsky.FeedDefs_FeedViewPost var cursor string if requests == 1 { - // Page 1: one post after the window end (skipped), with a cursor. + // Page 1: one out-of-range (before start), one out-of-range (after end), and a cursor. feed = []*bsky.FeedDefs_FeedViewPost{ + mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/4", "2024-01-01T00:00:00Z"), } cursor = "page-2-cursor" } else { - // Page 2: two in-range posts, then one older than start (terminates). + // Page 2: two in-range posts and no cursor (end of feed). feed = []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/2", "2023-08-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), - mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/2", "2023-08-01T00:00:00Z"), } } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) + json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) }) server := httptest.NewServer(mux) @@ -495,9 +493,6 @@ func TestGetActor(t *testing.T) { if profile.Platform != "Bluesky" { t.Errorf("Expected platform 'Bluesky', got %q", profile.Platform) } - if profile.URL != "https://bsky.app/profile/"+actor { - t.Errorf("Expected URL %q, got %q", "https://bsky.app/profile/"+actor, profile.URL) - } if profile.FollowersCount != 100 || profile.FollowingCount != 50 || profile.PostCount != 200 { t.Errorf("Unexpected counts: followers=%d, following=%d, posts=%d", profile.FollowersCount, profile.FollowingCount, profile.PostCount) } @@ -534,10 +529,6 @@ func TestSearchActors(t *testing.T) { if result.Platform != "Bluesky" { t.Errorf("Result %d: expected platform 'Bluesky', got %q", i, result.Platform) } - expectedURL := "https://bsky.app/profile/" + expectedHandle - if result.URL != expectedURL { - t.Errorf("Result %d: expected URL %q, got %q", i, expectedURL, result.URL) - } } } @@ -592,12 +583,6 @@ func TestSanitizeFeedURI(t *testing.T) { expected: "at://did:plc:abc/app.bsky.feed.generator/xyz", wantErr: false, }, - { - name: "Non-HTTP scheme", - input: "gopher://bsky.app/profile/did:plc:abc/feed/xyz", - expected: "", - wantErr: true, - }, { name: "Invalid URI", input: "not-a-uri", @@ -624,322 +609,3 @@ func TestSanitizeFeedURI(t *testing.T) { }) } } - -func TestGetLatestNPostsLimitClamp(t *testing.T) { - server := setupMockServer() - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - actor := "did:plc:testactor" - - t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { - output, err := client.GetLatestNPosts(ctx, actor, 250) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(output) != maxBskyFeedLimit { - t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output)) - } - }) - - t.Run("ZeroLimitReturnsNil", func(t *testing.T) { - output, err := client.GetLatestNPosts(ctx, actor, 0) - if err != nil { - t.Fatalf("Expected no error with zero limit, got %v", err) - } - if output != nil { - t.Errorf("Expected nil output for zero limit, got %d posts", len(output)) - } - }) - - t.Run("NegativeLimitReturnsNil", func(t *testing.T) { - output, err := client.GetLatestNPosts(ctx, actor, -5) - if err != nil { - t.Fatalf("Expected no error with negative limit, got %v", err) - } - if output != nil { - t.Errorf("Expected nil output for negative limit, got %d posts", len(output)) - } - }) -} - -func TestGetFeedHonorsLimit(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/app.bsky.feed.getFeed", func(w http.ResponseWriter, r *http.Request) { - limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) - feed := make([]*bsky.FeedDefs_FeedViewPost, 0, limit) - for i := 0; i < limit; i++ { - feed = append(feed, &bsky.FeedDefs_FeedViewPost{ - Post: &bsky.FeedDefs_PostView{ - Uri: fmt.Sprintf("at://did:plc:test/app.bsky.feed.post/%d", i), - Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:test", Handle: "test.handle"}, - }, - }) - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(bsky.FeedGetFeed_Output{Feed: feed}) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - feedURI := "at://did:plc:abc/app.bsky.feed.generator/xyz" - - t.Run("ExplicitLimitPassedThrough", func(t *testing.T) { - output, err := client.GetFeed(ctx, feedURI, 75) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(output.Feed) != 75 { - t.Errorf("Expected 75 posts, got %d", len(output.Feed)) - } - }) - - t.Run("ZeroLimitDefaultsTo50", func(t *testing.T) { - output, err := client.GetFeed(ctx, feedURI, 0) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(output.Feed) != 50 { - t.Errorf("Expected 50 posts (default), got %d", len(output.Feed)) - } - }) - - t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { - output, err := client.GetFeed(ctx, feedURI, 1000) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(output.Feed) != maxBskyFeedLimit { - t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output.Feed)) - } - }) -} - -func TestGetPostInvalidRecordKey(t *testing.T) { - requests := 0 - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/app.bsky.feed.getPosts", func(w http.ResponseWriter, r *http.Request) { - requests++ - http.Error(w, `{"error":"InternalServerError"}`, http.StatusInternalServerError) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - - post, err := client.GetPost(ctx, "did:plc:testactor", "", "bad/key") - if post != nil { - t.Fatalf("Expected nil post, got %v", post) - } - if err == nil { - t.Fatal("Expected error for invalid record key, got nil") - } - if requests != 0 { - t.Errorf("Expected no network request for invalid record key, got %d", requests) - } -} - -func TestWithRefreshConcurrent(t *testing.T) { - var refreshCount atomic.Int32 - - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/app.bsky.actor.getProfile", func(w http.ResponseWriter, r *http.Request) { - auth := r.Header.Get("Authorization") - if auth != "Bearer refreshed_access_jwt" { - // The initial access token is expired; the session must be refreshed - // before the retry succeeds. - w.WriteHeader(http.StatusUnauthorized) - _ = json.NewEncoder(w).Encode(map[string]string{"error": "ExpiredToken", "message": "Token expired, refresh required"}) - return - } - resp := map[string]any{ - "did": "did:plc:test", - "handle": "test.handle", - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(resp) - }) - mux.HandleFunc("/xrpc/com.atproto.server.refreshSession", func(w http.ResponseWriter, r *http.Request) { - refreshCount.Add(1) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(atproto.ServerRefreshSession_Output{ - AccessJwt: "refreshed_access_jwt", - RefreshJwt: "refreshed_refresh_jwt", - }) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - if _, err := client.GetActor(ctx, "test.handle"); err != nil { - t.Errorf("expected no error, got %v", err) - } - }() - } - wg.Wait() - - if refreshCount.Load() != 1 { - t.Errorf("expected exactly 1 session refresh under concurrency, got %d", refreshCount.Load()) - } -} - -func TestGetPostsInTimeframeStopsAtStart(t *testing.T) { - requests := 0 - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { - requests++ - mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { - return &bsky.FeedDefs_FeedViewPost{ - Post: &bsky.FeedDefs_PostView{ - Uri: uri, - Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, - Record: &util.LexiconTypeDecoder{ - Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, - }, - }, - } - } - feed := []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), - mk("at://did:plc:testactor/app.bsky.feed.post/0", "2022-01-01T00:00:00Z"), - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - - start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) - - posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(posts) != 1 { - t.Fatalf("expected 1 in-range post before early exit, got %d", len(posts)) - } - if requests != 1 { - t.Errorf("expected scan to stop after the first page, made %d requests", requests) - } -} - -func TestGetPostsInTimeframeIncludesBoundaries(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { - mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { - return &bsky.FeedDefs_FeedViewPost{ - Post: &bsky.FeedDefs_PostView{ - Uri: uri, - Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, - Record: &util.LexiconTypeDecoder{ - Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, - }, - }, - } - } - feed := []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/end", "2023-12-31T23:59:59Z"), - mk("at://did:plc:testactor/app.bsky.feed.post/start", "2023-01-01T00:00:00Z"), - mk("at://did:plc:testactor/app.bsky.feed.post/older", "2022-12-31T23:59:59Z"), - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - - start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) - - posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - // Both boundary posts are inclusive; the post one second before start is not. - if len(posts) != 2 { - t.Fatalf("expected boundary posts to be included (got %d)", len(posts)) - } - got := map[string]bool{} - for _, p := range posts { - got[p.Post.Uri] = true - } - if !got["at://did:plc:testactor/app.bsky.feed.post/start"] { - t.Error("expected post exactly at start to be included") - } - if !got["at://did:plc:testactor/app.bsky.feed.post/end"] { - t.Error("expected post exactly at end to be included") - } -} - -func TestResolveUserCache(t *testing.T) { - var resolveRequests atomic.Int32 - mux := http.NewServeMux() - mux.HandleFunc("/xrpc/com.atproto.identity.resolveHandle", func(w http.ResponseWriter, r *http.Request) { - resolveRequests.Add(1) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(atproto.IdentityResolveHandle_Output{Did: "did:plc:resolved"}) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - client := newTestClient(t, server.URL) - ctx := context.Background() - - // First resolution hits the network and populates the cache. - did, err := client.ResolveUser(ctx, "test.handle") - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if did.String() != "did:plc:resolved" { - t.Errorf("Unexpected DID %q", did.String()) - } - if resolveRequests.Load() != 1 { - t.Errorf("expected 1 resolve request, got %d", resolveRequests.Load()) - } - - // Second resolution is served from cache. - if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { - t.Fatalf("Expected no error on cached resolution, got %v", err) - } - if resolveRequests.Load() != 1 { - t.Errorf("expected cached resolution to skip the network, got %d requests", resolveRequests.Load()) - } - - // An expired cache entry is refetched. - client.cacheMu.Lock() - entry := client.didCache["test.handle"] - entry.expiresAt = time.Now().Add(-time.Minute) - client.didCache["test.handle"] = entry - client.cacheMu.Unlock() - - if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { - t.Fatalf("Expected no error after expiry, got %v", err) - } - if resolveRequests.Load() != 2 { - t.Errorf("expected a fresh resolve after TTL expiry, got %d requests", resolveRequests.Load()) - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 1c5ab7f..afe01b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,7 +30,6 @@ type Config struct { OllamaHost string MigrateOnStart bool MigrationsDir string - ReadyzAddress string ReadyzPort string MetricsPort string X402FacilitatorURL string @@ -87,14 +86,13 @@ func Load() *Config { BlueskyAppPassword: getEnv("BLUESKY_APP_PASSWORD", ""), BlueskyHost: getEnv("BLUESKY_HOST", "https://bsky.social"), NostrRelays: splitComma(getEnv("NOSTR_RELAYS", "wss://relay.damus.io,wss://nos.lol")), - TwitterBearerToken: getEnvOptional("TWITTER_BEARER_TOKEN"), - PrewarmAuthors: splitComma(getEnvOptional("PREWARM_AUTHORS")), - PrewarmFeedURI: getEnvOptional("PREWARM_FEED_URI"), + TwitterBearerToken: getEnv("TWITTER_BEARER_TOKEN", ""), + PrewarmAuthors: splitComma(getEnv("PREWARM_AUTHORS", "")), + PrewarmFeedURI: getEnv("PREWARM_FEED_URI", ""), EnrichCrossPlatform: getEnvBool("ENRICH_CROSS_PLATFORM", "true"), OllamaHost: getEnv("OLLAMA_HOST", "http://localhost:11434"), MigrateOnStart: getEnvBool("MIGRATE_ON_START", "true"), MigrationsDir: getEnv("MIGRATIONS_DIR", "sql/schema"), - ReadyzAddress: getEnv("READYZ_ADDRESS", "127.0.0.1"), ReadyzPort: getEnv("READYZ_PORT", "9090"), MetricsPort: getEnv("METRICS_PORT", "9091"), X402FacilitatorURL: getEnv("X402_FACILITATOR_URL", "https://api.cdp.coinbase.com/platform/v2/x402"), @@ -114,12 +112,6 @@ func Load() *Config { AnalyticsTTLDrivers: getEnvDuration("ANALYTICS_TTL_DRIVERS", "1h"), } - for _, port := range []string{cfg.ReadyzPort, cfg.MetricsPort, cfg.ListenPort} { - if _, err := strconv.Atoi(port); err != nil { - log.Fatalf("Invalid port %q: must be a numeric port value", port) - } - } - // Default MatrixServerID to system hostname if not explicitly set if cfg.MatrixServerID == "" { hostname, err := os.Hostname() @@ -141,14 +133,6 @@ func getEnv(key, fallback string) string { return fallback } -func getEnvOptional(key string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - log.Warn("Environment variable is not set (optional)", "key", key) - return "" -} - func getEnvBool(key, fallback string) bool { v := getEnv(key, fallback) switch strings.ToLower(strings.TrimSpace(v)) { diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 952045d..5c22907 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/url" - "strings" "time" "fiatjaf.com/nostr" @@ -14,44 +12,15 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) -const ( - // relayQueryTimeout bounds connect + subscribe + collect against a single - // relay. The websocket dial, subscription handshake and event collection all - // share this budget. - relayQueryTimeout = 5 * time.Second - - // maxTotalRelayTime caps the fan-out across all relays when the caller's - // context carries no deadline of its own. Without it a long relay list could - // multiply the per-relay budget unboundedly. - maxTotalRelayTime = 20 * time.Second -) - type Client struct { relays []string } func NewNostrClient(relays []string) (*Client, error) { - seen := make(map[string]struct{}, len(relays)) - cleaned := make([]string, 0, len(relays)) - for _, r := range relays { - r = strings.TrimSpace(r) - if r == "" { - continue - } - u, err := url.Parse(r) - if err != nil || (u.Scheme != "ws" && u.Scheme != "wss") { - return nil, fmt.Errorf("invalid relay URL %q: must be a ws:// or wss:// endpoint", r) - } - if _, dup := seen[r]; dup { - continue - } - seen[r] = struct{}{} - cleaned = append(cleaned, r) - } - if len(cleaned) == 0 { + if len(relays) == 0 { return nil, fmt.Errorf("at least one relay is required") } - return &Client{relays: cleaned}, nil + return &Client{relays: relays}, nil } type Post struct { @@ -61,149 +30,190 @@ type Post struct { CreatedAt time.Time } -// queryBudget returns a context bounded to maxTotalRelayTime when ctx has no -// deadline, so the relay fan-out cannot run away across many relays. Callers -// must invoke the returned cancel. -func (c *Client) queryBudget(ctx context.Context) (context.Context, context.CancelFunc) { - if _, ok := ctx.Deadline(); ok { - return context.WithCancel(ctx) - } - return context.WithTimeout(ctx, maxTotalRelayTime) -} - -// collectFromRelay connects to url within ctx, subscribes with filter and -// appends matching events to posts (deduplicated by event id) until the limit is -// reached, EOSE arrives, or ctx is done. It reports whether the relay was -// actually queryable (connected + subscribed), letting callers distinguish a -// total relay outage from a legitimate empty result. -func (c *Client) collectFromRelay(ctx context.Context, url string, filter nostr.Filter, limit int, posts *[]*Post, seen map[string]struct{}) (used bool) { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) +func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { + pubkey, err := resolvePubkey(actor) if err != nil { - return false + return nil, err } - defer relay.Close() - sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) - if err != nil { - return false - } - defer sub.Unsub() - - for { - select { - case <-sub.EndOfStoredEvents: - return true - case ev, ok := <-sub.Events: - if !ok { - return true - } - id := ev.ID.Hex() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - *posts = append(*posts, &Post{ - ID: id, - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }) - if len(*posts) >= limit { - return true - } - case <-ctx.Done(): - return true - } - } -} - -// collectPosts subscribes to filter across every relay, stopping once limit posts -// are collected or the query context is done. It returns the deduplicated posts, -// or an error if no relay could be queried at all or the operation was aborted. -func (c *Client) collectPosts(ctx context.Context, filter nostr.Filter, limit int) ([]*Post, error) { posts := make([]*Post, 0, limit) seen := make(map[string]struct{}) - queryCtx, cancel := c.queryBudget(ctx) - defer cancel() + connected := false - var anyUsed bool for _, url := range c.relays { - if queryCtx.Err() != nil { - break + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + continue } - relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) - used := c.collectFromRelay(relayCtx, url, filter, limit, &posts, seen) - relayCancel() - anyUsed = anyUsed || used + connected = true + sub, err := relay.Subscribe(ctx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Limit: limit, + }, nostr.SubscriptionOptions{}) + if err != nil { + relay.Close() + continue + } + + func() { + defer sub.Unsub() + defer relay.Close() + + collect: + for { + select { + case <-sub.EndOfStoredEvents: + break collect + case ev, ok := <-sub.Events: + if !ok { + break collect + } + id := ev.ID.Hex() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + posts = append(posts, &Post{ + ID: id, + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }) + case <-ctx.Done(): + break collect + } + } + }() + if len(posts) >= limit { break } } - if err := queryCtx.Err(); err != nil { - return nil, fmt.Errorf("nostr query aborted: %w", err) - } - if !anyUsed { + if !connected { return nil, fmt.Errorf("could not connect to any relay") } return posts, nil } +func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { + pubkey, err := resolvePubkey(actor) + if err != nil { + return nil, err + } + + since := nostr.Timestamp(start.Unix()) + until := nostr.Timestamp(end.Unix()) + + posts := make([]*Post, 0, limit) + seen := make(map[string]struct{}) + + connected := false + + for _, url := range c.relays { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + continue + } + connected = true + sub, err := relay.Subscribe(ctx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Since: since, + Until: until, + Limit: limit, + }, nostr.SubscriptionOptions{}) + if err != nil { + relay.Close() + continue + } + + func() { + defer sub.Unsub() + defer relay.Close() + + collect: + for { + select { + case <-sub.EndOfStoredEvents: + break collect + case ev, ok := <-sub.Events: + if !ok { + break collect + } + id := ev.ID.Hex() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + posts = append(posts, &Post{ + ID: id, + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }) + case <-ctx.Done(): + break collect + } + } + }() + + if len(posts) >= limit { + break + } + } + + if !connected { + return nil, fmt.Errorf("could not connect to any relay") + } + + return posts, nil +} + func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { eid, err := resolveEventID(id) if err != nil { return nil, err } - filter := nostr.Filter{ - IDs: []nostr.ID{eid}, - Kinds: []nostr.Kind{1}, - } - posts, err := c.collectPosts(ctx, filter, 1) - if err != nil { - return nil, err - } - if len(posts) == 0 { - return nil, fmt.Errorf("post %q not found across configured relays", id) - } - return posts[0], nil -} + for _, url := range c.relays { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + continue + } + sub, err := relay.Subscribe(ctx, nostr.Filter{ + IDs: []nostr.ID{eid}, + Kinds: []nostr.Kind{1}, + }, nostr.SubscriptionOptions{}) + if err != nil { + relay.Close() + continue + } + defer sub.Unsub() + defer relay.Close() -func (c *Client) GetLatestNPosts(ctx context.Context, author string, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil + for { + select { + case <-sub.EndOfStoredEvents: + return nil, fmt.Errorf("post %q not found", id) + case ev, ok := <-sub.Events: + if !ok { + return nil, fmt.Errorf("post %q not found", id) + } + return &Post{ + ID: ev.ID.Hex(), + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } } - pubkey, err := resolvePubkey(author) - if err != nil { - return nil, err - } - - filter := nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Limit: limit, - } - return c.collectPosts(ctx, filter, limit) -} - -func (c *Client) GetPostsInTimeframe(ctx context.Context, author string, start, end time.Time, limit int) ([]*Post, error) { - if limit <= 0 { - return nil, nil - } - pubkey, err := resolvePubkey(author) - if err != nil { - return nil, err - } - - filter := nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Since: nostr.Timestamp(start.Unix()), - Until: nostr.Timestamp(end.Unix()), - Limit: limit, - } - return c.collectPosts(ctx, filter, limit) + return nil, fmt.Errorf("could not connect to any relay") } func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -213,128 +223,83 @@ func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, e } npub := nip19.EncodeNpub(pubkey) - filter := nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{0}, - Limit: 1, - } - - queryCtx, cancel := c.queryBudget(ctx) - defer cancel() - - var anyUsed bool - for _, url := range c.relays { - if queryCtx.Err() != nil { - break + for _, u := range c.relays { + relay, err := nostr.RelayConnect(ctx, u, nostr.RelayOptions{}) + if err != nil { + continue } - relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) - profile, used := c.actorFromRelay(relayCtx, url, filter, npub) - relayCancel() - anyUsed = anyUsed || used - if profile != nil { - return profile, nil + sub, err := relay.Subscribe(ctx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{0}, + Limit: 1, + }, nostr.SubscriptionOptions{}) + if err != nil { + relay.Close() + continue } - } + defer sub.Unsub() + defer relay.Close() - if err := queryCtx.Err(); err != nil { - return nil, fmt.Errorf("nostr query aborted: %w", err) - } - if !anyUsed { - return nil, fmt.Errorf("could not connect to any relay") + collect: + for { + select { + case <-sub.EndOfStoredEvents: + break collect + case ev, ok := <-sub.Events: + if !ok { + break collect + } + var meta struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + About string `json:"about"` + Picture string `json:"picture"` + Website string `json:"website"` + } + if err := json.Unmarshal([]byte(ev.Content), &meta); err == nil { + return &actor.Profile{ + Platform: "Nostr", + Handle: npub, + DisplayName: meta.DisplayName, + AvatarURL: meta.Picture, + Bio: meta.About, + URL: "https://njump.me/" + npub, + }, nil + } + break collect + case <-ctx.Done(): + return nil, ctx.Err() + } + } } return nil, fmt.Errorf("no profile metadata found for %q", author) } -// actorFromRelay looks up one kind-0 metadata event on url. It returns a -// non-nil profile when a usable metadata event was found. used reports whether -// the relay was queried at all (so a fully unreachable relay list is -// distinguishable from an author without metadata). -func (c *Client) actorFromRelay(ctx context.Context, url string, filter nostr.Filter, npub string) (profile *actor.Profile, used bool) { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - return nil, false - } - defer relay.Close() - - sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) - if err != nil { - return nil, false - } - defer sub.Unsub() - - select { - case <-sub.EndOfStoredEvents: - return nil, true - case ev, ok := <-sub.Events: - if !ok { - return nil, true - } - return profileFromEvent(ev, npub), true - case <-ctx.Done(): - return nil, true - } -} - -// profileFromEvent builds an actor.Profile from a kind-0 metadata event, or nil -// when the content is not valid JSON or carries no usable fields. -func profileFromEvent(ev nostr.Event, npub string) *actor.Profile { - var meta struct { - Name string `json:"name"` - DisplayName string `json:"display_name"` - About string `json:"about"` - Picture string `json:"picture"` - Website string `json:"website"` - } - if err := json.Unmarshal([]byte(ev.Content), &meta); err != nil { - return nil - } - if meta.Name == "" && meta.DisplayName == "" && meta.About == "" && meta.Picture == "" { - return nil - } - displayName := meta.DisplayName - if displayName == "" { - displayName = meta.Name - } - return &actor.Profile{ - Platform: "Nostr", - Handle: npub, - DisplayName: displayName, - AvatarURL: meta.Picture, - Bio: meta.About, - URL: "https://njump.me/" + npub, - } -} - -func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { +func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { + // Nostr has no global actor search; best-effort: if the query resolves to a + // pubkey/npub, return that single profile. if _, err := resolvePubkey(query); err != nil { - // Not a pubkey we can resolve (e.g. a name query). Treat it as no match - // rather than an error so a cross-platform search isn't failed by a - // non-nostr query. - return nil, nil + return []actor.Profile{}, nil } prof, err := c.GetActor(ctx, query) if err != nil { - return nil, err + return []actor.Profile{}, nil } - return []*actor.Profile{prof}, nil + return []actor.Profile{*prof}, nil } func resolvePubkey(actor string) (nostr.PubKey, error) { prefix, value, err := nip19.Decode(actor) if err == nil { - switch prefix { - case "npub": - if pk, ok := value.(nostr.PubKey); ok { - return pk, nil - } - case "nprofile": - if pp, ok := value.(nostr.ProfilePointer); ok { - return pp.PublicKey, nil - } + if prefix != "npub" { + return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) } - return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) + pk, ok := value.(nostr.PubKey) + if !ok { + return nostr.PubKey{}, fmt.Errorf("unexpected decoded value type %T", value) + } + return pk, nil } - if len(actor) == 64 { pk, e := nostr.PubKeyFromHex(actor) if e != nil { @@ -348,19 +313,15 @@ func resolvePubkey(actor string) (nostr.PubKey, error) { func resolveEventID(id string) (nostr.ID, error) { prefix, value, err := nip19.Decode(id) if err == nil { - switch prefix { - case "note": - if eid, ok := value.(nostr.ID); ok { - return eid, nil - } - case "nevent": - if ep, ok := value.(nostr.EventPointer); ok { - return ep.ID, nil - } + if prefix != "note" { + return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) } - return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) + eid, ok := value.(nostr.ID) + if !ok { + return nostr.ID{}, fmt.Errorf("unexpected decoded value type %T", value) + } + return eid, nil } - if len(id) == 64 { eid, e := nostr.IDFromHex(id) if e != nil { diff --git a/internal/nostr/client_test.go b/internal/nostr/client_test.go index e65bbb1..0980361 100644 --- a/internal/nostr/client_test.go +++ b/internal/nostr/client_test.go @@ -189,80 +189,6 @@ func TestRelayConnectionFailureReturnsError(t *testing.T) { } } -func TestNewNostrClientInvalidScheme(t *testing.T) { - _, err := NewNostrClient([]string{"http://relay.example"}) - if err == nil { - t.Fatal("expected error for non-websocket relay URL") - } - if !strings.Contains(err.Error(), "invalid relay URL") { - t.Fatalf("expected 'invalid relay URL' error, got %v", err) - } - - // Blank and duplicate entries are dropped, the scheme is validated. - c, err := NewNostrClient([]string{ - "wss://relay.a", - " wss://relay.a ", - "ws://relay.b", - " ", - }) - if err != nil { - t.Fatal(err) - } - if len(c.relays) != 2 { - t.Fatalf("expected 2 unique relays after cleaning, got %d: %v", len(c.relays), c.relays) - } -} - -func TestGetLatestNPostsZeroOrNegativeLimit(t *testing.T) { - c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) - if err != nil { - t.Fatal(err) - } - - sk := nostr.Generate() - npub := nip19.EncodeNpub(sk.Public()) - - if posts, err := c.GetLatestNPosts(context.Background(), npub, 0); err != nil || posts != nil { - t.Fatalf("expected nil, nil for zero limit, got %v, %v", posts, err) - } - if posts, err := c.GetLatestNPosts(context.Background(), npub, -5); err != nil || posts != nil { - t.Fatalf("expected nil, nil for negative limit, got %v, %v", posts, err) - } - - start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - end := time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC) - if posts, err := c.GetPostsInTimeframe(context.Background(), npub, start, end, 0); err != nil || posts != nil { - t.Fatalf("expected nil, nil for zero timeframe limit, got %v, %v", posts, err) - } -} - -func TestSearchActors(t *testing.T) { - c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) - if err != nil { - t.Fatal(err) - } - - ctx := context.Background() - - // A non-key query (e.g. a display name) is a valid no-match, not an error. - results, err := c.SearchActors(ctx, "alice", 1) - if err != nil { - t.Fatalf("expected no error for non-key query, got %v", err) - } - if results != nil { - t.Fatalf("expected nil results for non-key query, got %v", results) - } - - // A valid pubkey that fails on every relay must surface the outage. - sk := nostr.Generate() - npub := nip19.EncodeNpub(sk.Public()) - if _, err := c.SearchActors(ctx, npub, 1); err == nil { - t.Fatal("expected error when no relay is reachable") - } else if !strings.Contains(err.Error(), "could not connect to any relay") { - t.Fatalf("expected 'could not connect to any relay' error, got %v", err) - } -} - func TestGetPostWithHexID(t *testing.T) { c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) if err != nil { @@ -297,9 +223,9 @@ func TestGetActorWithHexPubkey(t *testing.T) { if err == nil { t.Fatal("expected error when relay unavailable") } - // GetActor reports a total relay outage as a connection failure rather than - // pretending the author has no metadata. - if !strings.Contains(err.Error(), "could not connect to any relay") { - t.Fatalf("expected 'could not connect to any relay' error, got %v", err) + // GetActor surfaces the failure as an absence of a profile after the relay + // loop runs without connecting. + if !strings.Contains(err.Error(), "no profile metadata found") { + t.Fatalf("expected 'no profile metadata found' error, got %v", err) } } diff --git a/internal/ollama/client.go b/internal/ollama/client.go index 60515ae..69df20c 100644 --- a/internal/ollama/client.go +++ b/internal/ollama/client.go @@ -2,150 +2,52 @@ package ollama import ( "context" - "errors" - "fmt" "net/http" "net/url" "strings" - "sync" - "time" "github.com/nacorid/naco-api/internal/clustering" "github.com/ollama/ollama/api" ) -const ( - // embedConcurrency caps how many embedding requests run in parallel. Ollama - // serializes model requests server-side (OLLAMA_NUM_PARALLEL defaults to 1), - // so unbounded fan-out would only add latency and memory for no throughput. - embedConcurrency = 4 - - // embedRetries and embedRetryBackoff cover transient server-side embedding - // failures (e.g. cold model load, transient 5xx/429). - embedRetries = 2 - embedRetryBackoff = 150 * time.Millisecond - - // generateRetries and generateRetryBackoff cover transient failures that - // produce no output at all; a truncated stream is never retried because the - // partial response cannot be safely replayed. - generateRetries = 1 - generateRetryBackoff = 500 * time.Millisecond -) - type OllamaClient struct { client api.Client embeddingModel string styleModel string } -func New(host *url.URL, httpClient *http.Client, embeddingModel, styleModel string) *OllamaClient { - client := api.NewClient(host, httpClient) - return &OllamaClient{client: *client, embeddingModel: embeddingModel, styleModel: styleModel} +func New(ctx context.Context, host *url.URL, http *http.Client, em, sm string) *OllamaClient { + client := api.NewClient(host, http) + return &OllamaClient{client: *client, embeddingModel: em, styleModel: sm} } -// EmbedTexts computes one embedding per post. Embeddings are returned in the -// same order as posts; callers rely on index correspondence (e.g. -// items[i].Embedding = embeddings[i]) so posts are never skipped. Requests run -// in a bounded worker pool and transient failures are retried with backoff. func (o *OllamaClient) EmbedTexts(ctx context.Context, posts []clustering.Item) ([][]float64, error) { if len(posts) == 0 { return nil, nil } + embeddings := make([][]float64, 0, len(posts)) - embeddings := make([][]float64, len(posts)) - - // poolCtx is a child of ctx: cancelling it on the first failure must not - // cancel the caller's request context. - poolCtx, cancel := context.WithCancel(ctx) - defer cancel() - - var ( - mu sync.Mutex - firstErr error - sema = make(chan struct{}, embedConcurrency) - wg sync.WaitGroup - ) - for i, post := range posts { - wg.Add(1) - go func(idx int, text string) { - defer wg.Done() - select { - case sema <- struct{}{}: - case <-poolCtx.Done(): - return - } - defer func() { <-sema }() - - emb, err := o.embedWithRetry(poolCtx, strings.TrimSpace(text)) - if err != nil { - mu.Lock() - if firstErr == nil { - firstErr = fmt.Errorf("embedding post %d (%q): %w", idx, text, err) - } - mu.Unlock() - cancel() - return - } - embeddings[idx] = emb - }(i, post.Text) - } - wg.Wait() - - if firstErr := atomicErr(&mu, &firstErr); firstErr != nil { - return embeddings, firstErr - } - // Every worker may have exited via the poolCtx.Done() branch without - // recording an error, e.g. when the caller's context is already canceled. - if err := ctx.Err(); err != nil { - return nil, err - } - return embeddings, nil -} - -func atomicErr(mu *sync.Mutex, err *error) error { - mu.Lock() - defer mu.Unlock() - return *err -} - -func (o *OllamaClient) embedWithRetry(ctx context.Context, text string) ([]float64, error) { - backoff := embedRetryBackoff - var lastErr error - for attempt := 0; attempt <= embedRetries; attempt++ { - if attempt > 0 { - if err := sleepCtx(ctx, backoff); err != nil { - return nil, err - } - backoff *= 2 - } + for _, post := range posts { resp, err := o.client.Embeddings(ctx, &api.EmbeddingRequest{ Model: o.embeddingModel, - Prompt: text, + Prompt: post.Text, }) - if err == nil { - return resp.Embedding, nil - } - lastErr = err - if !isTransientErr(err) { + if err != nil { return nil, err } + embeddings = append(embeddings, resp.Embedding) } - return nil, lastErr + + return embeddings, nil } func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.Item) (string, error) { if len(posts) == 0 { return "", nil } - - promptParts := make([]string, 0, len(posts)) + var promptParts []string for _, post := range posts { - if text := strings.TrimSpace(post.Text); text != "" { - promptParts = append(promptParts, text) - } - } - if len(promptParts) == 0 { - return "", fmt.Errorf("no post text to summarize") + promptParts = append(promptParts, post.Text) } prompt := "Summarize the user's writing style. Analyze:\n" + @@ -160,74 +62,16 @@ func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.It "\n\nStyle summary:" var fullResponse strings.Builder - backoff := generateRetryBackoff - var lastErr error - for attempt := 0; attempt <= generateRetries; attempt++ { - if attempt > 0 { - if err := sleepCtx(ctx, backoff); err != nil { - return "", err - } - } - - // Reset between attempts so a failed run never leaks a partial summary. - fullResponse.Reset() - done := false - doneReason := "" - err := o.client.Generate(ctx, &api.GenerateRequest{ - Model: o.styleModel, - Prompt: prompt, - }, func(resp api.GenerateResponse) error { - fullResponse.WriteString(resp.Response) - if resp.Done { - done = true - doneReason = resp.DoneReason - } - return nil - }) - if err != nil { - lastErr = err - // Only retry transport/5xx failures that produced no output; a - // partial stream must not be replayed. - if !isTransientErr(err) || fullResponse.Len() > 0 { - return "", fmt.Errorf("ollama generate with model %q failed: %w", o.styleModel, err) - } - continue - } - - // The underlying stream reader can finish without an error when the - // request context is canceled or the connection drops mid-stream (the - // library never inspects bufio.Scanner.Err), so a bare nil from - // Generate does not mean the response is complete. - if ctxErr := ctx.Err(); ctxErr != nil { - return "", ctxErr - } - if !done { - return "", fmt.Errorf("ollama stream for model %q ended before completion", o.styleModel) - } - if doneReason == "length" { - return "", fmt.Errorf("ollama response for model %q was truncated (context limit reached)", o.styleModel) - } - return fullResponse.String(), nil - } - return "", fmt.Errorf("ollama generate with model %q failed after retries: %w", o.styleModel, lastErr) -} - -func sleepCtx(ctx context.Context, d time.Duration) error { - select { - case <-time.After(d): + err := o.client.Generate(ctx, &api.GenerateRequest{ + Model: o.styleModel, + Prompt: prompt, + }, func(resp api.GenerateResponse) error { + fullResponse.WriteString(resp.Response) return nil - case <-ctx.Done(): - return ctx.Err() + }) + if err != nil { + return "", err } -} -// isTransientErr reports whether err is a retryable server-side failure. Non-2xx -// HTTP errors that carry a StatusCode (429, 5xx) qualify; context and transport -// errors below that never do, so they surface immediately. -func isTransientErr(err error) bool { - var se api.StatusError - if errors.As(err, &se) { - return se.StatusCode == http.StatusTooManyRequests || se.StatusCode >= http.StatusInternalServerError - } - return false + return fullResponse.String(), nil } diff --git a/internal/ollama/client_test.go b/internal/ollama/client_test.go index 9ade0e3..16b18f3 100644 --- a/internal/ollama/client_test.go +++ b/internal/ollama/client_test.go @@ -2,89 +2,27 @@ package ollama import ( "context" - "encoding/json" - "fmt" "net/http" "net/http/httptest" "net/url" - "strings" - "sync" - "sync/atomic" "testing" - "time" "github.com/nacorid/naco-api/internal/clustering" ) -// setupTestServer creates a test server and an initialized client. -func setupTestServer(t *testing.T, handler http.Handler) (*httptest.Server, *OllamaClient) { - t.Helper() - server := httptest.NewServer(handler) - - host, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("failed to parse test server URL: %v", err) - } - - client := New(host, http.DefaultClient, "test-embed", "test-style") - return server, client -} - -// ============================================================================ -// EmbedTexts Tests -// ============================================================================ - func TestEmbedTexts(t *testing.T) { - type embedRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - } - - var mu sync.Mutex - var receivedRequests []embedRequest - mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - // 1. Validate HTTP Method - if r.Method != http.MethodPost { - t.Errorf("expected method POST, got %s", r.Method) - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - // 2. Validate Headers - if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { - t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) - http.Error(w, "invalid content-type", http.StatusBadRequest) - return - } - - // 3. Validate Request Payload - var req embedRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("failed to decode request body: %v", err) - http.Error(w, "bad request", http.StatusBadRequest) - return - } - - if req.Model != "test-embed" { - t.Errorf("expected model 'test-embed', got %q", req.Model) - } - - mu.Lock() - receivedRequests = append(receivedRequests, req) - mu.Unlock() - - // 4. Return valid response w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "embedding": []float64{0.1, 0.2, 0.3}, - }) + w.Write([]byte(`{"embedding":[0.1,0.2,0.3]}`)) }) - server, client := setupTestServer(t, mux) + server := httptest.NewServer(mux) defer server.Close() + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + posts := []clustering.Item{ {ID: "1", Text: "hello", Embedding: nil}, {ID: "2", Text: "world", Embedding: nil}, @@ -95,264 +33,66 @@ func TestEmbedTexts(t *testing.T) { t.Fatalf("EmbedTexts failed: %v", err) } - // Verify outgoing requests: ensure both prompts were sent - mu.Lock() - if len(receivedRequests) != 2 { - t.Errorf("expected 2 HTTP requests, got %d", len(receivedRequests)) - } - promptsReceived := map[string]bool{} - for _, req := range receivedRequests { - promptsReceived[req.Prompt] = true - } - mu.Unlock() - - if !promptsReceived["hello"] || !promptsReceived["world"] { - t.Errorf("expected prompts 'hello' and 'world' to be sent, got: %+v", receivedRequests) - } - - // Verify parsed results if len(embeddings) != 2 { t.Fatalf("expected 2 embeddings, got %d", len(embeddings)) } + for i, emb := range embeddings { if len(emb) != 3 { - t.Errorf("embedding %d has wrong dimension: %d", i, len(emb)) + t.Errorf("embedding %d has wrong dim: %d", i, len(emb)) } if emb[0] != 0.1 || emb[1] != 0.2 || emb[2] != 0.3 { - t.Errorf("embedding %d has wrong values: %v", i, emb) + t.Errorf("embedding %d wrong values: %v", i, emb) } } } -func TestEmbedTexts_EmptyInput(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - t.Error("embeddings endpoint should not be called when input is empty") - http.Error(w, "should not be called", http.StatusInternalServerError) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - embeddings, err := client.EmbedTexts(context.Background(), []clustering.Item{}) - if err != nil { - t.Fatalf("EmbedTexts with empty input failed: %v", err) - } - - if len(embeddings) != 0 { - t.Errorf("expected 0 embeddings, got %d", len(embeddings)) - } -} - func TestEmbedTexts_Error(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error": "model test-embed not found"}`)) + w.Write([]byte("server error")) }) - server, client := setupTestServer(t, mux) + server := httptest.NewServer(mux) defer server.Close() - posts := []clustering.Item{{ID: "1", Text: "hello"}} + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + + posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} _, err := client.EmbedTexts(context.Background(), posts) if err == nil { - t.Fatal("expected error from failing server, got nil") + t.Fatal("expected error from failing server") } } -func TestEmbedTexts_ContextCanceled(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - // Hang until client cancels the request - <-r.Context().Done() - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - _, err := client.EmbedTexts(ctx, posts) - if err == nil { - t.Fatal("expected error due to canceled context, got nil") - } -} - -func TestEmbedTexts_RetryThenSuccess(t *testing.T) { - var requests atomic.Int32 - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - if requests.Add(1) == 1 { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) - return - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "embedding": []float64{9.9}, - }) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - embeddings, err := client.EmbedTexts(context.Background(), posts) - if err != nil { - t.Fatalf("EmbedTexts failed after transient error: %v", err) - } - if len(embeddings) != 1 || len(embeddings[0]) != 1 || embeddings[0][0] != 9.9 { - t.Fatalf("unexpected embeddings after retry: %v", embeddings) - } - if got := requests.Load(); got != 2 { - t.Errorf("expected 2 requests (1 failure + 1 retry), got %d", got) - } -} - -func TestEmbedTexts_RetryExhausted(t *testing.T) { - var requests atomic.Int32 - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - _, err := client.EmbedTexts(context.Background(), posts) - if err == nil { - t.Fatal("expected error after retries exhausted, got nil") - } - if got := requests.Load(); got != embedRetries+1 { - t.Errorf("expected %d attempts (initial + %d retries), got %d", embedRetries+1, embedRetries, got) - } -} - -func TestEmbedTexts_ParallelOrder(t *testing.T) { - // Shake ordering: handler sleeps and varies the embedding by prompt length, - // so a naive (unordered) worker pool would misalign results. - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - var req struct { - Prompt string `json:"prompt"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("failed to decode request body: %v", err) - http.Error(w, "bad request", http.StatusBadRequest) - return - } - time.Sleep(30 * time.Millisecond) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "embedding": []float64{float64(len(req.Prompt))}, - }) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := make([]clustering.Item, 8) - want := make([]float64, 8) - for i := range posts { - posts[i] = clustering.Item{ID: fmt.Sprintf("%d", i), Text: strings.Repeat("x", i+1)} - want[i] = float64(i + 1) - } - - embeddings, err := client.EmbedTexts(context.Background(), posts) - if err != nil { - t.Fatalf("EmbedTexts failed: %v", err) - } - if len(embeddings) != len(posts) { - t.Fatalf("expected %d embeddings, got %d", len(posts), len(embeddings)) - } - for i, emb := range embeddings { - if len(emb) != 1 || emb[0] != want[i] { - t.Errorf("embedding %d out of order or wrong value: got %v, want %v", i, emb, want[i]) - } - } -} - -// ============================================================================ -// SummarizeStyle Tests -// ============================================================================ - func TestSummarizeStyle(t *testing.T) { - type generateRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - } - - var reqBody generateRequest - callCount := 0 - mux := http.NewServeMux() mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - callCount++ - - // 1. Validate HTTP Method - if r.Method != http.MethodPost { - t.Errorf("expected method POST, got %s", r.Method) - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return - } - - // 2. Validate Headers - if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { - t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) - http.Error(w, "invalid content-type", http.StatusBadRequest) - return - } - - // 3. Validate Request Payload - if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { - t.Errorf("failed to decode request body: %v", err) - http.Error(w, "bad request", http.StatusBadRequest) - return - } - - if reqBody.Model != "test-style" { - t.Errorf("expected model 'test-style', got %q", reqBody.Model) - } - - // Ensure the prompt includes input text - if !strings.Contains(reqBody.Prompt, "First post") || !strings.Contains(reqBody.Prompt, "Second post") { - t.Errorf("prompt missing post contents: %q", reqBody.Prompt) - } - - // 4. Stream NDJSON Response flusher, ok := w.(http.Flusher) if !ok { - t.Error("ResponseWriter is not a Flusher") - return + t.Fatal("response writer doesn't support flushing") } - w.Header().Set("Content-Type", "application/x-ndjson") - - chunks := []string{ - `{"response":"Style summary: "}` + "\n", - `{"response":"concise and "}` + "\n", - `{"response":"clear.","done":true}` + "\n", - } - - for _, chunk := range chunks { - _, _ = w.Write([]byte(chunk)) + // Stream response as newline-delimited JSON + w.Write([]byte(`{"response":"Style summary: "}` + "\n")) + if flusher != nil { flusher.Flush() } + w.Write([]byte(`{"response":"concise and clear."}` + "\n")) }) - server, client := setupTestServer(t, mux) + server := httptest.NewServer(mux) defer server.Close() + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + posts := []clustering.Item{ - {ID: "1", Text: "First post"}, - {ID: "2", Text: "Second post"}, + {ID: "1", Text: "First post", Embedding: nil}, + {ID: "2", Text: "Second post", Embedding: nil}, } summary, err := client.SummarizeStyle(context.Background(), posts) @@ -360,26 +100,110 @@ func TestSummarizeStyle(t *testing.T) { t.Fatalf("SummarizeStyle failed: %v", err) } - expected := "Style summary: concise and clear." - if summary != expected { - t.Errorf("got summary %q, want %q", summary, expected) + if summary != "Style summary: concise and clear." { + t.Errorf("unexpected summary: %q", summary) + } +} + +func TestSummarizeStyle_Error(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("server error")) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + + posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error from failing server") + } +} + +func TestSummarizeStyle_Streaming(t *testing.T) { + callCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + callCount++ + flusher, ok := w.(http.Flusher) + if !ok { + t.Fatal("response writer doesn't support flushing") + } + w.Header().Set("Content-Type", "application/x-ndjson") + // Simulate streaming response + w.Write([]byte(`{"response":"part1"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + w.Write([]byte(`{"response":"part2"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + w.Write([]byte(`{"response":"part3","done":true}` + "\n")) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + + posts := []clustering.Item{{ID: "1", Text: "test", Embedding: nil}} + + summary, err := client.SummarizeStyle(context.Background(), posts) + if err != nil { + t.Fatalf("SummarizeStyle failed: %v", err) + } + + if summary != "part1part2part3" { + t.Errorf("streaming summary = %q, want %q", summary, "part1part2part3") } if callCount != 1 { - t.Errorf("expected 1 call to /api/generate, got %d", callCount) + t.Errorf("generate endpoint called %d times, want 1", callCount) + } +} + +func TestEmbedTexts_EmptyInput(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + t.Fatal("embeddings endpoint should not be called for empty input") + }) + + server := httptest.NewServer(mux) + defer server.Close() + + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + + embeddings, err := client.EmbedTexts(context.Background(), []clustering.Item{}) + if err != nil { + t.Fatalf("EmbedTexts with empty input failed: %v", err) + } + + if len(embeddings) != 0 { + t.Errorf("expected empty embeddings, got %d", len(embeddings)) } } func TestSummarizeStyle_EmptyInput(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - t.Error("generate endpoint should not be called when input is empty") - http.Error(w, "should not be called", http.StatusInternalServerError) + t.Fatal("generate endpoint should not be called for empty input") }) - server, client := setupTestServer(t, mux) + server := httptest.NewServer(mux) defer server.Close() + host, _ := url.Parse(server.URL) + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + summary, err := client.SummarizeStyle(context.Background(), []clustering.Item{}) if err != nil { t.Fatalf("SummarizeStyle with empty input failed: %v", err) @@ -389,128 +213,3 @@ func TestSummarizeStyle_EmptyInput(t *testing.T) { t.Errorf("expected empty summary, got %q", summary) } } - -func TestSummarizeStyle_Error(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error":"internal model failure"}`)) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error from failing server, got nil") - } -} - -func TestSummarizeStyle_MalformedJSONStream(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, _ := w.(http.Flusher) - w.Header().Set("Content-Type", "application/x-ndjson") - - // First line is valid JSON, second is corrupt - _, _ = w.Write([]byte(`{"response":"valid part"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - _, _ = w.Write([]byte(`{corrupt_json` + "\n")) - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error on malformed JSON stream, got nil") - } -} - -func TestSummarizeStyle_ContextCanceled(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, _ := w.(http.Flusher) - w.Header().Set("Content-Type", "application/x-ndjson") - - // Send initial chunk - _, _ = w.Write([]byte(`{"response":"waiting..."}` + "\n")) - if flusher != nil { - flusher.Flush() - } - - // Wait until client context cancels - <-r.Context().Done() - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - - _, err := client.SummarizeStyle(ctx, posts) - if err == nil { - t.Fatal("expected error on canceled context, got nil") - } -} - -func TestSummarizeStyle_StreamNeverCompletes(t *testing.T) { - // Streaming ends without a done:true chunk (simulates a dropped connection - // or the http.Client timeout firing mid-generation). - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, _ := w.(http.Flusher) - w.Header().Set("Content-Type", "application/x-ndjson") - _, _ = w.Write([]byte(`{"response":"partial answer"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - }) // handler returns without ever writing done:true - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error when stream ends without completion, got nil") - } - if !strings.Contains(err.Error(), "ended before completion") { - t.Errorf("expected 'ended before completion' error, got %v", err) - } -} - -func TestSummarizeStyle_LengthTruncated(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, _ := w.(http.Flusher) - w.Header().Set("Content-Type", "application/x-ndjson") - _, _ = w.Write([]byte(`{"response":"summary...","done":true,"done_reason":"length"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - }) - - server, client := setupTestServer(t, mux) - defer server.Close() - - posts := []clustering.Item{{ID: "1", Text: "hello"}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error when response is truncated at context limit, got nil") - } - if !strings.Contains(err.Error(), "truncated") { - t.Errorf("expected 'truncated' error, got %v", err) - } -} diff --git a/internal/server/actors.go b/internal/server/actors.go index 205c8fa..724285c 100644 --- a/internal/server/actors.go +++ b/internal/server/actors.go @@ -7,16 +7,16 @@ import ( "sync" ) -func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]*ActorProfile, error) { +func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]ActorProfile, error) { var ( mu sync.Mutex - results []*ActorProfile + results []ActorProfile wg sync.WaitGroup errMu sync.Mutex errs []error success int ) - call := func(fn func(context.Context, string, int) ([]*ActorProfile, error)) { + call := func(fn func(context.Context, string, int) ([]ActorProfile, error)) { defer wg.Done() r, err := fn(ctx, query, limit) if err != nil { @@ -44,9 +44,9 @@ func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ( // dedupeActors removes near-duplicate profiles across platforms. Two results are considered // the same person when they share a normalized handle (the '@' and any instance/host stripped, // lower-cased) or a normalized display name. The first occurrence wins. -func dedupeActors(in []*ActorProfile) []*ActorProfile { +func dedupeActors(in []ActorProfile) []ActorProfile { seen := make(map[string]struct{}, len(in)) - out := make([]*ActorProfile, 0, len(in)) + out := make([]ActorProfile, 0, len(in)) for _, p := range in { key := normalizeHandle(p.Handle) if key == "" { diff --git a/internal/server/analytics.go b/internal/server/analytics.go index cb38bcf..ea0faf7 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -152,19 +152,14 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to return nil, fmt.Errorf("invalid time range: %w", err) } - var platforms []string + platforms := []string{} if args.Platform != "" { platforms = []string{args.Platform} } else { - // Twitter is included by default only when the API is enabled: the API - // costs it incurs are passed on to the caller through the upto pricing - // of these tools (see registerTools), so aggregating over it does not - // run the server at a loss. When no bearer token is configured there is - // no Twitter to aggregate over. - platforms = []string{"Bluesky", "ActivityPub", "Nostr"} - if s.twitterEnabled { - platforms = append(platforms, "Twitter") - } + // Twitter is included by default - the API costs it incurs are passed + // on to the caller through the paid tool pricing, so aggregating over + // it does not run the server at a loss. + platforms = []string{"Twitter", "Bluesky", "ActivityPub", "Nostr"} } timeRangeHash := hashTimeRange(start, end, limit) @@ -180,13 +175,12 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if cached, ok := s.getAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash); ok { var cachedResult json.RawMessage - _ = json.Unmarshal(cached, &cachedResult) + json.Unmarshal(cached, &cachedResult) return s.toToolResult(cachedResult) } // Fetch posts from all platforms var allPosts []clustering.Item - twitterPosts := 0 for _, p := range platforms { plat, ok := platformFromString(p) if !ok { @@ -198,9 +192,6 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to continue } allPosts = append(allPosts, posts...) - if plat == Twitter { - twitterPosts += len(posts) - } } if len(allPosts) == 0 { @@ -212,40 +203,14 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if err != nil { return nil, err } - result = setAnalyticsTwitterPostCount(result, twitterPosts) // Cache the result under the matching scope key. ttl := s.getCacheTTL(toolName) - _ = s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) + s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) return s.toToolResult(result) } -// setAnalyticsTwitterPostCount records how many posts in an analytics result -// were sourced from the Twitter API, so upto-scheme settlement can pass the -// realized Twitter costs on to the caller. Unknown result types are returned -// unchanged. -func setAnalyticsTwitterPostCount(result any, count int) any { - switch v := result.(type) { - case EngagementMetricsResponse: - v.TwitterPostCount = count - return v - case PostingPatternsResponse: - v.TwitterPostCount = count - return v - case TopicModelingResponse: - v.TwitterPostCount = count - return v - case SentimentTrendsResponse: - v.TwitterPostCount = count - return v - case EngagementDriversResponse: - v.TwitterPostCount = count - return v - } - return result -} - func (s *Server) fetchAnalyticsPosts(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]clustering.Item, error) { posts, err := s.fetchPostsInTimeForAnalytics(ctx, platform, author, start, end, limit) if err != nil { @@ -633,16 +598,20 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T return []TopicCluster{}, map[string]string{}, nil } + // Use embeddings from ollama for clustering items := posts + // We need embeddings for clustering - compute them embeddings, err := s.ollama.EmbedTexts(ctx, items) if err != nil { return nil, nil, fmt.Errorf("failed to compute embeddings: %w", err) } + // Update items with embeddings for i := range items { items[i].Embedding = embeddings[i] } + // Determine optimal number of topics topicCount := maxTopicCount if len(items) < maxTopicCount { topicCount = len(items) @@ -651,6 +620,7 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T topicCount = minTopicCount } + // Run KMeans results, err := clustering.KMeans(items, topicCount, 50, 42) if err != nil { return nil, nil, fmt.Errorf("kmeans failed: %w", err) @@ -664,6 +634,7 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T continue } + // Collect sample texts for labeling var samples []string for _, item := range cluster.Items { if len(samples) >= 3 { @@ -811,7 +782,8 @@ func (s *Server) computeSentimentTrends(ctx context.Context, args AnalyticsReque } func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsRequest, posts []clustering.Item) (any, error) { - // Topic modeling first, getting post->topic membership for correlation. + // Topic modeling first, getting post->topic membership so we can correlate + // each topic with the engagement of its member posts. topics, membership, err := s.clusterPosts(ctx, posts) if err != nil { return nil, err @@ -829,7 +801,8 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq topicEngagement[label] = append(topicEngagement[label], total) } - // Baseline is the mean engagement across all posts, for comparison. + // Baseline is the mean engagement across all posts, used as the comparison + // point for relevance ranking. var allEngagement []float64 for _, post := range posts { allEngagement = append(allEngagement, float64(post.Likes+post.Reposts+post.Replies+post.Quotes)) @@ -840,6 +813,9 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq for _, topic := range topics { values := topicEngagement[topic.Label] topicMean := mean(values) + // Correlation is the normalized difference between the topic's mean + // engagement and the overall baseline: positive when the topic draws + // above-average engagement, negative when below. Bounded to [-1, 1]. correlation := 0.0 if denom := topicMean + baseline; denom > 0 { correlation = (topicMean - baseline) / denom @@ -935,7 +911,7 @@ Texts: Neutral float64 `json:"neutral"` Emotions map[string]int `json:"emotions"` } - _ = json.Unmarshal([]byte(resp), &result) + json.Unmarshal([]byte(resp), &result) // Normalize sum := result.Positive + result.Negative + result.Neutral diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index e74eef6..d96a439 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -9,7 +9,6 @@ import ( "github.com/nacorid/naco-api/internal/activitypub" "github.com/nacorid/naco-api/internal/clustering" - "github.com/nacorid/naco-api/internal/twitter" ) var errSome = fmt.Errorf("some error") @@ -17,8 +16,8 @@ var errSome = fmt.Errorf("some error") func analyticsAPPosts() *fakeAP { base := time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC) return &fakeAP{posts: []*activitypub.Post{ - {ID: "p1", Author: "@a@mastodon.social", Content: "first post", CreatedAt: base}, - {ID: "p2", Author: "@a@mastodon.social", Content: "second post", CreatedAt: base.Add(time.Hour)}, + {ID: "p1", Author: "@a@mastodon.social", Content: "first post", Text: "first post", CreatedAt: base}, + {ID: "p2", Author: "@a@mastodon.social", Content: "second post", Text: "second post", CreatedAt: base.Add(time.Hour)}, }} } @@ -142,54 +141,6 @@ func TestAnalyticsPartialPlatformFailureTolerated(t *testing.T) { } } -// TestAnalyticsTwitterPostCount verifies the response reports how many posts -// were sourced from the Twitter API, so upto-scheme settlement can charge the -// realized Twitter API costs. -func TestAnalyticsTwitterPostCount(t *testing.T) { - tw := &fakeTwitter{posts: []*twitter.Post{ - {ID: "t1", Content: "first tweet", CreatedAt: time.Date(2024, 1, 11, 12, 0, 0, 0, time.UTC)}, - {ID: "t2", Content: "second tweet", CreatedAt: time.Date(2024, 1, 12, 12, 0, 0, 0, time.UTC)}, - {ID: "t3", Content: "third tweet", CreatedAt: time.Date(2024, 1, 13, 12, 0, 0, 0, time.UTC)}, - }} - st := newFakeStore() - s := newServerWith(&fakeBluesky{}, analyticsAPPosts(), &fakeNostr{}, tw, st, &fakeOllama{}) - - res, err := callTool(s, "getEngagementMetrics", AnalyticsRequest{ - Author: "@alice", - StartTime: "2024-01-01T00:00:00Z", - EndTime: "2024-12-31T00:00:00Z", - Platform: "Twitter", - }) - if err != nil { - t.Fatalf("getEngagementMetrics error: %v", err) - } - var out EngagementMetricsResponse - if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if out.TwitterPostCount != 3 { - t.Fatalf("expected TwitterPostCount 3, got %d", out.TwitterPostCount) - } - - // A non-Twitter platform reports 0 and the default aggregate excludes - // Twitter when the API is not enabled (twitterEnabled=false in tests). - res, err = callTool(s, "getEngagementMetrics", AnalyticsRequest{ - Author: "@a@mastodon.social", - StartTime: "2024-01-01T00:00:00Z", - EndTime: "2024-12-31T00:00:00Z", - Platform: "ActivityPub", - }) - if err != nil { - t.Fatalf("getEngagementMetrics error: %v", err) - } - if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if out.TwitterPostCount != 0 { - t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out.TwitterPostCount) - } -} - // ---- pure compute functions ---- func TestComputePostingPatterns(t *testing.T) { diff --git a/internal/server/auth.go b/internal/server/auth.go index 07c8c24..f822e48 100644 --- a/internal/server/auth.go +++ b/internal/server/auth.go @@ -2,7 +2,7 @@ package server import ( "context" - "crypto" + "crypto/ecdsa" "crypto/ed25519" "crypto/rand" "crypto/x509" @@ -20,7 +20,7 @@ import ( type CDPAuth struct { keyName string - privateKey crypto.Signer + privateKey interface{} // *ecdsa.PrivateKey or ed25519.PrivateKey signingMethod jwt.SigningMethod } @@ -36,12 +36,13 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { // Convert to real newlines before decoding; idempotent if already multiline. privateKeyPEM = strings.ReplaceAll(privateKeyPEM, "\\n", "\n") - // Check if it's a base64-encoded Ed25519 key (64 bytes). - decoded, err := base64.StdEncoding.DecodeString(privateKeyPEM) - if err == nil && len(decoded) == 64 { + // Check if it's a base64-encoded Ed25519 key (64 bytes) + if isValidEd25519Key(privateKeyPEM) { + decoded, _ := base64.StdEncoding.DecodeString(privateKeyPEM) + privateKey := ed25519.PrivateKey(decoded) return &CDPAuth{ keyName: keyName, - privateKey: ed25519.PrivateKey(decoded), + privateKey: privateKey, signingMethod: jwt.SigningMethodEdDSA, }, nil } @@ -65,6 +66,15 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { }, nil } +// isValidEd25519Key checks if a string is a valid base64-encoded Ed25519 private key (64 bytes). +func isValidEd25519Key(str string) bool { + decoded, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return false + } + return len(decoded) == 64 +} + func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { nonceBytes := make([]byte, 16) if _, err := rand.Read(nonceBytes); err != nil { @@ -91,8 +101,10 @@ func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { var err error switch a.signingMethod { - case jwt.SigningMethodES256, jwt.SigningMethodEdDSA: - signedToken, err = token.SignedString(a.privateKey) + case jwt.SigningMethodES256: + signedToken, err = token.SignedString(a.privateKey.(*ecdsa.PrivateKey)) + case jwt.SigningMethodEdDSA: + signedToken, err = token.SignedString(a.privateKey.(ed25519.PrivateKey)) default: return "", fmt.Errorf("unsupported signing method: %v", a.signingMethod) } diff --git a/internal/server/backends.go b/internal/server/backends.go index 3182a3b..8bd39a6 100644 --- a/internal/server/backends.go +++ b/internal/server/backends.go @@ -26,15 +26,15 @@ type blueskyBackend interface { GetFeed(ctx context.Context, feedURI string, limit int) (*bsky.FeedGetFeed_Output, error) ResolveUser(ctx context.Context, user string) (syntax.DID, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) } type apBackend interface { GetPost(ctx context.Context, postID string) (*activitypub.Post, error) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*activitypub.Post, error) - GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) + GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) } type nostrBackend interface { @@ -42,7 +42,7 @@ type nostrBackend interface { GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*nostr.Post, error) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*nostr.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) } type twitterBackend interface { diff --git a/internal/server/cache.go b/internal/server/cache.go index 09b7f2a..a9b5869 100644 --- a/internal/server/cache.go +++ b/internal/server/cache.go @@ -6,7 +6,6 @@ import ( ) const postCacheTTL = 5 * time.Minute -const postCacheMaxItems = 10000 type ttlCacheEntry struct { value any @@ -14,17 +13,15 @@ type ttlCacheEntry struct { } type ttlCache struct { - mu sync.Mutex - items map[string]ttlCacheEntry - ttl time.Duration - maxItems int + mu sync.Mutex + items map[string]ttlCacheEntry + ttl time.Duration } func newTTLCache(ttl time.Duration) *ttlCache { return &ttlCache{ - items: make(map[string]ttlCacheEntry), - ttl: ttl, - maxItems: postCacheMaxItems, + items: make(map[string]ttlCacheEntry), + ttl: ttl, } } @@ -45,22 +42,6 @@ func (c *ttlCache) get(key string) (any, bool) { func (c *ttlCache) set(key string, value any) { c.mu.Lock() defer c.mu.Unlock() - if len(c.items) >= c.maxItems { - // Evict the oldest expired entry, or if none expired, the first one. - var oldestKey string - var oldestTime time.Time - first := true - for k, e := range c.items { - if first || e.exp.Before(oldestTime) { - oldestKey = k - oldestTime = e.exp - first = false - } - } - if oldestKey != "" { - delete(c.items, oldestKey) - } - } c.items[key] = ttlCacheEntry{value: value, exp: time.Now().Add(c.ttl)} } @@ -83,6 +64,6 @@ func cachedFetch[T any](c *ttlCache, key string, fetch func() (T, error)) (T, er } type latestCacheValue struct { - posts []*GetPostResponsePost + posts []GetPostResponsePost aid string } diff --git a/internal/server/cache_version.go b/internal/server/cache_version.go index 40ea07e..25fc192 100644 --- a/internal/server/cache_version.go +++ b/internal/server/cache_version.go @@ -8,7 +8,6 @@ const ( // IsProfileStale returns true if the profile's versions are older than the current versions. // This is used to mark profiles stale for lazy recompute on the next request. -// Currently all three version fields are 1 since this is the first version of the combination. func IsProfileStale(model, schema, algorithm int) bool { return model != CurrentModelVersion || schema != CurrentSchemaVersion || diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 1ff56b1..9e5a1df 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -27,7 +27,7 @@ type fakeBluesky struct { postView *bsky.FeedDefs_PostView feedViews []*bsky.FeedDefs_FeedViewPost profile *actor.Profile - actors []*actor.Profile + actors []actor.Profile feedOut *bsky.FeedGetFeed_Output err error calls int @@ -76,7 +76,7 @@ func (f *fakeBluesky) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { +func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -89,7 +89,7 @@ type fakeAP struct { post *activitypub.Post posts []*activitypub.Post profile *actor.Profile - actors []*actor.Profile + actors []actor.Profile err error calls int lastLimit int @@ -109,13 +109,15 @@ func (f *fakeAP) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } return f.posts, nil } -func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) { +func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) { f.calls++ if f.err != nil { return nil, f.err } - out := make([]*activitypub.Post, 0, len(f.posts)) - out = append(out, f.posts...) + out := make([]activitypub.Post, 0, len(f.posts)) + for _, p := range f.posts { + out = append(out, *p) + } return out, nil } func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -125,7 +127,7 @@ func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, e } return f.profile, nil } -func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { +func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -138,7 +140,7 @@ type fakeNostr struct { post *nostr.Post posts []*nostr.Post profile *actor.Profile - actors []*actor.Profile + actors []actor.Profile err error calls int lastLimit int @@ -172,7 +174,7 @@ func (f *fakeNostr) GetActor(ctx context.Context, author string) (*actor.Profile } return f.profile, nil } -func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { +func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -185,7 +187,7 @@ type fakeTwitter struct { post *twitter.Post posts []*twitter.Post profile *actor.Profile - actors []*actor.Profile + actors []actor.Profile err error calls int } @@ -218,7 +220,7 @@ func (f *fakeTwitter) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { +func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { f.calls++ if f.err != nil { return nil, f.err @@ -392,10 +394,10 @@ func TestGetActorDispatch(t *testing.T) { } func TestSearchActorsAllSucceed(t *testing.T) { - bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice", Limit: 10}) if err != nil { @@ -423,8 +425,8 @@ func TestSearchActorsAllFail(t *testing.T) { func TestSearchActorsPartialSuccess(t *testing.T) { bsky := &fakeBluesky{err: fmt.Errorf("boom")} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} tw := &fakeTwitter{err: fmt.Errorf("boom")} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice"}) @@ -738,9 +740,9 @@ func TestGetActorTwitter(t *testing.T) { // ---- searchActors edge cases ---- func TestSearchActorsLimitClamp(t *testing.T) { - bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, &fakeTwitter{}) // Limit > 50 should be clamped to 50 in the backend call. @@ -763,10 +765,10 @@ func TestSearchActorsLimitClamp(t *testing.T) { func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { // Twitter is not supported for search in the current implementation - bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "test", Limit: 10}) if err != nil { @@ -784,8 +786,8 @@ func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { func TestSearchActorsNormalizationAndDedup(t *testing.T) { // Test that handles normalize correctly and deduplicate - bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} + bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} s := newTestServer(bsky, ap, &fakeNostr{}, &fakeTwitter{}) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "user", Limit: 10}) if err != nil { @@ -823,7 +825,7 @@ func TestPostCacheDifferentKeys(t *testing.T) { func TestCacheVersionStaleness(t *testing.T) { st := newFakeStore() - // Add a stale profile (older than 30 days) + // Add a stale profile (older than 7 days) st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", Platform: "ActivityPub", @@ -831,9 +833,9 @@ func TestCacheVersionStaleness(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, } - ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}}} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "@a@mastodon.social"}) if err != nil { diff --git a/internal/server/platform.go b/internal/server/platform.go index a0330cd..6dcb2bf 100644 --- a/internal/server/platform.go +++ b/internal/server/platform.go @@ -26,7 +26,7 @@ func determinePlatform(author string) (platform, error) { if len(author) == 64 { isHex := true for _, c := range author { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { isHex = false break } diff --git a/internal/server/posts.go b/internal/server/posts.go index cd2bd28..eed3144 100644 --- a/internal/server/posts.go +++ b/internal/server/posts.go @@ -16,34 +16,34 @@ func (s *Server) fetchSinglePost(ctx context.Context, platform platform, author, return nil, err } r := bskyPostViewToResponse(res) - return r, nil + return &r, nil case ActivityPub: p, err := s.apClient.GetPost(ctx, recordKey) if err != nil { return nil, err } - r := apPostToResponse(p) - return r, nil + r := apPostToResponse(*p) + return &r, nil case Nostr: p, err := s.nostrClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := nostrPostToResponse(p) - return r, nil + return &r, nil case Twitter: p, err := s.twitterClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := twitterPostToResponse(p) - return r, nil + return &r, nil } return nil, fmt.Errorf("unsupported platform %q", platform) }) } -func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]*GetPostResponsePost, string, error) { +func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]GetPostResponsePost, string, error) { key := fmt.Sprintf("latest:%s|%s|%d", platform, author, limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -84,7 +84,7 @@ func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]*GetPostResponsePost, string, error) { +func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]GetPostResponsePost, string, error) { key := fmt.Sprintf("time:%s|%s|%s|%s|%d", platform, author, start.Format(time.RFC3339), end.Format(time.RFC3339), limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -125,9 +125,9 @@ func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]*GetPostResponsePost, error) { +func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]GetPostResponsePost, error) { key := fmt.Sprintf("feed:%s|%s", platform, uri) - return cachedFetch(s.postCache, key, func() ([]*GetPostResponsePost, error) { + return cachedFetch(s.postCache, key, func() ([]GetPostResponsePost, error) { if platform == Bluesky { res, err := s.bskyClient.GetFeed(ctx, uri, 50) if err != nil { diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index f35b394..181c9fc 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -10,7 +10,7 @@ import ( "github.com/nacorid/naco-api/internal/twitter" ) -func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) *GetPostResponsePost { +func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { author := "" if p.Author != nil { author = p.Author.Did @@ -34,7 +34,7 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) *GetPostResponsePost { if p.ReplyCount != nil { replies = *p.ReplyCount } - return &GetPostResponsePost{ + return GetPostResponsePost{ Author: author, Content: content, Likes: likes, @@ -45,12 +45,12 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) *GetPostResponsePost { } } -func apPostToResponse(p *activitypub.Post) *GetPostResponsePost { +func apPostToResponse(p activitypub.Post) GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return &GetPostResponsePost{ + return GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.FavouritesCount), @@ -60,24 +60,24 @@ func apPostToResponse(p *activitypub.Post) *GetPostResponsePost { } } -func nostrPostToResponse(p *nostr.Post) *GetPostResponsePost { +func nostrPostToResponse(p *nostr.Post) GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return &GetPostResponsePost{ + return GetPostResponsePost{ Author: p.Author, Content: p.Content, CreatedAt: created, } } -func twitterPostToResponse(p *twitter.Post) *GetPostResponsePost { +func twitterPostToResponse(p *twitter.Post) GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return &GetPostResponsePost{ + return GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.Likes), @@ -90,8 +90,8 @@ func twitterPostToResponse(p *twitter.Post) *GetPostResponsePost { // ---- Batch converters for GetPostResponsePost ---- -func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]*GetPostResponsePost, string) { - posts := make([]*GetPostResponsePost, 0, len(views)) +func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]GetPostResponsePost, string) { + posts := make([]GetPostResponsePost, 0, len(views)) var aid string for _, p := range views { if p.Post == nil { @@ -105,8 +105,18 @@ func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]*GetPos return posts, aid } -func apPostsPtrToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { - out := make([]*GetPostResponsePost, 0, len(posts)) +func apPostsPtrToResponses(posts []*activitypub.Post) ([]GetPostResponsePost, string) { + out := make([]GetPostResponsePost, 0, len(posts)) + var aid string + for _, p := range posts { + out = append(out, apPostToResponse(*p)) + aid = p.Author + } + return out, aid +} + +func apPostsToResponses(posts []activitypub.Post) ([]GetPostResponsePost, string) { + out := make([]GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, apPostToResponse(p)) @@ -115,18 +125,8 @@ func apPostsPtrToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, s return out, aid } -func apPostsToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { - out := make([]*GetPostResponsePost, 0, len(posts)) - var aid string - for _, p := range posts { - out = append(out, apPostToResponse(p)) - aid = p.Author - } - return out, aid -} - -func nostrPostsToResponses(posts []*nostr.Post) ([]*GetPostResponsePost, string) { - out := make([]*GetPostResponsePost, 0, len(posts)) +func nostrPostsToResponses(posts []*nostr.Post) ([]GetPostResponsePost, string) { + out := make([]GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, nostrPostToResponse(p)) @@ -135,8 +135,8 @@ func nostrPostsToResponses(posts []*nostr.Post) ([]*GetPostResponsePost, string) return out, aid } -func twitterPostsToResponses(posts []*twitter.Post) ([]*GetPostResponsePost, string) { - out := make([]*GetPostResponsePost, 0, len(posts)) +func twitterPostsToResponses(posts []*twitter.Post) ([]GetPostResponsePost, string) { + out := make([]GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, twitterPostToResponse(p)) @@ -154,9 +154,6 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I if !ok { continue } - if fp.Text == "" { - continue - } out = append(out, clustering.Item{ID: p.Post.Uri, Text: fp.Text}) } return out @@ -165,8 +162,8 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I func apPostsPtrToItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Content != "" { - out = append(out, clustering.Item{ID: p.ID, Text: p.Content}) + if p.Text != "" { + out = append(out, clustering.Item{ID: p.ID, Text: p.Text}) } } return out @@ -217,12 +214,12 @@ func blueskyFeedViewsToAnalyticsItems(views []*bsky.FeedDefs_FeedViewPost) []clu return out } -func apPostsToAnalyticsItems(posts []*activitypub.Post) []clustering.Item { +func apPostsToAnalyticsItems(posts []activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Content != "" { + if p.Text != "" { out = append(out, clustering.Item{ - ID: p.ID, Text: p.Content, Timestamp: p.CreatedAt, + ID: p.ID, Text: p.Text, Timestamp: p.CreatedAt, Likes: int64(p.FavouritesCount), Reposts: int64(p.ReblogsCount), Replies: int64(p.RepliesCount), }) } diff --git a/internal/server/prewarm.go b/internal/server/prewarm.go index 2fd9041..55caf69 100644 --- a/internal/server/prewarm.go +++ b/internal/server/prewarm.go @@ -83,11 +83,19 @@ func (s *Server) discoverInterestingAuthors(ctx context.Context) []string { } func (s *Server) prewarmOne(ctx context.Context, author string) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() if _, err := s.requestStyleProfile(ctx, author); err != nil { log.WarnWithContext(ctx, "prewarm: failed to build profile", "author", author, "error", err) return } log.InfoWithContext(ctx, "prewarmed style profile", "author", author) } + +func (s *Server) startPostPruner() { + ticker := time.NewTicker(24 * time.Hour) + defer ticker.Stop() + for range ticker.C { + if err := s.db.PruneStylePosts(context.Background()); err != nil { + log.Error("failed to prune style posts", "error", err) + } + } +} diff --git a/internal/server/searchactors_test.go b/internal/server/searchactors_test.go index 61fa8c6..dcab2f3 100644 --- a/internal/server/searchactors_test.go +++ b/internal/server/searchactors_test.go @@ -29,7 +29,7 @@ func TestNormalizeHandle(t *testing.T) { } func TestDedupeActors(t *testing.T) { - in := []*ActorProfile{ + in := []ActorProfile{ {Handle: "@alice", DisplayName: "Alice"}, {Handle: "alice@mastodon.social", DisplayName: "Alice"}, {Handle: "", DisplayName: ""}, @@ -45,7 +45,7 @@ func TestDedupeActors(t *testing.T) { func TestDedupeActorsByDisplayNameOnly(t *testing.T) { // When handles are empty but display names collide, dedupe by display name. - in := []*ActorProfile{ + in := []ActorProfile{ {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Other"}, @@ -59,10 +59,10 @@ func TestDedupeActorsByDisplayNameOnly(t *testing.T) { func TestSearchActorsTwitterExcluded(t *testing.T) { // searchActorsAll only queries Bluesky, ActivityPub and Nostr. A Twitter // result must never appear because the backend is not consulted. - tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} - bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} + bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "q"}) diff --git a/internal/server/server.go b/internal/server/server.go index 4cccf25..ef4508b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -48,12 +48,6 @@ type Server struct { analyticsTTLTopics time.Duration analyticsTTLSentiment time.Duration analyticsTTLDrivers time.Duration - - // twitterEnabled reports whether the Twitter API is available (a bearer - // token is configured). It flips the Twitter-capable tools to upto - // variable pricing so the realized Twitter API costs are passed on to - // callers (see registerTools). - twitterEnabled bool } func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky *bluesky.BlueskyClient, n *nostr.Client, t *twitter.Client, o *ollama.OllamaClient, cfg *config.Config) *Server { @@ -84,7 +78,6 @@ func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky analyticsTTLTopics: cfg.AnalyticsTTLTopics, analyticsTTLSentiment: cfg.AnalyticsTTLSentiment, analyticsTTLDrivers: cfg.AnalyticsTTLDrivers, - twitterEnabled: cfg.TwitterBearerToken != "", } // Combine address and port for TCP (not unix socket) if port not already present @@ -111,10 +104,6 @@ func (s *Server) MetricsText() string { } func (s *Server) Start() error { - //go s.startPostPruner() + go s.startPostPruner() return s.mcpHandler.Start(s.listenAddress, s.permissions) } - -func (s *Server) Shutdown(ctx context.Context) error { - return s.mcpHandler.Shutdown(ctx) -} diff --git a/internal/server/server_init.go b/internal/server/server_init.go index 832d103..88f801b 100644 --- a/internal/server/server_init.go +++ b/internal/server/server_init.go @@ -65,7 +65,7 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } srv, err := x402.NewX402Server( - "x402 enabled social media lookingglass for autonomous agents", + "x402 enabled Bluesky lookingglass for autonomous Agents", "0.2.0", cfg.X402FacilitatorURL, auth, @@ -85,9 +85,7 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "amount", requirements.Amount, "asset", requirements.Asset, ) - if err := persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer); err != nil { - log.WarnWithContext(r.Context(), "failed to persist HTTP settlement", "error", err) - } + persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer) }, networkStrings, ) @@ -105,11 +103,9 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "asset", settleCtx.Requirements.GetAsset(), "scheme", settleCtx.Requirements.GetScheme(), ) - if err := persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, + persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, settleCtx.Requirements.GetAmount(), settleCtx.Requirements.GetAsset(), - string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer); err != nil { - log.WarnWithContext(settleCtx.Ctx, "failed to persist settlement", "error", err) - } + string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer) // Webhook is best-effort and non-blocking. go sendSettlementWebhook(settleCtx.Ctx, cfg, metrics, settleCtx.Result.Transaction) @@ -120,8 +116,8 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } // persistSettlement is a thin wrapper around settleHook to keep call sites uniform. -func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) error { - return settleHook(ctx, db, transaction, amount, asset, network, to, from) +func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) { + _ = settleHook(ctx, db, transaction, amount, asset, network, to, from) } // sendSettlementWebhook builds and sends the Matrix webhook payload for a settled transaction. @@ -136,7 +132,8 @@ func sendSettlementWebhook(ctx context.Context, cfg *config.Config, metrics *Met "x402_transaction_hash": txHash, }, "metrics": map[string]any{ - "platform": "unknown", + "function_runtime_ms": 0, + "platform": "unknown", }, "timestamp": time.Now().UTC().Format(time.RFC3339), "server_id": cfg.MatrixServerID, diff --git a/internal/server/settlement.go b/internal/server/settlement.go index 4acb2a0..03c1a34 100644 --- a/internal/server/settlement.go +++ b/internal/server/settlement.go @@ -36,7 +36,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass } if err != nil { - log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "error", err) + log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "value", ret, "error", err) return err } @@ -44,7 +44,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass return nil } -func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string, timeout int) []x402v2.PaymentRequirements { +func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string) []x402v2.PaymentRequirements { requirements := make([]x402v2.PaymentRequirements, 0, len(s.activeNetworks)) for _, info := range s.activeNetworks { @@ -57,7 +57,7 @@ func (s *Server) buildRequirements(amount string, extra map[string]any, scheme s Asset: info.Asset, Amount: amount, PayTo: s.x402Wallet, - MaxTimeoutSeconds: timeout, + MaxTimeoutSeconds: 600, Extra: mergedExtra, }) } diff --git a/internal/server/storefake_test.go b/internal/server/storefake_test.go index 4ab3b24..066ca5e 100644 --- a/internal/server/storefake_test.go +++ b/internal/server/storefake_test.go @@ -94,7 +94,7 @@ func (f *fakeStore) GetAnalyticsPostsInRange(ctx context.Context, arg store.GetA Quotes: p.Quotes, }) } - sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.Before(rows[j].PostTimestamp) }) + sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.After(rows[j].PostTimestamp) }) if int32(len(rows)) > arg.Limit { rows = rows[:arg.Limit] } diff --git a/internal/server/style.go b/internal/server/style.go index 8015d50..5f9e217 100644 --- a/internal/server/style.go +++ b/internal/server/style.go @@ -9,8 +9,6 @@ import ( "sync" "time" - "strings" - "github.com/modelcontextprotocol/go-sdk/mcp" log "github.com/nacorid/logger" "github.com/nacorid/naco-api/internal/clustering" @@ -54,7 +52,7 @@ func (s *Server) getStyleProfile(ctx context.Context, request *mcp.CallToolReque } // requestStyleProfile returns a style profile for author. It computes and persists the -// profile only when a request is made AND the cached profile is older than a month (or +// profile only when a request is made AND the cached profile is older than a week (or // missing); otherwise it serves the cached copy. The prewarmer calls this same path, so a // pre-fetch is treated as a system-initiated request and obeys the identical update policy. func (s *Server) requestStyleProfile(ctx context.Context, author string) (*GetStyleProfileResponse, error) { @@ -88,7 +86,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor Platform: platform.String(), }) if err == nil { - isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*30*time.Hour + isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*7*time.Hour if isRecent { embedding, perr := utils.ParseEmbedding(styleProfile.Embedding) if perr != nil { @@ -112,7 +110,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor // enrichCrossPlatform looks for author on every platform other than sourcePlatform, in the // background. It uses the same identifier on each platform (matching only where that handle -// exists; mismatches fail gracefully and are skipped) and obeys the same 1-month update gate +// exists; mismatches fail gracefully and are skipped) and obeys the same 1-week update gate // as a live request. Concurrency is bounded to the other three platforms. func (s *Server) enrichCrossPlatform(ctx context.Context, sourcePlatform platform, author string) { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) @@ -193,14 +191,7 @@ func (s *Server) getSimilarProfiles(ctx context.Context, request *mcp.CallToolRe }) } - twitterPostCount := 0 - if profile.SourcePlatform == "Twitter" { - twitterPostCount = profile.PostCount - } - respBytes, err := json.Marshal(FindSimilarUsersResponse{ - Matches: matches, - TwitterPostCount: twitterPostCount, - }) + respBytes, err := json.Marshal(FindSimilarUsersResponse{Matches: matches}) if err != nil { return nil, err } @@ -220,21 +211,6 @@ func (s *Server) buildStyleProfile(ctx context.Context, platform platform, autho return nil, fmt.Errorf("no posts found for author %s", author) } - // Drop posts without any text (e.g. Bluesky image/reply-only posts). Embedding them - // yields empty vectors, which would poison the persona-average and the persisted - // profile. - filtered := posts[:0] - for _, p := range posts { - if strings.TrimSpace(p.Text) == "" { - continue - } - filtered = append(filtered, p) - } - if len(filtered) == 0 { - return nil, fmt.Errorf("no posts found for author %s", author) - } - posts = filtered - // Delete old posts for non-Twitter platforms before rebuild if platform != Twitter { if err := s.db.DeleteStylePosts(ctx, store.DeleteStylePostsParams{ diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index 9051f18..ffc8079 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -3,7 +3,6 @@ package server import ( "database/sql" "fmt" - "reflect" "strings" "testing" "time" @@ -17,8 +16,8 @@ import ( func TestGetStyleProfileBuildsFresh(t *testing.T) { ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "alpha post one", CreatedAt: time.Now()}, - {Author: "@a@mastodon.social", Content: "alpha post two", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post one", Text: "alpha post one", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post two", Text: "alpha post two", CreatedAt: time.Now()}, }} st := newFakeStore() oll := &fakeOllama{embeddings: [][]float64{{0.1, 0.2}, {0.3, 0.4}}} @@ -47,7 +46,7 @@ func TestGetStyleProfileBuildsFresh(t *testing.T) { } func TestGetStyleProfileServesFreshCache(t *testing.T) { - // A profile modified less than the 30-day stale threshold is served without hitting the + // A profile modified less than a week ago is served without hitting the // backend or ollama. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ @@ -83,7 +82,7 @@ func TestGetStyleProfileServesFreshCache(t *testing.T) { } func TestGetStyleProfileRebuildsStale(t *testing.T) { - // A stale (older than a month) cached profile triggers a rebuild. + // A stale (older than a week) cached profile triggers a rebuild. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", @@ -92,10 +91,10 @@ func TestGetStyleProfileRebuildsStale(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, } ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}, }} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) @@ -135,7 +134,7 @@ func TestGetStyleProfileNoPostsFound(t *testing.T) { } func TestGetStyleProfileOllamaEmbedFailure(t *testing.T) { - ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", Text: "post", CreatedAt: time.Now()}}} oll := &fakeOllama{embedErr: fmt.Errorf("embedding failed")} st := newFakeStore() s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, oll) @@ -166,61 +165,6 @@ func TestGetStyleProfileBlueskyResolution(t *testing.T) { } } -func TestGetStyleProfileFirstEmbeddingEmpty(t *testing.T) { - // Regression: an empty embedding returned by Ollama for a non-first post used to - // panic AverageVector when the first vector was also empty (accumulator sized to - // length 0, later non-empty vector indexed it). Empty vectors are skipped. - bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ - {Post: bskyPostView("first post")}, - {Post: bskyPostView("second post")}, - }} - oll := &fakeOllama{embeddings: [][]float64{{}, {0.1, 0.2}}} - st := newFakeStore() - s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) - - res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) - if err != nil { - t.Fatalf("getStyleProfile error: %v", err) - } - var out GetStyleProfileResponse - if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if !reflect.DeepEqual(out.PersonaEmbedding, []float64{0.1, 0.2}) { - t.Fatalf("expected averaged embedding {0.1,0.2}, got %v", out.PersonaEmbedding) - } - if out.PostCount != 2 { - t.Fatalf("expected post count 2, got %d", out.PostCount) - } -} - -func TestGetStyleProfileSkipsEmptyTextPosts(t *testing.T) { - // A Bluesky post with no text (image/reply-only) must not reach Ollama: its empty - // embedding would poison the persona average. It is dropped before embedding. - bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ - {Post: bskyPostView("")}, - {Post: bskyPostView("real post")}, - }} - oll := &fakeOllama{} - st := newFakeStore() - s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) - - res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) - if err != nil { - t.Fatalf("getStyleProfile error: %v", err) - } - var out GetStyleProfileResponse - if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if out.PostCount != 1 { - t.Fatalf("expected filtered post count 1, got %d", out.PostCount) - } - if len(out.PersonaEmbedding) != 2 { - t.Fatalf("expected 2-dim embedding, got %v", out.PersonaEmbedding) - } -} - // ---- findSimilarUsers ---- func TestFindSimilarUsers(t *testing.T) { @@ -304,61 +248,6 @@ func TestFindSimilarUsersEmptyMatches(t *testing.T) { } } -// TestFindSimilarUsersTwitterPostCount verifies the response reports how many -// posts went into the query author's style profile when that author is on -// Twitter, so upto-scheme settlement can charge the realized Twitter API costs. -func TestFindSimilarUsersTwitterPostCount(t *testing.T) { - st := newFakeStore() - st.similarRows = []store.FindSimilarUsersRow{ - {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, - } - // Serve a fresh cached Twitter profile with 42 analyzed posts. - st.styleProfiles["@alice|Twitter"] = store.UserStyleProfile{ - UserID: "@alice", - Platform: "Twitter", - Embedding: "[0.1,0.2]", - PostCount: 42, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, - } - s := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) - res, err := callTool(s, "findSimilarUsers", FindSimilarUsersRequest{Author: "@alice"}) - if err != nil { - t.Fatalf("findSimilarUsers error: %v", err) - } - var out FindSimilarUsersResponse - if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if out.TwitterPostCount != 42 { - t.Fatalf("expected TwitterPostCount 42, got %d", out.TwitterPostCount) - } - - // A non-Twitter query author reports 0. - st2 := newFakeStore() - st2.similarRows = []store.FindSimilarUsersRow{ - {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, - } - st2.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ - UserID: "@a@mastodon.social", - Platform: "ActivityPub", - Embedding: "[0.1,0.2]", - PostCount: 9, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, - } - s2 := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st2, &fakeOllama{}) - res2, err := callTool(s2, "findSimilarUsers", FindSimilarUsersRequest{Author: "@a@mastodon.social"}) - if err != nil { - t.Fatalf("findSimilarUsers error: %v", err) - } - var out2 FindSimilarUsersResponse - if err := jsonUnmarshal(t, textOf(t, res2), &out2); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if out2.TwitterPostCount != 0 { - t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out2.TwitterPostCount) - } -} - func TestFindSimilarUsersDBError(t *testing.T) { st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ diff --git a/internal/server/tools.go b/internal/server/tools.go index bb91c3d..510aef4 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -7,7 +7,6 @@ import ( "time" "github.com/modelcontextprotocol/go-sdk/mcp" - "github.com/nacorid/naco-api/internal/x402" ) // registerTools registers all MCP tools with their x402 payment requirements. @@ -37,26 +36,14 @@ func (s *Server) registerTools() { } type payableDef struct { - Name string - Desc string - Input any - Output any - Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) - Price string - MaxTimeoutSeconds int - // UsesTwitter marks tools that may read posts (or set up lookups) - // through the Twitter API and return only transformative/aggregated - // results, as permitted by the X API policy. With Twitter enabled these - // tools charge upto-scheme: base price plus the realized Twitter API - // costs. Tools that resell raw post/profile/feed data (getPost, - // getLatestPosts, ...) never set this. - UsesTwitter bool - // TwitterPostCap is the post count used to compute the advertised upto - // maximum (see x402.MaxTwitterCostMicro). It only bounds the largest - // settlement the tool can demand; the actual charge is settled from the - // response. - TwitterPostCap int64 - Extra map[string]any + Name string + Desc string + Input any + Output any + Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) + Price string + Scheme string + Extra map[string]any } payable := []payableDef{ @@ -106,16 +93,8 @@ func (s *Server) registerTools() { ), }, { - // base price is $0.25 (x402.BaseProfileCostMicro). With Twitter - // enabled the tool is advertised as upto at the maximum possible - // cost - $0.25 base + $0.01 lookup + $0.005/post at the 100-post - // cap = $0.76 (x402.MaxTwitterCostMicro(BaseProfileCostMicro, - // MaxProfileCountedPosts)) - and the actual charge is settled from - // the posts read via x402.NewStyleProfileCostResolver. Name: "getStyleProfile", Desc: "Get an actor's style Profile", - Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.25", - MaxTimeoutSeconds: 600, - UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, + Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.36", Scheme: "upto", Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -130,8 +109,7 @@ func (s *Server) registerTools() { }, { Name: "findSimilarUsers", Desc: "Find authors with a similar writing style to the given author, across all platforms", - Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.10", - UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, + Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.05", Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -169,7 +147,6 @@ func (s *Server) registerTools() { { Name: "getEngagementMetrics", Desc: "Get time-series engagement metrics (likes, reposts, replies, quotes) per post", Input: AnalyticsRequest{}, Output: EngagementMetricsResponse{}, Handler: s.getEngagementMetrics, Price: "0.15", - UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "metrics": Bodyfield{Type: "array", Description: "Engagement metrics per post"}, @@ -178,7 +155,6 @@ func (s *Server) registerTools() { { Name: "getPostingPatterns", Desc: "Get posting heatmap and patterns (hour-of-day x day-of-week)", Input: AnalyticsRequest{}, Output: PostingPatternsResponse{}, Handler: s.getPostingPatterns, Price: "0.10", - UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "heatmap": Bodyfield{Type: "array", Description: "24x7 matrix: hour (0-23) x day of week (0-6 = Sun-Sat)"}, @@ -192,7 +168,6 @@ func (s *Server) registerTools() { { Name: "getTopicModeling", Desc: "Get LLM-based topic extraction and clustering from posts", Input: AnalyticsRequest{}, Output: TopicModelingResponse{}, Handler: s.getTopicModeling, Price: "0.20", - UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "topics": Bodyfield{Type: "array", Description: "Identified topic clusters"}, @@ -201,7 +176,6 @@ func (s *Server) registerTools() { { Name: "getSentimentTrends", Desc: "Get sentiment analysis and emotional arcs over time", Input: AnalyticsRequest{}, Output: SentimentTrendsResponse{}, Handler: s.getSentimentTrends, Price: "0.15", - UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "timeSeries": Bodyfield{Type: "array", Description: "Sentiment over time"}, @@ -213,7 +187,6 @@ func (s *Server) registerTools() { { Name: "getEngagementDrivers", Desc: "Get topics/keywords that correlate with high engagement", Input: AnalyticsRequest{}, Output: EngagementDriversResponse{}, Handler: s.getEngagementDrivers, Price: "0.20", - UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "drivers": Bodyfield{Type: "array", Description: "Topics/keywords driving engagement"}, @@ -222,29 +195,14 @@ func (s *Server) registerTools() { } for _, d := range payable { - // Twitter-capable tools are advertised as upto (variable-priced) only - // when the Twitter API is actually enabled: the upto maximum adds the - // realized Twitter lookup and per-post costs, and the actual charge is - // settled from the response. Without Twitter, every tool is exact at - // its flat base price. - scheme := "exact" - price := d.Price - var resolver x402.SettleAmountResolver - if d.UsesTwitter && s.twitterEnabled { - scheme = "upto" - baseMicro := x402.MicroFromPriceString(d.Price) - price = x402.FormatPriceMicro(x402.MaxTwitterCostMicro(baseMicro, d.TwitterPostCap)) - if d.Name == "getStyleProfile" { - resolver = x402.NewStyleProfileCostResolver() - } else { - resolver = x402.NewTwitterCountCostResolver(baseMicro) - } + scheme := d.Scheme + if scheme == "" { + scheme = "exact" } s.mcpHandler.AddPayableTool( mcp.Tool{Name: d.Name, Description: d.Desc, InputSchema: d.Input, OutputSchema: d.Output}, d.Handler, - resolver, - s.buildRequirements(price, d.Extra, scheme, d.MaxTimeoutSeconds)..., + s.buildRequirements(d.Price, d.Extra, scheme)..., ) } @@ -375,7 +333,7 @@ func (s *Server) getPostsInTime(ctx context.Context, request *mcp.CallToolReques if authorID == "" { authorID = args.Author } - resp := GetPostsInTimeResponse{ + resp := &GetPostsInTimeResponse{ Author: authorID, Posts: posts, } diff --git a/internal/server/types.go b/internal/server/types.go index 7d833c6..b1915b7 100644 --- a/internal/server/types.go +++ b/internal/server/types.go @@ -26,8 +26,8 @@ type GetLatestPostsRequest struct { } type GetLatestPostsResponse struct { - Author string `json:"actor" jsonschema_description:"Did of the actor"` - Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` + Author string `json:"actor" jsonschema_description:"Did of the actor"` + Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` } type GetPostsInTimeRequest struct { @@ -38,8 +38,8 @@ type GetPostsInTimeRequest struct { } type GetPostsInTimeResponse struct { - Author string `json:"author" jsonschema_description:"Did of the author"` - Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` + Author string `json:"author" jsonschema_description:"Did of the author"` + Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` } type GetPostRequest struct { @@ -63,8 +63,8 @@ type GetFeedRequest struct { } type GetFeedResponse struct { - URI string `json:"uri" jsonschema_description:"The URI of the feed"` - Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` + URI string `json:"uri" jsonschema_description:"The URI of the feed"` + Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` } type GetStyleProfileRequest struct { @@ -87,10 +87,6 @@ type FindSimilarUsersRequest struct { type FindSimilarUsersResponse struct { Matches []SimilarUser `json:"matches" jsonschema_description:"Ranked list of similar profiles by persona embedding similarity"` - // TwitterPostCount reports how many posts were sourced from the Twitter API - // while building the query author's style profile. It is 0 when the query - // author is not on Twitter. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts used for the query author's profile"` } type ActorProfile = actor.Profile @@ -109,7 +105,7 @@ type SearchActorsRequest struct { } type SearchActorsResponse struct { - Results []*ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` + Results []ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` } type SimilarUser struct { @@ -169,10 +165,6 @@ type AnalyticsRequest struct { type EngagementMetricsResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Metrics []EngagementMetricPoint `json:"metrics" jsonschema_description:"Engagement metrics per post"` - // TwitterPostCount reports how many posts in the result were sourced from - // the Twitter API, so upto-scheme settlement can pass the realized Twitter - // costs on to the caller. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementMetricPoint struct { @@ -193,19 +185,11 @@ type PostingPatternsResponse struct { AvgPostsPerDay float64 `json:"avgPostsPerDay" jsonschema_description:"Average posts per day"` TotalPosts int `json:"totalPosts" jsonschema_description:"Total posts in time range"` PostingDays int `json:"postingDays" jsonschema_description:"Number of days with at least one post"` - // TwitterPostCount reports how many posts in the result were sourced from - // the Twitter API, so upto-scheme settlement can pass the realized Twitter - // costs on to the caller. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicModelingResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Topics []TopicCluster `json:"topics" jsonschema_description:"Identified topic clusters"` - // TwitterPostCount reports how many posts in the result were sourced from - // the Twitter API, so upto-scheme settlement can pass the realized Twitter - // costs on to the caller. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicCluster struct { @@ -223,10 +207,6 @@ type SentimentTrendsResponse struct { OverallSentiment SentimentSummary `json:"overallSentiment" jsonschema_description:"Overall sentiment summary"` DominantEmotions []EmotionScore `json:"dominantEmotions" jsonschema_description:"Most frequent emotions"` SentimentShifts []SentimentShift `json:"sentimentShifts" jsonschema_description:"Significant sentiment changes"` - // TwitterPostCount reports how many posts in the result were sourced from - // the Twitter API, so upto-scheme settlement can pass the realized Twitter - // costs on to the caller. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type SentimentPoint struct { @@ -258,10 +238,6 @@ type SentimentShift struct { type EngagementDriversResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Drivers []EngagementDriver `json:"drivers" jsonschema_description:"Topics/keywords driving engagement"` - // TwitterPostCount reports how many posts in the result were sourced from - // the Twitter API, so upto-scheme settlement can pass the realized Twitter - // costs on to the caller. - TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementDriver struct { diff --git a/internal/server/webhook.go b/internal/server/webhook.go index adda05e..75708c1 100644 --- a/internal/server/webhook.go +++ b/internal/server/webhook.go @@ -47,12 +47,7 @@ func (wh *WebhookSender) Send(ctx context.Context, jsonBytes []byte) { for attempt := 1; attempt <= 5; attempt++ { delay := exponentialBackoff(attempt) if attempt > 1 { - select { - case <-time.After(delay): - case <-ctx.Done(): - log.WarnWithContext(ctx, "webhook cancelled during backoff", "attempt", attempt, "error", ctx.Err()) - return - } + time.Sleep(delay) } err := wh.sendAttempt(ctx, jsonBytes, attempt) diff --git a/internal/store/integration_test.go b/internal/store/integration_test.go index 5c9aeaf..2adc0b4 100644 --- a/internal/store/integration_test.go +++ b/internal/store/integration_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "os" - "reflect" "strings" "testing" "time" @@ -13,140 +12,93 @@ import ( _ "github.com/lib/pq" ) -// setupTestDB handles connection pooling, migrations, and cleanup per test run. -func setupTestDB(t *testing.T) (*sql.DB, *Queries) { - t.Helper() - +// TestStoreRoundTrip exercises the sqlc queries against a real Postgres+pgvector database. +// It is skipped unless TEST_DATABASE_URL is set (e.g. a local instance with the vector +// extension available), so it never fails in environments without a database. +func TestStoreRoundTrip(t *testing.T) { dsn := os.Getenv("TEST_DATABASE_URL") if dsn == "" { t.Skip("TEST_DATABASE_URL not set; skipping store integration test") } - ctx := context.Background() conn, err := sql.Open("postgres", dsn) if err != nil { - t.Fatalf("failed to open db: %v", err) + t.Fatal(err) } - + defer conn.Close() if err := conn.PingContext(ctx); err != nil { - _ = conn.Close() t.Fatalf("cannot reach test database: %v", err) } - - t.Cleanup(func() { - _ = conn.Close() - }) - if err := Migrate(ctx, conn, "sql/schema"); err != nil { t.Fatalf("migrate: %v", err) } - - return conn, New(conn) -} - -func makeEmbedding(dim int, val string) string { - return "[" + strings.TrimSuffix(strings.Repeat(val+",", dim), ",") + "]" -} - -func TestStoreRoundTrip(t *testing.T) { - conn, q := setupTestDB(t) - ctx := context.Background() + q := New(conn) uid := fmt.Sprintf("test_%d", time.Now().UnixNano()) - platform := "Bluesky" - emb1 := makeEmbedding(1024, "0.1") + emb := "[" + strings.TrimSuffix(strings.Repeat("0.0,", 1024), ",") + "]" - // Clean up after the test so leftover records do not pollute subsequent runs - t.Cleanup(func() { - _, _ = conn.ExecContext(context.Background(), "DELETE FROM style_profiles WHERE user_id = $1", uid) - }) - - // 1. Test INSERT path of Upsert - err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ + if err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ UserID: uid, - Platform: platform, - Embedding: emb1, + Platform: "Bluesky", + Embedding: emb, TopKeywords: []string{"alpha", "beta"}, - StyleSummary: "original summary", + StyleSummary: "summary", PostCount: 5, - }) + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: "Bluesky"}) if err != nil { - t.Fatalf("initial upsert failed: %v", err) + t.Fatalf("get: %v", err) + } + if p.UserID != uid || p.Platform != "Bluesky" || p.PostCount != 5 { + t.Fatalf("unexpected profile: %+v", p) } - // 2. Test GET and verify fields - p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) + rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{QueryEmbedding: emb, LimitVal: 10}) if err != nil { - t.Fatalf("get failed: %v", err) + t.Fatalf("find similar: %v", err) } - - if p.UserID != uid || p.Platform != platform || p.PostCount != 5 { - t.Errorf("mismatched basic fields: %+v", p) - } - if p.StyleSummary != "original summary" { - t.Errorf("got summary %q, want %q", p.StyleSummary, "original summary") - } - if !reflect.DeepEqual(p.TopKeywords, []string{"alpha", "beta"}) { - t.Errorf("got keywords %v, want %v", p.TopKeywords, []string{"alpha", "beta"}) - } - - // 3. Test UPDATE path of Upsert (Verify conflict resolution works) - emb2 := makeEmbedding(1024, "0.2") - err = q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ - UserID: uid, - Platform: platform, - Embedding: emb2, - TopKeywords: []string{"gamma"}, - StyleSummary: "updated summary", - PostCount: 10, - }) - if err != nil { - t.Fatalf("secondary upsert (update) failed: %v", err) - } - - updated, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) - if err != nil { - t.Fatalf("get updated profile failed: %v", err) - } - if updated.PostCount != 10 || updated.StyleSummary != "updated summary" { - t.Errorf("upsert failed to update existing record: %+v", updated) - } - - // 4. Test pgvector similarity search - rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{ - QueryEmbedding: emb2, - LimitVal: 10, - }) - if err != nil { - t.Fatalf("find similar failed: %v", err) - } - found := false for _, r := range rows { if r.UserID == uid { found = true - break } } if !found { - t.Errorf("expected user %s in similarity results, but was not found", uid) + t.Fatalf("expected newly inserted profile to be returned by similarity search") } } +// TestStoreSaveTransaction exercises SaveTransaction against a real Postgres database. +// It is skipped unless TEST_DATABASE_URL is set. func TestStoreSaveTransaction(t *testing.T) { - conn, q := setupTestDB(t) + dsn := os.Getenv("TEST_DATABASE_URL") + if dsn == "" { + t.Skip("TEST_DATABASE_URL not set; skipping store integration test") + } ctx := context.Background() + conn, err := sql.Open("postgres", dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.PingContext(ctx); err != nil { + t.Fatalf("cannot reach test database: %v", err) + } + if err := Migrate(ctx, conn, "sql/schema"); err != nil { + t.Fatalf("migrate: %v", err) + } + q := New(conn) - txHash := fmt.Sprintf("0x_%d", time.Now().UnixNano()) + txHash := fmt.Sprintf("0x_%d", time.Now().Unix()) amount := "10.0" asset := "ETH" network := "eip155:1" - recipient := "0x1234567890abcdef" - sender := "0xabcdef1234567890" - - t.Cleanup(func() { - _, _ = conn.ExecContext(context.Background(), "DELETE FROM transactions WHERE x402_transaction_hash = $1", txHash) - }) + now := time.Now().Unix() + recipient := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000), "0") + sender := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000+0xab), "0") ret, err := q.SaveTransaction(ctx, SaveTransactionParams{ X402TransactionHash: txHash, @@ -161,34 +113,22 @@ func TestStoreSaveTransaction(t *testing.T) { } if len(ret) != 1 { - t.Fatalf("expected 1 result row returned, got %d", len(ret)) + t.Fatalf("expected 1 result, got %d", len(ret)) } - saved := ret[0] - if saved.ID <= 0 { - t.Errorf("expected positive ID, got %d", saved.ID) - } - if !saved.CreatedAt.Valid { - t.Error("expected CreatedAt to be set") + id := ret[0].ID + if id <= 0 { + t.Fatalf("expected positive ID, got %d", id) } - // Since the sqlc query only returns `id` and `created_at`, - // verify directly that PostgreSQL stored the remaining columns accurately: - var ( - gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender string - ) - err = conn.QueryRowContext(ctx, ` - SELECT x402_transaction_hash, amount, asset, network, recipient, sender - FROM transactions - WHERE id = $1 - `, saved.ID).Scan(&gotHash, &gotAmount, &gotAsset, &gotNetwork, &gotRecipient, &gotSender) - if err != nil { - t.Fatalf("failed to query inserted transaction from db: %v", err) + // Verify the returned row has the expected fields + if !ret[0].CreatedAt.Valid { + t.Fatal("expected CreatedAt to be set") } - if gotHash != txHash || gotAmount != amount || gotAsset != asset || - gotNetwork != network || gotRecipient != recipient || gotSender != sender { - t.Errorf("stored values do not match input params: got (%s, %s, %s, %s, %s, %s)", - gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender) - } + // Verify we can retrieve the transaction back + // Note: we'd need a SELECT query for full verification, but the insert + // returning id and created_at is the critical path tested here + + t.Logf("SaveTransaction successful: id=%d, hash=%s", id, txHash) } diff --git a/internal/twitter/client.go b/internal/twitter/client.go index f849796..08e97e2 100644 --- a/internal/twitter/client.go +++ b/internal/twitter/client.go @@ -25,12 +25,6 @@ func parseTwitterTime(s string) time.Time { const apiBase = "https://api.x.com/2" -// httpTimeout bounds a single HTTP request so a stalled X backend cannot block -// a request indefinitely. -const httpTimeout = 60 * time.Second - -const userAgent = "naco-api/1.0" - type Client struct { httpClient *http.Client bearerToken string @@ -38,7 +32,7 @@ type Client struct { func NewTwitterClient(bearerToken string) *Client { return &Client{ - httpClient: &http.Client{Timeout: httpTimeout}, + httpClient: http.DefaultClient, bearerToken: bearerToken, } } @@ -111,150 +105,95 @@ type singleTweetResponse struct { } `json:"errors"` } -const ( - maxRetries = 4 - maxRateLimitWait = 30 * time.Second // Never stall a request longer than 30s -) - -var defaultBackoff = 500 * time.Millisecond - +// doGet issues a GET and retries with exponential backoff when Twitter returns HTTP 429 +// (Too Many Requests). It honors the x-rate-limit-* headers and caps retries so a +// sustained rate limit still surfaces as a real error to the caller (and thus isn't billed). func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { - backoff := defaultBackoff - + const maxRetries = 4 + backoff := 500 * time.Millisecond for attempt := 0; ; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, fmt.Errorf("failed to build request: %w", err) } req.Header.Set("Authorization", "Bearer "+c.bearerToken) - req.Header.Set("User-Agent", userAgent) resp, err := c.httpClient.Do(req) if err != nil { - // Transport-level failure (connection refused/reset, timeout). - if attempt >= maxRetries { - return nil, fmt.Errorf("twitter request failed after %d attempts: %w", maxRetries+1, err) - } - log.WarnWithContext(ctx, "twitter transport error, waiting to retry", "attempt", attempt+1, "error", err) - if werr := waitForRetry(ctx, backoff); werr != nil { - return nil, werr - } - backoff *= 2 - continue + return nil, fmt.Errorf("failed to do request: %w", err) } - - // Handle 429 Too Many Requests - if resp.StatusCode == http.StatusTooManyRequests { - // Drain & close body before retrying. - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - - if attempt >= maxRetries { - return nil, fmt.Errorf("twitter rate limit exceeded, max retries reached") - } - - wait := parseRateLimitWait(resp.Header, backoff) - - // Protect against 15-minute thread freezes - if wait > maxRateLimitWait { - return nil, fmt.Errorf("twitter rate limited: reset wait (%s) exceeds maximum allowed (%s)", wait, maxRateLimitWait) - } - - log.WarnWithContext(ctx, "twitter rate limited, waiting to retry", "attempt", attempt+1, "wait", wait.String()) - if werr := waitForRetry(ctx, wait); werr != nil { - return nil, werr - } - backoff *= 2 - continue - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) - _ = resp.Body.Close() + body, err := io.ReadAll(resp.Body) + resp.Body.Close() if err != nil { return nil, fmt.Errorf("failed to read response: %w", err) } - // Retry transient server-side failures (5xx) with backoff. 4xx/3xx fail - // fast below. - if resp.StatusCode >= http.StatusInternalServerError && attempt < maxRetries { - log.WarnWithContext(ctx, "twitter server error, waiting to retry", "attempt", attempt+1, "status", resp.Status) - if werr := waitForRetry(ctx, backoff); werr != nil { - return nil, werr + if resp.StatusCode == http.StatusTooManyRequests { + if attempt >= maxRetries { + return nil, fmt.Errorf("twitter rate limited after retries: %s", resp.Status) + } + // Honor x-rate-limit-* headers first + remaining := resp.Header.Get("x-rate-limit-remaining") + reset := resp.Header.Get("x-rate-limit-reset") + if ra := remaining; ra != "" { + if rem, perr := strconv.Atoi(ra); perr == nil { + if rem <= 0 && reset != "" { + if rst, perr2 := strconv.Atoi(reset); perr2 == nil { + // Wait until the rate limit resets (Unix timestamp) + now := time.Now().Unix() + rst64 := int64(rst) + wait := time.Duration(rst64-now) * time.Second + if wait > 0 { + log.WarnWithContext(ctx, "twitter rate limited, waiting until reset", "reset", reset, "wait", wait.String()) + select { + case <-time.After(wait): + case <-ctx.Done(): + return nil, ctx.Err() + } + backoff = 500 * time.Millisecond + continue + } + } + } + } + } + // Fall back to Retry-After if headers not useful + if ra := resp.Header.Get("Retry-After"); ra != "" { + if secs, perr := strconv.Atoi(ra); perr == nil && secs > 0 { + backoff = time.Duration(secs) * time.Second + } + } + log.WarnWithContext(ctx, "twitter rate limited, backing off", "attempt", attempt+1, "wait", backoff.String()) + select { + case <-time.After(backoff): + case <-ctx.Done(): + return nil, ctx.Err() } backoff *= 2 continue } if resp.StatusCode != http.StatusOK { - errMsg := strings.TrimSpace(string(body)) - if len(errMsg) > 512 { - errMsg = errMsg[:512] + "... (truncated)" - } - return nil, fmt.Errorf("twitter error (%s): %s", resp.Status, errMsg) + return nil, fmt.Errorf("twitter error: %s: %s", resp.Status, string(body)) } - return body, nil } } -// waitForRetry sleeps for wait, aborting early if the context is canceled. -// A timer is used (not time.After) so the timer can be stopped to free its -// resource when the caller abandons the wait. -func waitForRetry(ctx context.Context, wait time.Duration) error { - timer := time.NewTimer(wait) - defer timer.Stop() - select { - case <-timer.C: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -// parseRateLimitWait extracts x-rate-limit-* or Retry-After, falling back to backoff. -func parseRateLimitWait(header http.Header, fallback time.Duration) time.Duration { - // 1. Check x-rate-limit-reset (Unix timestamp) - if remStr := header.Get("x-rate-limit-remaining"); remStr != "" { - if rem, err := strconv.Atoi(remStr); err == nil && rem <= 0 { - if rstStr := header.Get("x-rate-limit-reset"); rstStr != "" { - if rst, err := strconv.ParseInt(rstStr, 10, 64); err == nil { - // Add 1s clock skew buffer - wait := time.Until(time.Unix(rst, 0)) + (1 * time.Second) - if wait > 0 { - return wait - } - } - } - } - } - - // 2. Check standard Retry-After (seconds) - if ra := header.Get("Retry-After"); ra != "" { - if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { - return time.Duration(secs)*time.Second + (1 * time.Second) - } - } - - return fallback -} - -func (c *Client) resolveUser(ctx context.Context, username string) (id, handle string, err error) { +func (c *Client) resolveUser(ctx context.Context, username string) (string, error) { endpoint := fmt.Sprintf("%s/users/by/username/%s", apiBase, url.PathEscape(username)) body, err := c.doGet(ctx, endpoint) if err != nil { - return "", "", err + return "", err } var ur userResponse if err := json.Unmarshal(body, &ur); err != nil { - return "", "", fmt.Errorf("failed to unmarshal user response: %w", err) + return "", fmt.Errorf("failed to unmarshal user response: %w", err) } if len(ur.Errors) > 0 { - return "", "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) + return "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) } - if ur.Data.ID == "" { - return "", "", fmt.Errorf("twitter user %q not found", username) - } - return ur.Data.ID, ur.Data.Username, nil + return ur.Data.ID, nil } func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { @@ -263,7 +202,7 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } username := strings.TrimPrefix(actor, "@") - userID, handle, err := c.resolveUser(ctx, username) + userID, err := c.resolveUser(ctx, username) if err != nil { return nil, err } @@ -271,34 +210,12 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( if limit > 100 { limit = 100 } + if limit < 5 { + limit = 5 + } endpoint := fmt.Sprintf("%s/users/%s/tweets?max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", apiBase, url.PathEscape(userID), limit) - return c.fetchTweets(ctx, handle, endpoint) -} - -func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { - if c.bearerToken == "" { - return nil, fmt.Errorf("twitter bearer token not configured") - } - username := strings.TrimPrefix(actor, "@") - - userID, handle, err := c.resolveUser(ctx, username) - if err != nil { - return nil, err - } - if limit > 100 { - limit = 100 - } - - endpoint := fmt.Sprintf("%s/users/%s/tweets?start_time=%s&end_time=%s&max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", - apiBase, url.PathEscape(userID), start.Format(time.RFC3339), end.Format(time.RFC3339), limit) - return c.fetchTweets(ctx, handle, endpoint) -} - -// fetchTweets performs the request and converts the tweet payload into Posts, -// skipping entries without an ID so downstream dedupe keys are never empty. -func (c *Client) fetchTweets(ctx context.Context, username, endpoint string) ([]*Post, error) { body, err := c.doGet(ctx, endpoint) if err != nil { return nil, err @@ -314,14 +231,61 @@ func (c *Client) fetchTweets(ctx context.Context, username, endpoint string) ([] posts := make([]*Post, 0, len(tr.Data)) for _, t := range tr.Data { - if t.ID == "" { - continue - } + created := parseTwitterTime(t.CreatedAt) posts = append(posts, &Post{ ID: t.ID, - Author: username, + Author: userID, Content: t.Text, - CreatedAt: parseTwitterTime(t.CreatedAt), + CreatedAt: created, + Likes: t.PublicMetrics.LikeCount, + Reposts: t.PublicMetrics.RetweetCount, + Quotes: t.PublicMetrics.QuoteCount, + Replies: t.PublicMetrics.ReplyCount, + }) + } + return posts, nil +} + +func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { + if c.bearerToken == "" { + return nil, fmt.Errorf("twitter bearer token not configured") + } + username := strings.TrimPrefix(actor, "@") + + userID, err := c.resolveUser(ctx, username) + if err != nil { + return nil, err + } + if limit > 100 { + limit = 100 + } + if limit < 5 { + limit = 5 + } + + endpoint := fmt.Sprintf("%s/users/%s/tweets?start_time=%s&end_time=%s&max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", + apiBase, url.PathEscape(userID), start.Format(time.RFC3339), end.Format(time.RFC3339), limit) + body, err := c.doGet(ctx, endpoint) + if err != nil { + return nil, err + } + + var tr tweetsResponse + if err := json.Unmarshal(body, &tr); err != nil { + return nil, fmt.Errorf("failed to unmarshal tweets response: %w", err) + } + if len(tr.Errors) > 0 { + return nil, fmt.Errorf("twitter error: %s: %s", tr.Errors[0].Title, tr.Errors[0].Detail) + } + + posts := make([]*Post, 0, len(tr.Data)) + for _, t := range tr.Data { + created := parseTwitterTime(t.CreatedAt) + posts = append(posts, &Post{ + ID: t.ID, + Author: userID, + Content: t.Text, + CreatedAt: created, Likes: t.PublicMetrics.LikeCount, Reposts: t.PublicMetrics.RetweetCount, Quotes: t.PublicMetrics.QuoteCount, @@ -352,18 +316,32 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { if tr.Data.ID == "" { return nil, fmt.Errorf("tweet %q not found", id) } - author := "unknown" + author := id if len(tr.Includes.Users) > 0 { author = tr.Includes.Users[0].Username } + created, _ := time.Parse(time.RFC3339, tr.Data.CreatedAt) return &Post{ ID: tr.Data.ID, Author: author, Content: tr.Data.Text, - CreatedAt: parseTwitterTime(tr.Data.CreatedAt), + CreatedAt: created, Likes: tr.Data.PublicMetrics.LikeCount, Reposts: tr.Data.PublicMetrics.RetweetCount, Quotes: tr.Data.PublicMetrics.QuoteCount, Replies: tr.Data.PublicMetrics.ReplyCount, }, nil } + +type userFields struct { + ID string `json:"id"` + Name string `json:"name"` + Username string `json:"username"` + Description string `json:"description"` + ProfileImageURL string `json:"profile_image_url"` + PublicMetrics struct { + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + TweetCount int `json:"tweet_count"` + } `json:"public_metrics"` +} diff --git a/internal/twitter/client_test.go b/internal/twitter/client_test.go index 87a6024..9dca301 100644 --- a/internal/twitter/client_test.go +++ b/internal/twitter/client_test.go @@ -2,22 +2,14 @@ package twitter import ( "context" - "errors" "fmt" - "io" "net/http" "net/http/httptest" "strings" - "sync" "testing" "time" ) -// Speed up test suite: override defaultBackoff to 1ms so retries are instantaneous. -func init() { - defaultBackoff = 1 * time.Millisecond -} - type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } @@ -54,8 +46,7 @@ func TestGetLatestNPosts(t *testing.T) { w.WriteHeader(http.StatusUnauthorized) return } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -63,8 +54,7 @@ func TestGetLatestNPosts(t *testing.T) { return } tweetsURL = r.URL.String() - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) + fmt.Fprint(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -77,7 +67,7 @@ func TestGetLatestNPosts(t *testing.T) { if len(posts) != 1 { t.Fatalf("expected 1 post, got %d", len(posts)) } - if posts[0].Content != "hello world" || posts[0].Author != "testuser" { + if posts[0].Content != "hello world" || posts[0].Author != "12345" { t.Fatalf("unexpected post: %+v", posts[0]) } if !strings.Contains(tweetsURL, "max_results=10") { @@ -85,18 +75,16 @@ func TestGetLatestNPosts(t *testing.T) { } } -func TestGetLatestNPostsLimitNoLowerFloor(t *testing.T) { +func TestGetLatestNPostsLimitClamp(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) + fmt.Fprint(w, `{"data":{"id":"1","username":"u"}}`) }) mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { - if got := r.URL.Query().Get("max_results"); got != "3" { - t.Fatalf("expected max_results=3 for limit=3, got %q", got) + if got := r.URL.Query().Get("max_results"); got != "5" { + t.Fatalf("expected clamped max_results=5 for limit=3, got %q", got) } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[]}`) + fmt.Fprint(w, `{"data":[]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -122,16 +110,14 @@ func TestGetLatestNPostsHTTPError(t *testing.T) { func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) + fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/tweets/999", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { w.WriteHeader(http.StatusUnauthorized) return } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) + fmt.Fprint(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -152,8 +138,7 @@ func TestGetPost(t *testing.T) { func TestGetPostsInTimeframe(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) + fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -163,8 +148,7 @@ func TestGetPostsInTimeframe(t *testing.T) { if r.URL.Query().Get("start_time") == "" || r.URL.Query().Get("end_time") == "" { t.Fatalf("expected start_time and end_time query params") } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) + fmt.Fprint(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -190,19 +174,54 @@ func TestRateLimitRetryWithResetHeader(t *testing.T) { mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ if callCount == 1 { - // First call: rate limited with reset header slightly in the past/present + // First call: rate limited with reset header w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Unix())) + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(100*time.Millisecond).Unix())) w.WriteHeader(http.StatusTooManyRequests) return } // Second call: success - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) + fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + // Use a short context timeout to avoid waiting too long + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + posts, err := c.GetLatestNPosts(ctx, "@testuser", 10) + if err != nil { + t.Fatalf("GetLatestNPosts error after retry: %v", err) + } + if callCount != 2 { + t.Fatalf("expected 2 calls (1 retry), got %d", callCount) + } + if len(posts) != 1 { + t.Fatalf("expected 1 post after retry, got %d", len(posts)) + } +} + +func TestRateLimitRetryWithRetryAfterHeader(t *testing.T) { + callCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + callCount++ + if callCount == 1 { + // First call: rate limited with Retry-After header + w.Header().Set("Retry-After", "1") // 1 second + w.WriteHeader(http.StatusTooManyRequests) + return + } + // Second call: success + fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + }) + mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -228,25 +247,29 @@ func TestRateLimitExhaustedAfterMaxRetries(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ + // Always rate limited - no reset header, no Retry-After, so exponential backoff is used w.Header().Set("x-rate-limit-remaining", "0") + // Don't set x-rate-limit-reset or Retry-After to force exponential backoff w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + // Use a timeout long enough for all retries (4 retries with exponential backoff: 500ms, 1s, 2s, 4s = ~7.5s) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error after max retries exhausted") } + // Should have tried maxRetries (4) + 1 initial = 5 times if callCount < 5 { t.Fatalf("expected at least 5 attempts (1 initial + 4 retries), got %d", callCount) } - if !strings.Contains(err.Error(), "rate limit") { - t.Fatalf("expected rate limit error, got %v", err) + if !strings.Contains(err.Error(), "rate limited after retries") { + t.Fatalf("expected 'rate limited after retries' error, got %v", err) } } @@ -254,8 +277,7 @@ func TestUserLookupNotFound(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) + fmt.Fprint(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -274,8 +296,7 @@ func TestUserLookupUnauthorized(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) + fmt.Fprint(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -293,153 +314,65 @@ func TestUserLookupUnauthorized(t *testing.T) { func TestUserLookupRateLimited(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "1") w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + // Need enough time for max retries (4 retries * 1 second = 4 seconds) + initial + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error for rate limited user lookup") } - if !strings.Contains(err.Error(), "rate limit") { - t.Fatalf("expected rate limit error, got %v", err) + if !strings.Contains(err.Error(), "twitter error") && !strings.Contains(err.Error(), "rate limited after retries") { + t.Fatalf("expected twitter error or rate limited after retries, got %v", err) } } func TestContextCancellationDuringRetry(t *testing.T) { - reqReceived := make(chan struct{}) - var once sync.Once - + callCount := 0 mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) - w.WriteHeader(http.StatusTooManyRequests) - once.Do(func() { - close(reqReceived) // Signal test that the 429 response was returned - }) + callCount++ + // First call: rate limited with long reset time + if callCount == 1 { + w.Header().Set("x-rate-limit-remaining", "0") + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) + w.WriteHeader(http.StatusTooManyRequests) + return + } }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) + // Cancel context immediately after first call ctx, cancel := context.WithCancel(context.Background()) - + // Start the request in a goroutine done := make(chan error, 1) go func() { _, err := c.GetLatestNPosts(ctx, "@testuser", 10) done <- err }() - // Deterministic wait: block until the mock server receives the request, then cancel immediately - select { - case <-reqReceived: - cancel() - case <-time.After(2 * time.Second): - t.Fatal("timed out waiting for mock request to be hit") - } + // Give it a moment to make the first call, then cancel + time.Sleep(100 * time.Millisecond) + cancel() select { case err := <-done: if err == nil { t.Fatal("expected error from context cancellation") } - if !errors.Is(err, context.Canceled) && !strings.Contains(err.Error(), "context canceled") { - t.Fatalf("expected context.Canceled, got %v", err) + if !strings.Contains(err.Error(), "context canceled") && !strings.Contains(err.Error(), "context cancelled") { + t.Fatalf("expected context cancellation error, got %v", err) } case <-time.After(2 * time.Second): - t.Fatal("test timed out waiting for goroutine to abort") - } -} - -func TestResolveUserEmptyData(t *testing.T) { - // 200 with an empty data object (banned/suspended or privacy-restricted - // account) must error rather than build a bogus /users//tweets request. - var tweetsCalled bool - mux := http.NewServeMux() - mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{}}`) - }) - mux.HandleFunc("/2/users//tweets", func(w http.ResponseWriter, r *http.Request) { - tweetsCalled = true - }) - server := httptest.NewServer(mux) - defer server.Close() - - c := newTestClient(server.URL) - _, err := c.GetLatestNPosts(context.Background(), "@ghost", 10) - if err == nil { - t.Fatal("expected error for empty user data, got nil") - } - if !strings.Contains(err.Error(), "not found") { - t.Fatalf("expected 'not found' error, got %v", err) - } - if tweetsCalled { - t.Fatal("tweets endpoint should not be called when user lookup returns empty data") - } -} - -func TestGetLatestNPostsSkipsEmptyID(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) - }) - mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[{"id":"","text":"bad"},{"id":"2","text":"good","created_at":"2024-01-01T00:00:00Z"}]}`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - c := newTestClient(server.URL) - posts, err := c.GetLatestNPosts(context.Background(), "u", 10) - if err != nil { - t.Fatalf("error: %v", err) - } - if len(posts) != 1 { - t.Fatalf("expected 1 post (empty-ID tweet skipped), got %d", len(posts)) - } - if posts[0].ID != "2" || posts[0].Content != "good" { - t.Fatalf("unexpected post: %+v", posts[0]) - } -} - -func TestServerErrorRetried(t *testing.T) { - callCount := 0 - mux := http.NewServeMux() - mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - callCount++ - if callCount == 1 { - w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"title":"Internal Server Error","detail":"transient"}`) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) - }) - mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"ok","created_at":"2024-01-01T00:00:00Z"}]}`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - c := newTestClient(server.URL) - posts, err := c.GetLatestNPosts(context.Background(), "u", 10) - if err != nil { - t.Fatalf("GetLatestNPosts error after 5xx retry: %v", err) - } - if callCount != 2 { - t.Fatalf("expected 2 calls (1 failure + 1 retry), got %d", callCount) - } - if len(posts) != 1 || posts[0].Content != "ok" { - t.Fatalf("unexpected posts after retry: %+v", posts) + t.Fatal("test timed out waiting for cancellation") } } diff --git a/internal/utils/styleProfiles.go b/internal/utils/styleProfiles.go index cb475a4..7a6a587 100644 --- a/internal/utils/styleProfiles.go +++ b/internal/utils/styleProfiles.go @@ -56,34 +56,17 @@ func FormatVector(v []float64) string { return "[" + strings.Join(parts, ",") + "]" } -// AverageVector returns the element-wise mean of vs. Zero-length vectors (e.g. an -// embedding model returning an empty vector for a post without text) are skipped, and -// the accumulator is sized from the first non-empty vector so a leading empty vector -// cannot panic later in the loop. Returns nil when no usable vector is present. func AverageVector(vs [][]float64) []float64 { - dim := 0 - var valid [][]float64 - for _, v := range vs { - if len(v) == 0 { - continue - } - if dim == 0 { - dim = len(v) - } - valid = append(valid, v) - } - if len(valid) == 0 { - return nil - } - + dim := len(vs[0]) avg := make([]float64, dim) - for _, v := range valid { - for i := 0; i < dim && i < len(v); i++ { - avg[i] += v[i] + + for _, v := range vs { + for i, val := range v { + avg[i] += val } } for i := range avg { - avg[i] /= float64(len(valid)) + avg[i] /= float64(len(vs)) } return avg } diff --git a/internal/utils/styleProfiles_test.go b/internal/utils/styleProfiles_test.go deleted file mode 100644 index 4692b95..0000000 --- a/internal/utils/styleProfiles_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package utils - -import ( - "math" - "testing" -) - -func assertVectorsEqual(t *testing.T, got, want []float64) { - t.Helper() - if len(got) != len(want) { - t.Fatalf("length %d, want %d", len(got), len(want)) - } - for i := range want { - if math.Abs(got[i]-want[i]) > 1e-9 { - t.Fatalf("index %d = %v, want %v", i, got, want) - } - } -} - -func TestAverageVector(t *testing.T) { - tests := []struct { - name string - in [][]float64 - want []float64 - }{ - {name: "empty input", in: nil, want: nil}, - {name: "empty slice", in: [][]float64{}, want: nil}, - {name: "all empty vectors", in: [][]float64{{}, {}, {}}, want: nil}, - {name: "single vector", in: [][]float64{{0.4, -0.2}}, want: []float64{0.4, -0.2}}, - {name: "multiple vectors", in: [][]float64{{0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, - {name: "first vector empty", in: [][]float64{{}, {0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, - {name: "trailing empty vectors", in: [][]float64{{0.1, 0.2}, {}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, - {name: "only first valid", in: [][]float64{{}, {}, {0.1, 0.2}}, want: []float64{0.1, 0.2}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := AverageVector(tt.in) - if tt.want == nil { - if got != nil { - t.Fatalf("AverageVector(%v) = %v, want nil", tt.in, got) - } - return - } - assertVectorsEqual(t, got, tt.want) - }) - } -} diff --git a/internal/x402/mcpwrapper.go b/internal/x402/mcpwrapper.go deleted file mode 100644 index 8d69b01..0000000 --- a/internal/x402/mcpwrapper.go +++ /dev/null @@ -1,156 +0,0 @@ -package x402 - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/modelcontextprotocol/go-sdk/mcp" - log "github.com/nacorid/logger" - x402 "github.com/x402-foundation/x402/go/v2" - "github.com/x402-foundation/x402/go/v2/types" -) - -// Payment meta keys exchanged over MCP, mirroring the x402 SDK contract. -const ( - PaymentMetaKey = "x402/payment" - PaymentResponseMetaKey = "x402/payment-response" -) - -// SettleAmountResolver computes the actual settlement amount for a completed -// tool call. It returns ("", false) to settle the amount advertised in the -// payment requirements as-is. -type SettleAmountResolver func(result *mcp.CallToolResult) (string, bool) - -// PaymentWrapperConfig configures a PaymentWrapper. Unlike the SDK's wrapper, -// it accepts a SettleAmountResolver so upto-scheme tools can charge the actual -// computed cost after the handler runs (the same behavior the REST path gets -// via nethttp.SetSettlementOverrides). -type PaymentWrapperConfig struct { - Accepts []x402.PaymentRequirements - Resource *types.ResourceInfo - Extensions map[string]interface{} - // SettleAmountResolver is invoked after a successful handler call. When it - // returns a non-empty amount, that amount is charged instead of the - // advertised maximum (which must therefore be <= the accept amount). - SettleAmountResolver SettleAmountResolver -} - -// PaymentWrapper gates an MCP tool behind x402 payments and settles them. -type PaymentWrapper struct { - server *x402.X402ResourceServer - config PaymentWrapperConfig -} - -// NewPaymentWrapper creates a PaymentWrapper for the given resource server. -func NewPaymentWrapper(server *x402.X402ResourceServer, config PaymentWrapperConfig) *PaymentWrapper { - if len(config.Accepts) == 0 { - panic("PaymentWrapperConfig.Accepts must have at least one payment requirement") - } - return &PaymentWrapper{server: server, config: config} -} - -// Wrap returns a ToolHandler that requires a valid x402 payment before invoking -// the wrapped handler and settles it afterwards. -func (w *PaymentWrapper) Wrap(handler mcp.ToolHandler) mcp.ToolHandler { - return func(ctx context.Context, request *mcp.CallToolRequest) (*mcp.CallToolResult, error) { - paymentData := extractPaymentFromRequest(request) - if paymentData == nil { - return w.paymentRequiredResult("Payment Required"), nil - } - - payloadBytes, err := json.Marshal(paymentData) - if err != nil { - return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil - } - var payload types.PaymentPayload - if err := json.Unmarshal(payloadBytes, &payload); err != nil { - return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil - } - - matched := w.server.FindMatchingRequirements(w.config.Accepts, payload) - if matched == nil { - return w.paymentRequiredResult("No matching payment requirements found"), nil - } - requirements := *matched - - verifyResp, err := w.server.VerifyPayment(ctx, payload, requirements) - if err != nil { - return w.paymentRequiredResult(fmt.Sprintf("Payment verification error: %v", err)), nil - } - if !verifyResp.IsValid { - return w.paymentRequiredResult(fmt.Sprintf("Payment verification failed: %s", verifyResp.InvalidReason)), nil - } - - result, err := handler(ctx, request) - if err != nil { - return nil, err - } - if result.IsError { - return result, nil - } - - var overrides *x402.SettlementOverrides - if w.config.SettleAmountResolver != nil { - if amount, ok := w.config.SettleAmountResolver(result); ok && amount != "" { - overrides = &x402.SettlementOverrides{Amount: amount} - log.Debug("MCP settlement override", "amount", amount) - } - } - - settleResp, err := w.server.SettlePayment(ctx, payload, requirements, overrides) - if err != nil { - return w.settlementFailedResult(fmt.Sprintf("Settlement error: %v", err)), nil - } - if !settleResp.Success { - return w.settlementFailedResult(fmt.Sprintf("Settlement failed: %s", settleResp.ErrorReason)), nil - } - - if result.Meta == nil { - result.Meta = mcp.Meta{} - } - result.Meta[PaymentResponseMetaKey] = settleResp - return result, nil - } -} - -func extractPaymentFromRequest(request *mcp.CallToolRequest) interface{} { - if request == nil || request.Params.Meta == nil { - return nil - } - return request.Params.Meta[PaymentMetaKey] -} - -func (w *PaymentWrapper) paymentRequiredResult(errorMessage string) *mcp.CallToolResult { - var resource *types.ResourceInfo - if w.config.Resource != nil { - resource = &types.ResourceInfo{ - URL: w.config.Resource.URL, - Description: w.config.Resource.Description, - } - } - payment := types.PaymentRequired{ - X402Version: 2, - Error: errorMessage, - Resource: resource, - Accepts: w.config.Accepts, - Extensions: w.config.Extensions, - } - b, err := json.Marshal(payment) - if err != nil { - panic(err) - } - return &mcp.CallToolResult{ - IsError: true, - Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}, - } -} - -func (w *PaymentWrapper) settlementFailedResult(errorMessage string) *mcp.CallToolResult { - return &mcp.CallToolResult{ - IsError: true, - Content: []mcp.Content{ - &mcp.TextContent{Text: fmt.Sprintf("{\"x402Version\":2,\"error\":%q}", errorMessage)}, - }, - } -} diff --git a/internal/x402/x402.go b/internal/x402/x402.go index 28eeb5c..e1cb2a6 100644 --- a/internal/x402/x402.go +++ b/internal/x402/x402.go @@ -7,13 +7,10 @@ import ( "fmt" "io" "log/slog" - "math" "net" "net/http" "os" - "os/user" "reflect" - "slices" "strconv" "strings" "sync" @@ -25,9 +22,9 @@ import ( x402 "github.com/x402-foundation/x402/go/v2" x402http "github.com/x402-foundation/x402/go/v2/http" nethttp "github.com/x402-foundation/x402/go/v2/http/nethttp" + x402mcp "github.com/x402-foundation/x402/go/v2/mcp" evm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/exact/server" uptoevm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/upto/server" - "github.com/x402-foundation/x402/go/v2/types" ) type X402Middlewares struct { @@ -58,14 +55,6 @@ type X402Server struct { resourceInitMu sync.Mutex resourceInitDone bool - - // settleResolvers maps payable tool name -> SettleAmountResolver used by - // both the MCP wrapper and the REST override path to charge the actual - // computed cost (instead of the advertised upto maximum). - settleResolvers map[string]SettleAmountResolver - - httpServer *http.Server - mu sync.Mutex } const VERSION1 = "v1/" @@ -119,7 +108,6 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au tools: make([]ToolInfo, 0), globalReqLog: globalLoggingMw, networks: networks, - settleResolvers: make(map[string]SettleAmountResolver), } // Warm the facilitator cache now so AddPayableTool can build Accepts with @@ -140,7 +128,7 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au s.mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) + w.Write([]byte("ok")) }) s.mux.HandleFunc("GET /openapi.json", s.serveOpenAPI) @@ -163,9 +151,9 @@ const facilitatorRetryTimeout = 10 * time.Second // facilitator outage at boot (or afterwards) no longer requires a restart. func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { s.resourceInitMu.Lock() - defer s.resourceInitMu.Unlock() - - if s.resourceInitDone { + done := s.resourceInitDone + s.resourceInitMu.Unlock() + if done { return } @@ -176,7 +164,9 @@ func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { return } + s.resourceInitMu.Lock() s.resourceInitDone = true + s.resourceInitMu.Unlock() } func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { @@ -200,12 +190,11 @@ func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { func (s *X402Server) AddPayableTool( tool mcp.Tool, handler mcp.ToolHandler, - resolver SettleAmountResolver, requirements ...x402.PaymentRequirements, ) { if len(requirements) == 0 { - panic(fmt.Sprintf("tool %s requires at least one payment requirement", tool.Name)) + log.Fatalf("tool %s requires at least one payment requirement", tool.Name) } // Convert each raw dollar price into the asset's smallest-unit integer amount using the @@ -223,7 +212,7 @@ func (s *X402Server) AddPayableTool( Extra: req.Extra, }) if err != nil { - panic(fmt.Sprintf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err)) + log.Fatalf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err) } mcpAccepts = append(mcpAccepts, built...) } @@ -231,14 +220,9 @@ func (s *X402Server) AddPayableTool( tool.InputSchema = toJSONSchema(tool.InputSchema) tool.OutputSchema = toJSONSchema(tool.OutputSchema) - if resolver != nil { - s.settleResolvers[tool.Name] = resolver - } - - wrapper := NewPaymentWrapper(s.x402ResourceServer, PaymentWrapperConfig{ - Accepts: mcpAccepts, - SettleAmountResolver: resolver, - Resource: &types.ResourceInfo{ + wrapper := x402mcp.NewPaymentWrapper(s.x402ResourceServer, x402mcp.PaymentWrapperConfig{ + Accepts: mcpAccepts, + Resource: &x402mcp.ResourceInfo{ URL: "mcp://tool/" + tool.Name, Description: tool.Description, }, @@ -256,16 +240,27 @@ func (s *X402Server) AddPayableTool( var options x402http.PaymentOptions for _, req := range requirements { options = append(options, x402http.PaymentOption{ - Scheme: req.Scheme, - PayTo: req.PayTo, - Price: req.Amount, - Network: x402.Network(req.Network), - MaxTimeoutSeconds: req.MaxTimeoutSeconds, - Extra: req.Extra, + Scheme: req.Scheme, + PayTo: req.PayTo, + Price: req.Amount, + Network: x402.Network(req.Network), }) } - schemes := buildSchemeConfigs(s.networks, requirements...) + schemes := make([]nethttp.SchemeConfig, 0, len(s.networks)*2) + for _, net := range s.networks { + // For getStyleProfile, only register upto scheme (variable pricing) + if tool.Name == "getStyleProfile" { + schemes = append(schemes, + nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, + ) + } else { + schemes = append(schemes, + nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}, + nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, + ) + } + } settlementHandler := func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) { requirements, _ := nethttp.RequirementsFromContext(r.Context()) @@ -360,32 +355,8 @@ func (s *X402Server) AddPayableTool( s.mux.Handle("POST /"+VERSION1+tool.Name, s.middlewareChain(paymentMiddleware(restHandler))) } -// buildSchemeConfigs maps the schemes actually declared in a tool's payment -// requirements to the EVM scheme servers a route needs to verify and settle -// them, for every configured network. Only the schemes present in the -// requirements are registered: an exact-only tool does not expose an upto -// scheme, and vice versa. This keeps upto (variable-pricing) authorization from -// being offered for tools that cannot vary their price. -func buildSchemeConfigs(networks []string, requirements ...x402.PaymentRequirements) []nethttp.SchemeConfig { - present := make(map[string]struct{}) - for _, req := range requirements { - present[req.Scheme] = struct{}{} - } - schemes := make([]nethttp.SchemeConfig, 0, len(networks)*len(present)) - for _, net := range networks { - if _, ok := present["exact"]; ok { - schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}) - } - if _, ok := present["upto"]; ok { - schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}) - } - } - return schemes -} - func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, toolName string, handler mcp.ToolHandler) { - defer r.Body.Close() - bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 5*1024*1024)) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read body", http.StatusBadRequest) return @@ -407,31 +378,30 @@ func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, tool } w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) if len(resp.Content) == 1 { if textContent, ok := resp.Content[0].(*mcp.TextContent); ok { var js json.RawMessage if json.Unmarshal([]byte(textContent.Text), &js) == nil { - if resolver := s.settleResolvers[toolName]; resolver != nil { - if actualCost, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{textContent}}); ok && actualCost != "" { - log.DebugWithContext(r.Context(), "Setting settlement override", - "tool", toolName, - "actualCost", actualCost, - ) - nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ - Amount: actualCost, - }) - } + // For upto scheme tools, compute actual cost and apply settlement override + if toolName == "getStyleProfile" { + actualCost := computeTwitterStyleProfileCost(r, textContent.Text) + log.DebugWithContext(r.Context(), "Setting settlement override", + "tool", toolName, + "actualCost", actualCost, + ) + nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ + Amount: actualCost, + }) } - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(textContent.Text)) + w.Write([]byte(textContent.Text)) return } } } - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(resp) } func (s *X402Server) OnAfterSettle(hook func(ctx x402.SettleResultContext) error) { @@ -522,23 +492,6 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to set socket permissions: %w", err) } - grp, err := user.LookupGroup("mcpserver") - if err != nil { - return fmt.Errorf("failed to lookup group mcpserver: %w", err) - } - gid, err := strconv.Atoi(grp.Gid) - if err != nil { - return fmt.Errorf("invalid gid %q: %w", grp.Gid, err) - } - - if err := os.Chown(socketPath, -1, gid); err != nil { - return fmt.Errorf("failed to set socket group ownership: %w", err) - } - - if err := os.Chmod(socketPath, permissions); err != nil { - return fmt.Errorf("failed to set socket permissions: %w", err) - } - log.Info("Starting X402 MCP Server on unix socket", "socketPath", socketPath) } else { log.Info("Starting X402 MCP Server on TCP", "addr", addr) @@ -548,33 +501,15 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to listen on tcp: %w", err) } } + defer listener.Close() srv := &http.Server{ - Handler: s.globalReqLog(s.mux), - ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), - ReadTimeout: 15 * time.Second, - WriteTimeout: 60 * time.Second, - IdleTimeout: 120 * time.Second, + Handler: s.globalReqLog(s.mux), + ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), } - - s.mu.Lock() - s.httpServer = srv - s.mu.Unlock() - return srv.Serve(listener) } -func (s *X402Server) Shutdown(ctx context.Context) error { - s.mu.Lock() - srv := s.httpServer - s.mu.Unlock() - - if srv == nil { - return nil - } - return srv.Shutdown(ctx) -} - func toJSONSchema(val any) json.RawMessage { if val == nil { return json.RawMessage(`{"type": "object"}`) @@ -635,8 +570,8 @@ func parseSchema(schemaAny any) any { func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { - for _, middleware := range slices.Backward(middlewares) { - next = middleware(next) + for i := len(middlewares) - 1; i >= 0; i-- { + next = middlewares[i](next) } return next } @@ -648,60 +583,38 @@ func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(h // Twitter API costs guarantees the call always clears the underlying API cost // and generates a profit - it can never break even or lose money. const ( - // BaseProfileCostMicro is the base cost of the style-profile endpoint for + // baseProfileCostMicro is the base cost of the style-profile endpoint for // the non-Twitter path. It doubles as the profit margin added to Twitter // requests, expressed in micro-dollars (USDC has 6 decimals). - BaseProfileCostMicro int64 = 250_000 // $0.25 - // TwitterUserLookupMicro is the per-request Twitter user lookup API cost. - TwitterUserLookupMicro int64 = 10_000 // $0.01 - // TwitterPostReadMicro is the per-post Twitter read API cost. - TwitterPostReadMicro int64 = 5_000 // $0.005 per post - // MaxProfileCountedPosts caps the post count a style profile can ever - // report. It is NOT applied inside computeTwitterStyleProfileCost (charges - // follow the posts actually read, and the server fetches at most this - // many); it only bounds the largest settlement the endpoint can demand. - MaxProfileCountedPosts int64 = 100 - // MaxAnalyticsCountedPosts caps the number of posts an analytics tool can - // include from any single platform (mirrors maxAnalyticsLimit in the server - // package). It bounds the largest settlement an upto analytics request can - // demand. - MaxAnalyticsCountedPosts int64 = 200 + baseProfileCostMicro int64 = 100_000 // $0.10 + // twitterUserLookupMicro is the per-request Twitter user lookup API cost. + twitterUserLookupMicro int64 = 10_000 // $0.01 + // twitterPostReadMicro is the per-post Twitter read API cost. + twitterPostReadMicro int64 = 5_000 // $0.005 per post ) -// maxProfileCostMicro is the largest price getStyleProfile can ever command: -// the flat base cost plus the Twitter lookup and per-post costs at the -// post-count cap. The price advertised for getStyleProfile in tools.go must -// equal formatDollarAmount(maxProfileCostMicro) ("$0.76"). -const maxProfileCostMicro = BaseProfileCostMicro + TwitterUserLookupMicro + MaxProfileCountedPosts*TwitterPostReadMicro - // computeTwitterStyleProfileCost calculates the actual charge for a Twitter // getStyleProfile request based on the number of posts fetched and the user -// lookup, and returns it as a dollar price string (e.g. "$0.31") suitable for a +// lookup, and returns it as a dollar price string (e.g. "$0.16") suitable for a // x402 settlement override. // -// Pricing: guaranteed base cost of $0.25 (the endpoint cost for other +// Pricing: guaranteed base cost of $0.10 (the endpoint cost for other // platforms), plus user lookup $0.01, plus $0.005 per post read. For example, -// 10 posts = $0.31. Non-Twitter platforms are charged the flat $0.25 base. -// -// The price always reflects the posts actually read - it is never clamped to a -// cap (the server only ever fetches at most maxProfileCountedPosts, so the -// derived max always stays at or below formatDollarAmount(maxProfileCostMicro)). -// A PostCount of 0 means no posts were read (server-side defaults are applied -// before the profile is built, so responses carry the real count). +// 10 posts = $0.16. Non-Twitter platforms are charged the flat $0.10 base. func computeTwitterStyleProfileCost(_ *http.Request, responseText string) string { // Parse the response to get PostCount and SourcePlatform var resp GetStyleProfileResponse if err := json.Unmarshal([]byte(responseText), &resp); err != nil { - return formatDollarAmount(BaseProfileCostMicro) // fallback: base cost + return formatDollarAmount(baseProfileCostMicro) // fallback: base cost } - costMicro := BaseProfileCostMicro + costMicro := baseProfileCostMicro // Twitter additionally passes on the user lookup and per-post API costs. if resp.SourcePlatform == "Twitter" { postCount := max(resp.PostCount, 0) - costMicro += TwitterUserLookupMicro - costMicro += int64(postCount) * TwitterPostReadMicro + costMicro += twitterUserLookupMicro + costMicro += int64(postCount) * twitterPostReadMicro } return formatDollarAmount(costMicro) @@ -725,74 +638,6 @@ func formatDollarAmount(micro int64) string { return "$" + s } -// FormatPriceMicro renders an amount in micro-dollars as a plain, un-prefixed -// numeric price string ("0.76") as expected by x402 PaymentRequirements.Amount. -func FormatPriceMicro(micro int64) string { - return strings.TrimPrefix(formatDollarAmount(micro), "$") -} - -// MicroFromPriceString parses a plain dollar price string ("0.76") into -// micro-dollars so pricing arithmetic can be done without float rounding. -func MicroFromPriceString(price string) int64 { - micro, err := strconv.ParseFloat(price, 64) - if err != nil { - return 0 - } - return int64(math.Round(micro * 1_000_000)) -} - -// MaxTwitterCostMicro is the largest cost a Twitter-capable tool can command -// when it reaches its post-count cap: the flat base cost plus the Twitter -// lookup and per-post costs at the cap. It is used to advertise the upto -// maximum for tools whose actual cost is settled from the response. -func MaxTwitterCostMicro(baseMicro, postCap int64) int64 { - return baseMicro + TwitterUserLookupMicro + postCap*TwitterPostReadMicro -} - -// NewStyleProfileCostResolver returns a SettleAmountResolver that bills a -// getStyleProfile request by the posts actually read (see -// computeTwitterStyleProfileCost), rather than the advertised upto maximum. -func NewStyleProfileCostResolver() SettleAmountResolver { - return func(result *mcp.CallToolResult) (string, bool) { - if len(result.Content) != 1 { - return "", false - } - textContent, ok := result.Content[0].(*mcp.TextContent) - if !ok { - return "", false - } - return computeTwitterStyleProfileCost(nil, textContent.Text), true - } -} - -// NewTwitterCountCostResolver returns a SettleAmountResolver for tools whose -// response carries a `twitterPostCount` JSON field. It settles the flat -// baseCostMicro, plus the Twitter lookup and per-post costs when Twitter posts -// were actually used (twitterPostCount > 0). When no Twitter posts went into -// the response the caller is charged only the base cost. -func NewTwitterCountCostResolver(baseCostMicro int64) SettleAmountResolver { - return func(result *mcp.CallToolResult) (string, bool) { - if len(result.Content) != 1 { - return "", false - } - textContent, ok := result.Content[0].(*mcp.TextContent) - if !ok { - return "", false - } - var payload struct { - TwitterPostCount int `json:"twitterPostCount"` - } - if json.Unmarshal([]byte(textContent.Text), &payload) != nil { - return "", false - } - costMicro := baseCostMicro - if payload.TwitterPostCount > 0 { - costMicro += TwitterUserLookupMicro + int64(payload.TwitterPostCount)*TwitterPostReadMicro - } - return formatDollarAmount(costMicro), true - } -} - // GetStyleProfileResponse is a local copy of the response schema for cost calculation type GetStyleProfileResponse struct { Author string `json:"author"` @@ -815,13 +660,6 @@ func (c *verifyErrorCapture) WriteHeader(code int) { c.ResponseWriter.WriteHeader(code) } -func (c *verifyErrorCapture) Write(b []byte) (int, error) { - if c.statusCode == 0 { - c.statusCode = http.StatusOK - } - return c.ResponseWriter.Write(b) -} - func decodePaymentRequiredHeader(header string) (x402.PaymentRequired, error) { data, err := base64.StdEncoding.DecodeString(header) if err != nil { diff --git a/internal/x402/x402_test.go b/internal/x402/x402_test.go index f26e9ca..2598be8 100644 --- a/internal/x402/x402_test.go +++ b/internal/x402/x402_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "strconv" "strings" - "sync" "sync/atomic" "testing" @@ -52,7 +51,6 @@ func TestX402Server_HTTPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, - nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -125,7 +123,6 @@ func TestX402Server_MCPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, - nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -139,14 +136,14 @@ func TestX402Server_MCPBoundaries(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer func() { _ = serverSession.Close() }() + defer serverSession.Close() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer func() { _ = clientSession.Close() }() + defer clientSession.Close() toolsResponse, err := clientSession.ListTools(ctx, nil) if err != nil { @@ -209,7 +206,6 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, - nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -222,11 +218,10 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, - nil, x402.PaymentRequirements{ Scheme: "upto", Network: "eip155:84532", - Amount: "0.51", + Amount: "0.36", PayTo: "0x0000000000000000000000000000000000000000", }, ) @@ -235,14 +230,14 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer func() { _ = serverSession.Close() }() + defer serverSession.Close() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer func() { _ = clientSession.Close() }() + defer clientSession.Close() cases := []struct { tool string @@ -250,7 +245,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { want string }{ {tool: "getExact", scheme: "exact", want: "10000"}, - {tool: "getUpto", scheme: "upto", want: "510000"}, + {tool: "getUpto", scheme: "upto", want: "360000"}, } for _, tc := range cases { @@ -444,13 +439,11 @@ func TestComputeTwitterStyleProfileCost(t *testing.T) { postCount int want string }{ - {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.25"}, - {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.31"}, - {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.26"}, - {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.51"}, - {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.275"}, - {name: "posts at the max counted cap", platform: "Twitter", postCount: 100, want: "$0.76"}, - {name: "beyond cap is not clamped", platform: "Twitter", postCount: 200, want: "$1.26"}, + {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.10"}, + {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.16"}, + {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.11"}, + {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.36"}, + {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.125"}, } for _, tc := range cases { @@ -476,15 +469,15 @@ func TestComputeTwitterStyleProfileCostAlwaysProfitable(t *testing.T) { r, _ := http.NewRequest("POST", "/v1/getStyleProfile", nil) got := computeTwitterStyleProfileCost(r, string(payload)) - // Cut the "$" prefix and compare numerically to BaseProfileCostMicro. + // Cut the "$" prefix and compare numerically to baseProfileCostMicro. dollars := strings.TrimPrefix(got, "$") parsed, err := strconv.ParseFloat(dollars, 64) if err != nil { t.Fatalf("failed to parse %q: %v", got, err) } micro := int64(parsed * 1_000_000) - if micro <= BaseProfileCostMicro { - t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, BaseProfileCostMicro) + if micro <= baseProfileCostMicro { + t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, baseProfileCostMicro) } } } @@ -622,7 +615,6 @@ func TestAddPayableTool_EmptyRequirements(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, - nil, // No requirements provided ) } @@ -659,7 +651,6 @@ func TestX402Server_MultipleTools(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"tool": "` + name + `"}`}}, }, nil }, - nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -700,11 +691,10 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { postCount int want string }{ - {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.25"}, // not "Twitter" so flat base - {name: "empty platform", platform: "", postCount: 10, want: "$0.25"}, - {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.26"}, // should clamp to 0 - {name: "few posts", platform: "Twitter", postCount: 1, want: "$0.265"}, - {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.26"}, + {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.10"}, // not "Twitter" so flat base + {name: "empty platform", platform: "", postCount: 10, want: "$0.10"}, + {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.11"}, // should clamp to 0 + {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.11"}, } for _, tc := range cases { @@ -720,309 +710,3 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { }) } } - -// TestStyleProfileMaxPriceStaysAdvertised guards the advertised getStyleProfile -// price in internal/server/tools.go against silent drift from the x402 pricing -// model. internal/server shares the x402 constants (BaseProfileCostMicro / -// MaxProfileCountedPosts) to compute the upto maximum, so this only needs to -// catch a hand-edited Price string going out of sync. -func TestStyleProfileMaxPriceStaysAdvertised(t *testing.T) { - if got := formatDollarAmount(maxProfileCostMicro); got != "$0.76" { - t.Fatalf("maxProfileCostMicro = %d (%s), want $0.76; update getStyleProfile base Price in internal/server/tools.go if the pricing model changed", maxProfileCostMicro, got) - } -} - -// TestX402Server_MCPStyleProfileSettlementOverride drives the full MCP flow for -// the getStyleProfile upto tool and asserts the facilitator is told to settle -// the computed per-request cost (not the advertised $0.76 maximum). -func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { - ctx := context.Background() - - var mu sync.Mutex - var settleAmounts []string - mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - switch { - case r.Method == "GET" && r.URL.Path == "/supported": - _, _ = w.Write([]byte(`{ - "kinds": [ - {"x402Version": 2, "scheme": "upto", "network": "eip155:84532"} - ] - }`)) - case r.Method == "POST" && r.URL.Path == "/verify": - _, _ = w.Write([]byte(`{"isValid": true, "invalidReason": ""}`)) - case r.Method == "POST" && r.URL.Path == "/settle": - var body struct { - PaymentRequirements map[string]json.RawMessage `json:"paymentRequirements"` - } - _ = json.NewDecoder(r.Body).Decode(&body) - if raw := body.PaymentRequirements["amount"]; len(raw) > 0 { - var s string - var f float64 - mu.Lock() - if json.Unmarshal(raw, &s) == nil { - settleAmounts = append(settleAmounts, s) - } else if json.Unmarshal(raw, &f) == nil { - settleAmounts = append(settleAmounts, strconv.FormatFloat(f, 'f', -1, 64)) - } - mu.Unlock() - } - _, _ = w.Write([]byte(`{ - "success": true, - "transaction": "0x1234abcd", - "network": "eip155:84532", - "payer": "0xabcd1234" - }`)) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer mock.Close() - - serverTransport, clientTransport := mcp.NewInMemoryTransports() - - srv, err := NewX402Server( - "Test Agent API", - "1.0.0", - mock.URL, - &DummyAuthProvider{}, - X402Middlewares{}, - func(next http.Handler) http.Handler { return next }, - func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) {}, - []string{"eip155:84532"}, - ) - if err != nil { - t.Fatalf("failed to create server: %v", err) - } - - responses := []GetStyleProfileResponse{ - {Author: "@a", TopKeywords: []string{}, StyleSummary: "s", PostCount: 10, SourcePlatform: "Bluesky"}, - {Author: "@b", TopKeywords: []string{}, StyleSummary: "s", PostCount: 50, SourcePlatform: "Twitter"}, - } - var idx int - srv.AddPayableTool( - mcp.Tool{Name: "getStyleProfile", Description: "get a style profile"}, - func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { - resp := responses[idx%len(responses)] - idx++ - b, _ := json.Marshal(resp) - return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}}, nil - }, - NewStyleProfileCostResolver(), - x402.PaymentRequirements{ - Scheme: "upto", - Network: "eip155:84532", - Amount: "0.76", - PayTo: "0x0000000000000000000000000000000000000000", - }, - ) - - serverSession, err := srv.mcpServer.Connect(ctx, serverTransport, nil) - if err != nil { - t.Fatalf("failed to connect server session: %v", err) - } - defer func() { _ = serverSession.Close() }() - - client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) - clientSession, err := client.Connect(ctx, clientTransport, nil) - if err != nil { - t.Fatalf("failed to connect client session: %v", err) - } - defer func() { _ = clientSession.Close() }() - - paidCall := func() (*mcp.CallToolResult, error) { - challenge, err := clientSession.CallTool(ctx, &mcp.CallToolParams{Name: "getStyleProfile", Arguments: json.RawMessage(`{}`)}) - if err != nil { - return nil, fmt.Errorf("challenge call failed: %w", err) - } - if !challenge.IsError { - return nil, fmt.Errorf("expected a payment challenge, got %+v", challenge) - } - text, ok := challenge.Content[0].(*mcp.TextContent) - if !ok { - return nil, fmt.Errorf("unexpected challenge content type %T", challenge.Content[0]) - } - var pr types.PaymentRequired - if err := json.Unmarshal([]byte(text.Text), &pr); err != nil { - return nil, fmt.Errorf("failed to decode payment challenge: %w", err) - } - if len(pr.Accepts) == 0 { - return nil, fmt.Errorf("expected at least one accept, got %d", len(pr.Accepts)) - } - payload := types.PaymentPayload{ - X402Version: 2, - Payload: map[string]interface{}{"chainId": 84532}, - Accepted: pr.Accepts[0], - } - return clientSession.CallTool(ctx, &mcp.CallToolParams{ - Name: "getStyleProfile", - Arguments: json.RawMessage(`{}`), - Meta: mcp.Meta{PaymentMetaKey: payload}, - }) - } - - for i := 0; i < 2; i++ { - result, err := paidCall() - if err != nil { - t.Fatalf("paid call: %v", err) - } - if result.IsError { - text, _ := result.Content[0].(*mcp.TextContent) - t.Fatalf("expected successful paid call, got error result: %+v", text) - } - if resp := result.Meta[PaymentResponseMetaKey]; resp == nil { - t.Fatalf("expected %q in result meta", PaymentResponseMetaKey) - } - } - - mu.Lock() - got := append([]string(nil), settleAmounts...) - mu.Unlock() - // The "$0.25" / "$0.51" overrides resolve to 250000 / 510000 atomic units, - // with up to 1 micro tolerance for float rounding in the SDK's dollar parser. - wants := []int64{250000, 510000} - if len(got) != len(wants) { - t.Fatalf("settle amount count = %d (%v), want %d: %v", len(got), got, len(wants), wants) - } - for i, want := range wants { - v, err := strconv.ParseInt(got[i], 10, 64) - if err != nil { - t.Fatalf("settle amount[%d] = %q is not an integer: %v", i, got[i], err) - } - diff := v - want - if diff < 0 { - diff = -diff - } - if diff > 1 { - t.Errorf("settle amount[%d] = %d, want ~%d", i, v, want) - } - } -} - -func TestNewStyleProfileCostResolver(t *testing.T) { - resolver := NewStyleProfileCostResolver() - - cases := []struct { - name string - resp GetStyleProfileResponse - want string - }{ - {name: "bluesky base", resp: GetStyleProfileResponse{SourcePlatform: "Bluesky", PostCount: 10}, want: "$0.25"}, - {name: "twitter with posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 50}, want: "$0.51"}, - {name: "twitter no posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 0}, want: "$0.26"}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - payload, _ := json.Marshal(tc.resp) - result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} - got, ok := resolver(result) - if !ok { - t.Fatalf("resolver returned ok=false") - } - if got != tc.want { - t.Fatalf("resolver = %q, want %q", got, tc.want) - } - }) - } - - // Non-text content resolves to "no override"; malformed JSON falls back to - // the flat base cost via computeTwitterStyleProfileCost. - if _, ok := resolver(&mcp.CallToolResult{}); ok { - t.Fatalf("expected no override for empty content") - } - got, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}) - if !ok { - t.Fatalf("expected a base-cost fallback for malformed JSON") - } - if got != "$0.25" { - t.Fatalf("malformed JSON resolver = %q, want base fallback $0.25", got) - } -} - -// TestNewTwitterCountCostResolver verifies the actual-cost settlement for -// tools whose responses carry a `twitterPostCount` field (analytics tools, -// findSimilarUsers): flat base price when no Twitter posts went into the -// response, and base + lookup + per-post when they did. -func TestNewTwitterCountCostResolver(t *testing.T) { - const baseMicro = 150_000 // getEngagementMetrics base - resolver := NewTwitterCountCostResolver(baseMicro) - - cases := []struct { - name string - twitterCount int - want string - }{ - {name: "no twitter posts", twitterCount: 0, want: "$0.15"}, - {name: "one twitter post", twitterCount: 1, want: "$0.165"}, - {name: "ten twitter posts", twitterCount: 10, want: "$0.21"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - payload, _ := json.Marshal(map[string]any{ - "metrics": []any{}, - "twitterPostCount": tc.twitterCount, - }) - result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} - got, ok := resolver(result) - if !ok { - t.Fatalf("resolver returned ok=false") - } - if got != tc.want { - t.Fatalf("resolver = %q, want %q", got, tc.want) - } - }) - } - - if _, ok := resolver(&mcp.CallToolResult{}); ok { - t.Fatalf("expected no override for empty content") - } - if _, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}); ok { - t.Fatalf("expected no override for malformed JSON") - } -} - -// TestBuildSchemeConfigs verifies scheme servers are derived from the payment -// requirements rather than the tool name, so upto is only registered for tools -// that actually declare it. -func TestBuildSchemeConfigs(t *testing.T) { - req := func(scheme string) x402.PaymentRequirements { - return x402.PaymentRequirements{Scheme: scheme, Network: "eip155:8453", Asset: "usdc", Amount: "1", PayTo: "to"} - } - - t.Run("exact only", func(t *testing.T) { - got := buildSchemeConfigs([]string{"eip155:8453"}, req("exact")) - if len(got) != 1 || got[0].Network != x402.Network("eip155:8453") { - t.Fatalf("expected 1 exact scheme for one network, got %+v", got) - } - }) - - t.Run("upto only", func(t *testing.T) { - got := buildSchemeConfigs([]string{"eip155:8453"}, req("upto")) - if len(got) != 1 { - t.Fatalf("expected 1 upto scheme, got %d", len(got)) - } - }) - - t.Run("both schemes per network", func(t *testing.T) { - got := buildSchemeConfigs([]string{"eip155:8453", "eip155:84532"}, req("exact"), req("upto")) - if len(got) != 4 { - t.Fatalf("expected 2 schemes x 2 networks = 4, got %d", len(got)) - } - }) -} - -func TestPricingHelpers(t *testing.T) { - if got := FormatPriceMicro(MaxTwitterCostMicro(BaseProfileCostMicro, MaxProfileCountedPosts)); got != "0.76" { - t.Fatalf("style profile upto max = %q, want 0.76", got) - } - if got := FormatPriceMicro(MaxTwitterCostMicro(150_000, MaxAnalyticsCountedPosts)); got != "1.16" { - t.Fatalf("engagement metrics upto max = %q, want 1.16", got) - } - if got := MicroFromPriceString("0.25"); got != BaseProfileCostMicro { - t.Fatalf("MicroFromPriceString(0.25) = %d, want %d", got, BaseProfileCostMicro) - } - if got := MicroFromPriceString("garbage"); got != 0 { - t.Fatalf("MicroFromPriceString(garbage) = %d, want 0", got) - } -} diff --git a/mcpServer.service b/mcpServer.service index d852b7b..d9e183e 100644 --- a/mcpServer.service +++ b/mcpServer.service @@ -7,7 +7,7 @@ User=mcpserver Group=mcpserver WorkingDirectory=/home/ubuntu/gitSources/mcpServer -ExecStart=/home/ubuntu/gitSources/mcpServer/bin/mcpserver +ExecStart=/home/ubuntu/gitSources/mcpServer/bin/main RuntimeDirectory=mcpserver RuntimeDirectoryMode=0750 diff --git a/sql/schema/003_add_cache_version.sql b/sql/schema/003_add_cache_version.sql index 4fdbf1e..f5d2496 100644 --- a/sql/schema/003_add_cache_version.sql +++ b/sql/schema/003_add_cache_version.sql @@ -1,8 +1,8 @@ -- +goose up ALTER TABLE user_style_profiles - ADD COLUMN IF NOT EXISTS model_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN IF NOT EXISTS schema_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN IF NOT EXISTS algorithm_version INTEGER NOT NULL DEFAULT 1; + ADD COLUMN model_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN schema_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN algorithm_version INTEGER NOT NULL DEFAULT 1; CREATE INDEX IF NOT EXISTS idx_user_style_profiles_versions ON user_style_profiles (model_version, schema_version, algorithm_version); diff --git a/sql/schema/004_add_style_posts.sql b/sql/schema/004_add_style_posts.sql index ae5bf08..cc020c1 100644 --- a/sql/schema/004_add_style_posts.sql +++ b/sql/schema/004_add_style_posts.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS user_style_posts ( UNIQUE (user_id, platform, post_id) ); -CREATE INDEX IF NOT EXISTS idx_user_style_posts_user_platform +CREATE INDEX idx_user_style_posts_user_platform ON user_style_posts (user_id, platform); -- +goose down diff --git a/testClient/mcp/test-mcp-endpoints.go b/testClient/mcp/test-mcp-endpoints.go index 79b35cb..b6668e8 100644 --- a/testClient/mcp/test-mcp-endpoints.go +++ b/testClient/mcp/test-mcp-endpoints.go @@ -87,7 +87,7 @@ func getEnv(key, fallback string) string { } func main() { - _ = godotenv.Load() + godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080") @@ -125,7 +125,7 @@ func main() { if err != nil { log.Fatal("Failed to connect MCP client:", err) } - defer func() { _ = session.Close() }() + defer session.Close() x402Mcp := x402mcp.NewX402MCPClientFromConfig(session, []x402mcp.SchemeRegistration{ {Network: x402.Network("eip155:84532"), Client: evm.NewExactEvmScheme(signer, nil)}, diff --git a/testClient/rest/test-rest-endpoints.go b/testClient/rest/test-rest-endpoints.go index c9e9f7b..f048e29 100644 --- a/testClient/rest/test-rest-endpoints.go +++ b/testClient/rest/test-rest-endpoints.go @@ -70,7 +70,7 @@ func callEndpoint(client *http.Client, method, endpoint, body string) error { if err != nil { return fmt.Errorf("%s HTTP request failed: %w", endpoint, err) } - defer func() { _ = resp.Body.Close() }() + defer resp.Body.Close() respBody, _ := io.ReadAll(resp.Body) fmt.Printf("%s Response Status: %s\n", endpoint, resp.Status) @@ -124,7 +124,7 @@ func getEnv(key, fallback string) string { } func main() { - _ = godotenv.Load() + godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080")