From a5a4ccd9726a2e43eae864c398a6582b38bb93a5 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 00:31:59 +0000 Subject: [PATCH 01/30] chore: bump nacorid/logger and promote x/sync to direct dep --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5a95b31..13124e1 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 github.com/modelcontextprotocol/go-sdk v1.6.1 - github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 + github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 github.com/ollama/ollama v0.12.10 github.com/x402-foundation/x402/go/v2 v2.19.0 golang.org/x/time v0.14.0 @@ -28,7 +28,7 @@ require ( github.com/klauspost/compress v1.18.1 // indirect github.com/supranational/blst v0.3.16 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - golang.org/x/sync v0.19.0 // indirect + golang.org/x/sync v0.19.0 ) require ( diff --git a/go.sum b/go.sum index 9ef062a..d43b247 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7B github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80 h1:nZwGInvJNQzPSFYpstVR/TLumqBEZt0+yUXchdyBZiI= -github.com/nacorid/logger v0.0.0-20251221013547-5fc8fd94ed80/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= +github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37 h1:o7CvgRLUikfsbyuZIJTTrfRac5rMXhca0b61VLBWSIA= +github.com/nacorid/logger v0.0.0-20260907215822-b3c536ed7b37/go.mod h1:hd9h4KEPU1f/Vv2qP0x/fZUwz2clFvOV/uIF0pkGo+s= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/ollama/ollama v0.12.10 h1:Dd0/SeCc+nv+FffxmWuQTGiRreib7Gt3nBhIIFuKwZA= github.com/ollama/ollama v0.12.10/go.mod h1:RUSmYywUWx/YZMaHrqtnT1ZChu+iSz/7jx2aO9+Mgfg= From d06ade217293b56cbd46c26c61b7fe7e9a7da438 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 00:32:29 +0000 Subject: [PATCH 02/30] chore: switch golangci-lint config to v2 format --- .golangci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6d5891e --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,25 @@ +version: "2" +linters: + settings: + errcheck: + exclude-functions: + - (*net/http.Response).Body.Close + - io.Closer.Close + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ From ee2829b818ad5c0c5c8ff0657633e5a5459b102f Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:08:28 +0000 Subject: [PATCH 03/30] refactor: pointer-ify profile/post slices across backends, converters, and tests --- internal/activitypub/client_test.go | 16 +-- internal/bluesky/client.go | 8 +- internal/bluesky/client_test.go | 28 +++--- internal/server/actors.go | 10 +- internal/server/backends.go | 8 +- internal/server/cache.go | 2 +- internal/server/mcp_test.go | 42 ++++---- internal/server/posts.go | 18 ++-- internal/server/posts_convert.go | 52 +++++----- internal/server/searchactors_test.go | 12 +-- internal/server/tools.go | 2 +- internal/server/types.go | 14 +-- internal/twitter/client_test.go | 140 ++++++++++++--------------- 13 files changed, 165 insertions(+), 187 deletions(-) diff --git a/internal/activitypub/client_test.go b/internal/activitypub/client_test.go index 2fdfa11..14e786a 100644 --- a/internal/activitypub/client_test.go +++ b/internal/activitypub/client_test.go @@ -16,7 +16,7 @@ type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func newTestClient(serverURL string) *APClient { - ap, err := NewAPClient(context.Background(), "https://example.com", "tok") + ap, err := NewAPClient("https://example.com", "tok") if err != nil { panic(err) } @@ -35,10 +35,10 @@ func newTestClient(serverURL string) *APClient { } func TestNewAPClientEmpty(t *testing.T) { - if _, err := NewAPClient(context.Background(), "", "tok"); err == nil { + if _, err := NewAPClient("", "tok"); err == nil { t.Fatal("expected error for empty host") } - if _, err := NewAPClient(context.Background(), "host", ""); err == nil { + if _, err := NewAPClient("host", ""); err == nil { t.Fatal("expected error for empty apikey") } } @@ -50,7 +50,7 @@ func TestGetLatestNPosts(t *testing.T) { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) + _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -71,7 +71,7 @@ func TestGetLatestNPosts(t *testing.T) { func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/statuses/42", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) + _, _ = fmt.Fprint(w, `{"id":"42","content":"a post","text":"a post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob"}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -149,7 +149,7 @@ func TestGetActor(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}`) + _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -179,7 +179,7 @@ func TestGetActorIDFallback(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) mux.HandleFunc("/api/v1/accounts/alice", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) + _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -201,7 +201,7 @@ func TestSearchActors(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) }) server := httptest.NewServer(mux) defer server.Close() diff --git a/internal/bluesky/client.go b/internal/bluesky/client.go index 2ddc4bf..ba542d9 100644 --- a/internal/bluesky/client.go +++ b/internal/bluesky/client.go @@ -243,17 +243,17 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro return result, err } -func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - var result []actor.Profile +func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { + var result []*actor.Profile err := c.withRefresh(ctx, func() error { params := map[string]any{"q": query, "limit": limit} var out bskyActorSearch if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { return fmt.Errorf("failed to search actors: %w", err) } - results := make([]actor.Profile, 0, len(out.Actors)) + results := make([]*actor.Profile, 0, len(out.Actors)) for _, a := range out.Actors { - results = append(results, actor.Profile{ + results = append(results, &actor.Profile{ Platform: "Bluesky", Handle: a.Handle, DisplayName: a.DisplayName, diff --git a/internal/bluesky/client_test.go b/internal/bluesky/client_test.go index 73e9977..f2b39a5 100644 --- a/internal/bluesky/client_test.go +++ b/internal/bluesky/client_test.go @@ -31,7 +31,7 @@ func setupMockServer() *httptest.Server { if input.Identifier == "fail.test" { w.WriteHeader(http.StatusUnauthorized) - fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) + _, _ = fmt.Fprintln(w, `{"error": "InvalidCredentials", "message": "Invalid handle or password"}`) return } @@ -43,7 +43,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:test", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetAuthorFeed @@ -81,7 +81,7 @@ func setupMockServer() *httptest.Server { Feed: feed, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetFeed @@ -107,7 +107,7 @@ func setupMockServer() *httptest.Server { }, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for bsky.FeedGetPosts (used by GetPost) @@ -140,7 +140,7 @@ func setupMockServer() *httptest.Server { Posts: posts, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.getProfile (used by GetActor) @@ -163,7 +163,7 @@ func setupMockServer() *httptest.Server { "url": "https://bsky.app/profile/" + actor, } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for app.bsky.actor.searchActors (used by SearchActors) @@ -194,7 +194,7 @@ func setupMockServer() *httptest.Server { resp := map[string]any{"actors": actors} w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.identity.resolveHandle (used by ResolveUser) @@ -209,7 +209,7 @@ func setupMockServer() *httptest.Server { Did: "did:plc:resolved", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) // Mock for com.atproto.server.refreshSession (used by withRefresh) @@ -218,7 +218,7 @@ func setupMockServer() *httptest.Server { auth := r.Header.Get("Authorization") if auth == "" { w.WriteHeader(http.StatusUnauthorized) - json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "TokenRequired"}) return } @@ -227,7 +227,7 @@ func setupMockServer() *httptest.Server { RefreshJwt: "refreshed_refresh_jwt", } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(resp) + _ = json.NewEncoder(w).Encode(resp) }) return httptest.NewServer(mux) @@ -302,7 +302,7 @@ func TestGetLatestNPosts(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() actor := "did:plc:testactor" - var limit int = 5 + limit := 5 output, err := client.GetLatestNPosts(ctx, actor, limit) if err != nil { @@ -329,7 +329,7 @@ func TestGetFeed(t *testing.T) { client := newTestClient(t, server.URL) ctx := context.Background() feedURI := "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.generator/with-friends" - var limit int = 10 + limit := 10 output, err := client.GetFeed(ctx, feedURI, limit) if err != nil { @@ -391,7 +391,7 @@ func TestGetPostsInTimeframe(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/xrpc/com.atproto.server.createSession", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) + _ = json.NewEncoder(w).Encode(atproto.ServerCreateSession_Output{Handle: "test.handle", Did: "did:plc:test"}) }) mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { requests++ @@ -428,7 +428,7 @@ func TestGetPostsInTimeframe(t *testing.T) { } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed, Cursor: &cursor}) }) server := httptest.NewServer(mux) diff --git a/internal/server/actors.go b/internal/server/actors.go index 724285c..205c8fa 100644 --- a/internal/server/actors.go +++ b/internal/server/actors.go @@ -7,16 +7,16 @@ import ( "sync" ) -func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]ActorProfile, error) { +func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ([]*ActorProfile, error) { var ( mu sync.Mutex - results []ActorProfile + results []*ActorProfile wg sync.WaitGroup errMu sync.Mutex errs []error success int ) - call := func(fn func(context.Context, string, int) ([]ActorProfile, error)) { + call := func(fn func(context.Context, string, int) ([]*ActorProfile, error)) { defer wg.Done() r, err := fn(ctx, query, limit) if err != nil { @@ -44,9 +44,9 @@ func (s *Server) searchActorsAll(ctx context.Context, query string, limit int) ( // dedupeActors removes near-duplicate profiles across platforms. Two results are considered // the same person when they share a normalized handle (the '@' and any instance/host stripped, // lower-cased) or a normalized display name. The first occurrence wins. -func dedupeActors(in []ActorProfile) []ActorProfile { +func dedupeActors(in []*ActorProfile) []*ActorProfile { seen := make(map[string]struct{}, len(in)) - out := make([]ActorProfile, 0, len(in)) + out := make([]*ActorProfile, 0, len(in)) for _, p := range in { key := normalizeHandle(p.Handle) if key == "" { diff --git a/internal/server/backends.go b/internal/server/backends.go index 8bd39a6..3182a3b 100644 --- a/internal/server/backends.go +++ b/internal/server/backends.go @@ -26,15 +26,15 @@ type blueskyBackend interface { GetFeed(ctx context.Context, feedURI string, limit int) (*bsky.FeedGetFeed_Output, error) ResolveUser(ctx context.Context, user string) (syntax.DID, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type apBackend interface { GetPost(ctx context.Context, postID string) (*activitypub.Post, error) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*activitypub.Post, error) - GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) + GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type nostrBackend interface { @@ -42,7 +42,7 @@ type nostrBackend interface { GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*nostr.Post, error) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*nostr.Post, error) GetActor(ctx context.Context, author string) (*actor.Profile, error) - SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) + SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) } type twitterBackend interface { diff --git a/internal/server/cache.go b/internal/server/cache.go index a9b5869..82c58ce 100644 --- a/internal/server/cache.go +++ b/internal/server/cache.go @@ -64,6 +64,6 @@ func cachedFetch[T any](c *ttlCache, key string, fetch func() (T, error)) (T, er } type latestCacheValue struct { - posts []GetPostResponsePost + posts []*GetPostResponsePost aid string } diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 9e5a1df..85cac53 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -27,7 +27,7 @@ type fakeBluesky struct { postView *bsky.FeedDefs_PostView feedViews []*bsky.FeedDefs_FeedViewPost profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile feedOut *bsky.FeedGetFeed_Output err error calls int @@ -76,7 +76,7 @@ func (f *fakeBluesky) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeBluesky) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -89,7 +89,7 @@ type fakeAP struct { post *activitypub.Post posts []*activitypub.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int lastLimit int @@ -109,15 +109,13 @@ func (f *fakeAP) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } return f.posts, nil } -func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]activitypub.Post, error) { +func (f *fakeAP) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*activitypub.Post, error) { f.calls++ if f.err != nil { return nil, f.err } - out := make([]activitypub.Post, 0, len(f.posts)) - for _, p := range f.posts { - out = append(out, *p) - } + out := make([]*activitypub.Post, 0, len(f.posts)) + out = append(out, f.posts...) return out, nil } func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -127,7 +125,7 @@ func (f *fakeAP) GetActor(ctx context.Context, author string) (*actor.Profile, e } return f.profile, nil } -func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeAP) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -140,7 +138,7 @@ type fakeNostr struct { post *nostr.Post posts []*nostr.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int lastLimit int @@ -174,7 +172,7 @@ func (f *fakeNostr) GetActor(ctx context.Context, author string) (*actor.Profile } return f.profile, nil } -func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeNostr) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ f.lastLimit = limit if f.err != nil { @@ -187,7 +185,7 @@ type fakeTwitter struct { post *twitter.Post posts []*twitter.Post profile *actor.Profile - actors []actor.Profile + actors []*actor.Profile err error calls int } @@ -220,7 +218,7 @@ func (f *fakeTwitter) GetActor(ctx context.Context, author string) (*actor.Profi } return f.profile, nil } -func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { +func (f *fakeTwitter) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { f.calls++ if f.err != nil { return nil, f.err @@ -394,10 +392,10 @@ func TestGetActorDispatch(t *testing.T) { } func TestSearchActorsAllSucceed(t *testing.T) { - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice", Limit: 10}) if err != nil { @@ -425,8 +423,8 @@ func TestSearchActorsAllFail(t *testing.T) { func TestSearchActorsPartialSuccess(t *testing.T) { bsky := &fakeBluesky{err: fmt.Errorf("boom")} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} tw := &fakeTwitter{err: fmt.Errorf("boom")} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "alice"}) @@ -740,9 +738,9 @@ func TestGetActorTwitter(t *testing.T) { // ---- searchActors edge cases ---- func TestSearchActorsLimitClamp(t *testing.T) { - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, &fakeTwitter{}) // Limit > 50 should be clamped to 50 in the backend call. diff --git a/internal/server/posts.go b/internal/server/posts.go index eed3144..cd2bd28 100644 --- a/internal/server/posts.go +++ b/internal/server/posts.go @@ -16,34 +16,34 @@ func (s *Server) fetchSinglePost(ctx context.Context, platform platform, author, return nil, err } r := bskyPostViewToResponse(res) - return &r, nil + return r, nil case ActivityPub: p, err := s.apClient.GetPost(ctx, recordKey) if err != nil { return nil, err } - r := apPostToResponse(*p) - return &r, nil + r := apPostToResponse(p) + return r, nil case Nostr: p, err := s.nostrClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := nostrPostToResponse(p) - return &r, nil + return r, nil case Twitter: p, err := s.twitterClient.GetPost(ctx, recordKey) if err != nil { return nil, err } r := twitterPostToResponse(p) - return &r, nil + return r, nil } return nil, fmt.Errorf("unsupported platform %q", platform) }) } -func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]GetPostResponsePost, string, error) { +func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author string, limit int) ([]*GetPostResponsePost, string, error) { key := fmt.Sprintf("latest:%s|%s|%d", platform, author, limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -84,7 +84,7 @@ func (s *Server) fetchLatestPosts(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]GetPostResponsePost, string, error) { +func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]*GetPostResponsePost, string, error) { key := fmt.Sprintf("time:%s|%s|%s|%s|%d", platform, author, start.Format(time.RFC3339), end.Format(time.RFC3339), limit) cv, err := cachedFetch(s.postCache, key, func() (latestCacheValue, error) { switch platform { @@ -125,9 +125,9 @@ func (s *Server) fetchPostsInTime(ctx context.Context, platform platform, author return cv.posts, cv.aid, nil } -func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]GetPostResponsePost, error) { +func (s *Server) fetchFeed(ctx context.Context, platform platform, uri string) ([]*GetPostResponsePost, error) { key := fmt.Sprintf("feed:%s|%s", platform, uri) - return cachedFetch(s.postCache, key, func() ([]GetPostResponsePost, error) { + return cachedFetch(s.postCache, key, func() ([]*GetPostResponsePost, error) { if platform == Bluesky { res, err := s.bskyClient.GetFeed(ctx, uri, 50) if err != nil { diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index 181c9fc..2475c10 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -10,7 +10,7 @@ import ( "github.com/nacorid/naco-api/internal/twitter" ) -func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { +func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) *GetPostResponsePost { author := "" if p.Author != nil { author = p.Author.Did @@ -34,7 +34,7 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { if p.ReplyCount != nil { replies = *p.ReplyCount } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: author, Content: content, Likes: likes, @@ -45,12 +45,12 @@ func bskyPostViewToResponse(p *bsky.FeedDefs_PostView) GetPostResponsePost { } } -func apPostToResponse(p activitypub.Post) GetPostResponsePost { +func apPostToResponse(p *activitypub.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.FavouritesCount), @@ -60,24 +60,24 @@ func apPostToResponse(p activitypub.Post) GetPostResponsePost { } } -func nostrPostToResponse(p *nostr.Post) GetPostResponsePost { +func nostrPostToResponse(p *nostr.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, CreatedAt: created, } } -func twitterPostToResponse(p *twitter.Post) GetPostResponsePost { +func twitterPostToResponse(p *twitter.Post) *GetPostResponsePost { created := "" if !p.CreatedAt.IsZero() { created = p.CreatedAt.Format(time.RFC3339) } - return GetPostResponsePost{ + return &GetPostResponsePost{ Author: p.Author, Content: p.Content, Likes: int64(p.Likes), @@ -90,8 +90,8 @@ func twitterPostToResponse(p *twitter.Post) GetPostResponsePost { // ---- Batch converters for GetPostResponsePost ---- -func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]GetPostResponsePost, string) { - posts := make([]GetPostResponsePost, 0, len(views)) +func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]*GetPostResponsePost, string) { + posts := make([]*GetPostResponsePost, 0, len(views)) var aid string for _, p := range views { if p.Post == nil { @@ -105,18 +105,8 @@ func blueskyFeedViewsToResponses(views []*bsky.FeedDefs_FeedViewPost) ([]GetPost return posts, aid } -func apPostsPtrToResponses(posts []*activitypub.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) - var aid string - for _, p := range posts { - out = append(out, apPostToResponse(*p)) - aid = p.Author - } - return out, aid -} - -func apPostsToResponses(posts []activitypub.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func apPostsPtrToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, apPostToResponse(p)) @@ -125,8 +115,18 @@ func apPostsToResponses(posts []activitypub.Post) ([]GetPostResponsePost, string return out, aid } -func nostrPostsToResponses(posts []*nostr.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func apPostsToResponses(posts []*activitypub.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) + var aid string + for _, p := range posts { + out = append(out, apPostToResponse(p)) + aid = p.Author + } + return out, aid +} + +func nostrPostsToResponses(posts []*nostr.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, nostrPostToResponse(p)) @@ -135,8 +135,8 @@ func nostrPostsToResponses(posts []*nostr.Post) ([]GetPostResponsePost, string) return out, aid } -func twitterPostsToResponses(posts []*twitter.Post) ([]GetPostResponsePost, string) { - out := make([]GetPostResponsePost, 0, len(posts)) +func twitterPostsToResponses(posts []*twitter.Post) ([]*GetPostResponsePost, string) { + out := make([]*GetPostResponsePost, 0, len(posts)) var aid string for _, p := range posts { out = append(out, twitterPostToResponse(p)) diff --git a/internal/server/searchactors_test.go b/internal/server/searchactors_test.go index dcab2f3..61fa8c6 100644 --- a/internal/server/searchactors_test.go +++ b/internal/server/searchactors_test.go @@ -29,7 +29,7 @@ func TestNormalizeHandle(t *testing.T) { } func TestDedupeActors(t *testing.T) { - in := []ActorProfile{ + in := []*ActorProfile{ {Handle: "@alice", DisplayName: "Alice"}, {Handle: "alice@mastodon.social", DisplayName: "Alice"}, {Handle: "", DisplayName: ""}, @@ -45,7 +45,7 @@ func TestDedupeActors(t *testing.T) { func TestDedupeActorsByDisplayNameOnly(t *testing.T) { // When handles are empty but display names collide, dedupe by display name. - in := []ActorProfile{ + in := []*ActorProfile{ {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Shared"}, {Handle: "", DisplayName: "Other"}, @@ -59,10 +59,10 @@ func TestDedupeActorsByDisplayNameOnly(t *testing.T) { func TestSearchActorsTwitterExcluded(t *testing.T) { // searchActorsAll only queries Bluesky, ActivityPub and Nostr. A Twitter // result must never appear because the backend is not consulted. - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "t"}}, calls: 0} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "q"}) diff --git a/internal/server/tools.go b/internal/server/tools.go index 510aef4..ebf7d22 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -333,7 +333,7 @@ func (s *Server) getPostsInTime(ctx context.Context, request *mcp.CallToolReques if authorID == "" { authorID = args.Author } - resp := &GetPostsInTimeResponse{ + resp := GetPostsInTimeResponse{ Author: authorID, Posts: posts, } diff --git a/internal/server/types.go b/internal/server/types.go index b1915b7..4e1733e 100644 --- a/internal/server/types.go +++ b/internal/server/types.go @@ -26,8 +26,8 @@ type GetLatestPostsRequest struct { } type GetLatestPostsResponse struct { - Author string `json:"actor" jsonschema_description:"Did of the actor"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` + Author string `json:"actor" jsonschema_description:"Did of the actor"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of latest posts"` } type GetPostsInTimeRequest struct { @@ -38,8 +38,8 @@ type GetPostsInTimeRequest struct { } type GetPostsInTimeResponse struct { - Author string `json:"author" jsonschema_description:"Did of the author"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` + Author string `json:"author" jsonschema_description:"Did of the author"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the timeframe"` } type GetPostRequest struct { @@ -63,8 +63,8 @@ type GetFeedRequest struct { } type GetFeedResponse struct { - URI string `json:"uri" jsonschema_description:"The URI of the feed"` - Posts []GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` + URI string `json:"uri" jsonschema_description:"The URI of the feed"` + Posts []*GetPostResponsePost `json:"posts" jsonschema_description:"List of posts in the feed"` } type GetStyleProfileRequest struct { @@ -105,7 +105,7 @@ type SearchActorsRequest struct { } type SearchActorsResponse struct { - Results []ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` + Results []*ActorProfile `json:"results" jsonschema_description:"Ranked list of matching actor profiles across platforms"` } type SimilarUser struct { diff --git a/internal/twitter/client_test.go b/internal/twitter/client_test.go index 9dca301..ebb687d 100644 --- a/internal/twitter/client_test.go +++ b/internal/twitter/client_test.go @@ -2,14 +2,22 @@ package twitter import ( "context" + "errors" "fmt" + "io" "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" ) +// Speed up test suite: override defaultBackoff to 1ms so retries are instantaneous. +func init() { + defaultBackoff = 1 * time.Millisecond +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } @@ -46,7 +54,8 @@ func TestGetLatestNPosts(t *testing.T) { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -54,7 +63,8 @@ func TestGetLatestNPosts(t *testing.T) { return } tweetsURL = r.URL.String() - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello world","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -78,13 +88,15 @@ func TestGetLatestNPosts(t *testing.T) { func TestGetLatestNPostsLimitClamp(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"1","username":"u"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) }) mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { if got := r.URL.Query().Get("max_results"); got != "5" { t.Fatalf("expected clamped max_results=5 for limit=3, got %q", got) } - fmt.Fprint(w, `{"data":[]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -110,14 +122,16 @@ func TestGetLatestNPostsHTTPError(t *testing.T) { func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/tweets/999", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { w.WriteHeader(http.StatusUnauthorized) return } - fmt.Fprint(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"999","text":"a single tweet","created_at":"2024-02-02T00:00:00Z","public_metrics":{"retweet_count":2,"reply_count":1,"like_count":5,"quote_count":0}},"includes":{"users":[{"username":"testuser"}]}}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -138,7 +152,8 @@ func TestGetPost(t *testing.T) { func TestGetPostsInTimeframe(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":{"id":"12345","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer test-token" { @@ -148,7 +163,8 @@ func TestGetPostsInTimeframe(t *testing.T) { if r.URL.Query().Get("start_time") == "" || r.URL.Query().Get("end_time") == "" { t.Fatalf("expected start_time and end_time query params") } - fmt.Fprint(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"timed post","created_at":"2024-03-03T00:00:00Z","public_metrics":{"retweet_count":0,"reply_count":0,"like_count":3,"quote_count":0}}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -174,54 +190,19 @@ func TestRateLimitRetryWithResetHeader(t *testing.T) { mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ if callCount == 1 { - // First call: rate limited with reset header + // First call: rate limited with reset header slightly in the past/present w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(100*time.Millisecond).Unix())) + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Unix())) w.WriteHeader(http.StatusTooManyRequests) return } // Second call: success - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) }) mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) - }) - server := httptest.NewServer(mux) - defer server.Close() - - c := newTestClient(server.URL) - // Use a short context timeout to avoid waiting too long - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - posts, err := c.GetLatestNPosts(ctx, "@testuser", 10) - if err != nil { - t.Fatalf("GetLatestNPosts error after retry: %v", err) - } - if callCount != 2 { - t.Fatalf("expected 2 calls (1 retry), got %d", callCount) - } - if len(posts) != 1 { - t.Fatalf("expected 1 post after retry, got %d", len(posts)) - } -} - -func TestRateLimitRetryWithRetryAfterHeader(t *testing.T) { - callCount := 0 - mux := http.NewServeMux() - mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - callCount++ - if callCount == 1 { - // First call: rate limited with Retry-After header - w.Header().Set("Retry-After", "1") // 1 second - w.WriteHeader(http.StatusTooManyRequests) - return - } - // Second call: success - fmt.Fprint(w, `{"data":{"id":"12345","name":"Test","username":"testuser"}}`) - }) - mux.HandleFunc("/2/users/12345/tweets", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"hello","created_at":"2024-01-01T00:00:00Z"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -247,29 +228,25 @@ func TestRateLimitExhaustedAfterMaxRetries(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { callCount++ - // Always rate limited - no reset header, no Retry-After, so exponential backoff is used w.Header().Set("x-rate-limit-remaining", "0") - // Don't set x-rate-limit-reset or Retry-After to force exponential backoff w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Use a timeout long enough for all retries (4 retries with exponential backoff: 500ms, 1s, 2s, 4s = ~7.5s) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error after max retries exhausted") } - // Should have tried maxRetries (4) + 1 initial = 5 times if callCount < 5 { t.Fatalf("expected at least 5 attempts (1 initial + 4 retries), got %d", callCount) } - if !strings.Contains(err.Error(), "rate limited after retries") { - t.Fatalf("expected 'rate limited after retries' error, got %v", err) + if !strings.Contains(err.Error(), "rate limit") { + t.Fatalf("expected rate limit error, got %v", err) } } @@ -277,7 +254,8 @@ func TestUserLookupNotFound(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - fmt.Fprint(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"errors":[{"title":"Not Found","detail":"User not found"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -296,7 +274,8 @@ func TestUserLookupUnauthorized(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) - fmt.Fprint(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"errors":[{"title":"Unauthorized","detail":"Invalid token"}]}`) }) server := httptest.NewServer(mux) defer server.Close() @@ -314,65 +293,66 @@ func TestUserLookupUnauthorized(t *testing.T) { func TestUserLookupRateLimited(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "1") w.WriteHeader(http.StatusTooManyRequests) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Need enough time for max retries (4 retries * 1 second = 4 seconds) + initial - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() _, err := c.GetLatestNPosts(ctx, "@testuser", 10) if err == nil { t.Fatal("expected error for rate limited user lookup") } - if !strings.Contains(err.Error(), "twitter error") && !strings.Contains(err.Error(), "rate limited after retries") { - t.Fatalf("expected twitter error or rate limited after retries, got %v", err) + if !strings.Contains(err.Error(), "rate limit") { + t.Fatalf("expected rate limit error, got %v", err) } } func TestContextCancellationDuringRetry(t *testing.T) { - callCount := 0 + reqReceived := make(chan struct{}) + var once sync.Once + mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { - callCount++ - // First call: rate limited with long reset time - if callCount == 1 { - w.Header().Set("x-rate-limit-remaining", "0") - w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) - w.WriteHeader(http.StatusTooManyRequests) - return - } + w.Header().Set("x-rate-limit-remaining", "0") + w.Header().Set("x-rate-limit-reset", fmt.Sprintf("%d", time.Now().Add(10*time.Second).Unix())) + w.WriteHeader(http.StatusTooManyRequests) + once.Do(func() { + close(reqReceived) // Signal test that the 429 response was returned + }) }) server := httptest.NewServer(mux) defer server.Close() c := newTestClient(server.URL) - // Cancel context immediately after first call ctx, cancel := context.WithCancel(context.Background()) - // Start the request in a goroutine + done := make(chan error, 1) go func() { _, err := c.GetLatestNPosts(ctx, "@testuser", 10) done <- err }() - // Give it a moment to make the first call, then cancel - time.Sleep(100 * time.Millisecond) - cancel() + // Deterministic wait: block until the mock server receives the request, then cancel immediately + select { + case <-reqReceived: + cancel() + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for mock request to be hit") + } select { case err := <-done: if err == nil { t.Fatal("expected error from context cancellation") } - if !strings.Contains(err.Error(), "context canceled") && !strings.Contains(err.Error(), "context cancelled") { - t.Fatalf("expected context cancellation error, got %v", err) + if !errors.Is(err, context.Canceled) && !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("expected context.Canceled, got %v", err) } case <-time.After(2 * time.Second): - t.Fatal("test timed out waiting for cancellation") + t.Fatal("test timed out waiting for goroutine to abort") } } From 3b443eff3689a2baf0280ee7b341ac3b663bc2c0 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:11:13 +0000 Subject: [PATCH 04/30] refactor: harden HTTP clients with timeouts, per-relay ctx, rate-limit handling --- cmd/check-facilitator/main.go | 19 +- cmd/server/main.go | 139 ++++++++++----- internal/activitypub/client.go | 208 +++++++--------------- internal/config/config.go | 14 +- internal/nostr/client.go | 307 +++++++++++++++++++-------------- internal/twitter/client.go | 144 +++++++++------- 6 files changed, 439 insertions(+), 392 deletions(-) diff --git a/cmd/check-facilitator/main.go b/cmd/check-facilitator/main.go index db8aa29..717cf00 100644 --- a/cmd/check-facilitator/main.go +++ b/cmd/check-facilitator/main.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/nacorid/naco-api/internal/server" ) @@ -116,18 +117,22 @@ func main() { os.Exit(1) } - resp, err := http.DefaultClient.Do(req) + httpClient := &http.Client{ + Timeout: 30 * time.Second, + } + resp, err := httpClient.Do(req) if err != nil { fmt.Fprintf(os.Stderr, "Request failed: %v\n", err) os.Exit(1) } defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - fmt.Printf("Status: %d\n", resp.StatusCode) - fmt.Printf("Response: %s\n", string(body)) - - if resp.StatusCode != http.StatusOK { - os.Exit(1) + maxBytes := int64(5 * 1024 * 1024) + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes)) + if err != nil { + fmt.Printf("failed reading response body: %v", err) } + + fmt.Printf("Status: %d\n", resp.StatusCode) + fmt.Printf("Response: %s\n", body) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 300f84c..7826b34 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,9 @@ package main import ( "context" "database/sql" + "errors" + "fmt" + "io" "net/http" "net/url" "os" @@ -20,25 +23,40 @@ import ( "github.com/nacorid/naco-api/internal/server" "github.com/nacorid/naco-api/internal/store" "github.com/nacorid/naco-api/internal/twitter" + "golang.org/x/sync/errgroup" ) func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() cfg := config.Load() - log.Init(log.Config{ + if err := log.Init(log.Config{ LogFilePath: cfg.LogFile, ConsoleLevel: log.LevelDebug, FileLevel: log.LevelTrace, - }) + }); err != nil { + log.Fatalf("Failed to init logger: %v", err) + } defer log.Close() + conn, err := sql.Open("postgres", cfg.DatabaseURL) if err != nil { - log.Fatalf("Failed to connect to database: %v", err) + log.Fatalf("Failed to initialize database connection: %v", err) } defer conn.Close() + + conn.SetMaxOpenConns(10) + conn.SetMaxIdleConns(10) + conn.SetConnMaxLifetime(5 * time.Minute) + + dbCtx, dbCancel := context.WithTimeout(ctx, 5*time.Second) + err = conn.PingContext(dbCtx) + dbCancel() + if err != nil { + log.Fatalf("Failed to ping database: %v", err) + } db := store.New(conn) if cfg.MigrateOnStart { @@ -62,7 +80,7 @@ func main() { } ollamaClient := ollama.New(ctx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) - apClient, err := activitypub.NewAPClient(ctx, cfg.ActivityPubHost, cfg.ActivityPubToken) + apClient, err := activitypub.NewAPClient(cfg.ActivityPubHost, cfg.ActivityPubToken) if err != nil { log.Fatal("Failed to create activitypub client", "error", err) } @@ -78,48 +96,79 @@ func main() { srv.StartPrewarm(ctx) - go func() { - mux := http.NewServeMux() - mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { - if err := conn.PingContext(r.Context()); err != nil { - http.Error(w, "database unavailable", http.StatusServiceUnavailable) - return - } - req, _ := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) - resp, err := http.DefaultClient.Do(req) - if err != nil || resp.StatusCode != http.StatusOK { - if resp != nil { - resp.Body.Close() - } - http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) - return - } - resp.Body.Close() - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - }) - mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") - w.WriteHeader(http.StatusOK) - w.Write([]byte(srv.MetricsText())) - }) - addr := ":" + cfg.ReadyzPort - log.Info("Readiness server starting", "address", addr) - if err := http.ListenAndServe(addr, mux); err != nil { - log.Error("Readiness server error", "error", err) - } - }() + g, gCtx := errgroup.WithContext(ctx) + startReadyzServer(ctx, g, cfg, conn) - go func() { + g.Go(func() error { log.Info("Server starting", "address", cfg.ListenAddress) - if err := srv.Start(); err != http.ErrServerClosed { - log.Fatal("HTTP server error", "error", err) + if err := srv.Start(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("main server error: %w", err) } - }() + return nil + }) - stopChan := make(chan os.Signal, 1) - signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM) - <-stopChan + g.Go(func() error { + <-gCtx.Done() + log.Info("Shutting down server...") + return nil + }) - log.Info("Shutting down server...") + if err := g.Wait(); err != nil && !errors.Is(err, context.Canceled) { + log.Error("Application terminated with error", "error", err) + } + + log.Info("Application shutdown complete") +} + +func startReadyzServer(ctx context.Context, g *errgroup.Group, cfg *config.Config, conn *sql.DB) { + mux := http.NewServeMux() + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + if err := conn.PingContext(r.Context()); err != nil { + http.Error(w, "database unavailable", http.StatusServiceUnavailable) + return + } + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, cfg.OllamaHost+"/api/tags", nil) + if err != nil { + http.Error(w, "invalid ollama configuration", http.StatusInternalServerError) + return + } + + resp, err := client.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) + return + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + _, _ = w.Write([]byte("ok")) + }) + + readyServer := &http.Server{ + Addr: ":" + cfg.ReadyzPort, + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + } + + g.Go(func() error { + log.Info("Readiness server starting", "address", readyServer.Addr) + if err := readyServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("readiness server failed: %w", err) + } + return nil + }) + + g.Go(func() error { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return readyServer.Shutdown(shutdownCtx) + }) } diff --git a/internal/activitypub/client.go b/internal/activitypub/client.go index cc45f07..2582f68 100644 --- a/internal/activitypub/client.go +++ b/internal/activitypub/client.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/url" + "strings" "time" "github.com/nacorid/naco-api/internal/actor" @@ -18,66 +19,59 @@ type APClient struct { bearerToken string } -func NewAPClient(ctx context.Context, host, apikey string) (*APClient, error) { +func NewAPClient(host, apikey string) (*APClient, error) { if host == "" || apikey == "" { return nil, fmt.Errorf("host and apikey must not be empty") } + return &APClient{ - client: http.DefaultClient, - host: host, - bearerToken: fmt.Sprintf("Bearer %s", apikey), + client: &http.Client{Timeout: 30 * time.Second}, + host: strings.TrimRight(host, "/"), + bearerToken: "Bearer " + apikey, }, nil } -func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { - url := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, postID) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) +func (ap *APClient) get(ctx context.Context, endpoint string, target any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Authorization", ap.bearerToken) + resp, err := ap.client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to get post: %w", err) + return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get post: %s", resp.Status) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + + if err := json.NewDecoder(resp.Body).Decode(target); err != nil { + return fmt.Errorf("failed to decode response: %w", err) } - var internal = &internalPost{} - if err = json.Unmarshal(body, internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal json response: %w", err) + + return nil +} + +func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { + endpoint := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, url.PathEscape(postID)) + var internal internalPost + if err := ap.get(ctx, endpoint, &internal); err != nil { + return nil, fmt.Errorf("failed to get post: %w", err) } return internal.toPost(), nil } -func (ap *APClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { - url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, actor, limit) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { +func (ap *APClient) GetLatestNPosts(ctx context.Context, actorID string, limit int) ([]*Post, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(actorID), limit) + var internal []*internalPost + if err := ap.get(ctx, endpoint, &internal); err != nil { return nil, fmt.Errorf("failed to get latest posts: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get latest posts: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var internal []*internalPost - if err = json.Unmarshal(body, &internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal json response: %w", err) - } + posts := make([]*Post, len(internal)) for i, p := range internal { posts[i] = p.toPost() @@ -85,8 +79,8 @@ func (ap *APClient) GetLatestNPosts(ctx context.Context, actor string, limit int return posts, nil } -func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]Post, error) { - var posts []Post +func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, start, end time.Time, limit int) ([]*Post, error) { + var posts []*Post var maxID string for { @@ -97,37 +91,14 @@ func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actor string, start break } - url := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=50", ap.host, actor) + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=50", ap.host, url.PathEscape(actorID)) if maxID != "" { - url += fmt.Sprintf("&max_id=%s", maxID) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Authorization", ap.bearerToken) - - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get posts: %w", err) - } - // !!IMPORTANT!! Manually close body instead of deferring, due to loop - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("failed to get posts: %s", resp.Status) - } - - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + endpoint += "&max_id=" + url.QueryEscape(maxID) } var internal []*internalPost - if err := json.Unmarshal(body, &internal); err != nil { - return nil, fmt.Errorf("failed to unmarshal response body: %w", err) + if err := ap.get(ctx, endpoint, &internal); err != nil { + return nil, fmt.Errorf("failed to get posts: %w", err) } if len(internal) == 0 { @@ -138,58 +109,53 @@ func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actor string, start if i.CreatedAt.After(end) { continue } - if i.CreatedAt.Before(start) { return posts, nil } - - posts = append(posts, *i.toPost()) - + posts = append(posts, i.toPost()) if len(posts) >= limit { return posts, nil } } - maxID = internal[len(internal)-1].ID + newMaxID := internal[len(internal)-1].ID + if newMaxID == maxID { + break + } + maxID = newMaxID } return posts, nil } -func (ap *APClient) GetActor(ctx context.Context, actor string) (*actor.Profile, error) { - // Mastodon exposes actor lookup by acct; fall back to a direct id lookup. - prof, err := ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actor))) - if err != nil { - prof, err = ap.fetchAccount(ctx, fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actor))) - if err != nil { - return nil, err +func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { + var acc Account + endpoint := fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actorHandle)) + + if err := ap.get(ctx, endpoint, &acc); err != nil { + fallback := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actorHandle)) + if fallbackErr := ap.get(ctx, fallback, &acc); fallbackErr != nil { + return nil, fmt.Errorf("actor lookup failed: %w (fallback: %v)", err, fallbackErr) } } - return prof, nil + return ap.toProfile(&acc), nil } -func (ap *APClient) fetchAccount(ctx context.Context, endpoint string) (*actor.Profile, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) +func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&limit=%d", ap.host, url.QueryEscape(query), limit) + var accounts []*Account + if err := ap.get(ctx, endpoint, &accounts); err != nil { + return nil, fmt.Errorf("failed to search actors: %w", err) } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get actor: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to get actor: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var acc Account - if err := json.Unmarshal(body, &acc); err != nil { - return nil, fmt.Errorf("failed to unmarshal account: %w", err) + + results := make([]*actor.Profile, len(accounts)) + for i, acc := range accounts { + results[i] = ap.toProfile(acc) } + return results, nil +} + +func (ap *APClient) toProfile(acc *Account) *actor.Profile { return &actor.Profile{ Platform: "ActivityPub", Handle: acc.Acct, @@ -200,45 +166,5 @@ func (ap *APClient) fetchAccount(ctx context.Context, endpoint string) (*actor.P FollowingCount: acc.FollowingCount, PostCount: acc.StatusesCount, URL: acc.URL, - }, nil -} - -func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&limit=%d", ap.host, url.QueryEscape(query), limit) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Authorization", ap.bearerToken) - resp, err := ap.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to search actors: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to search actors: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var accounts []Account - if err := json.Unmarshal(body, &accounts); err != nil { - return nil, fmt.Errorf("failed to unmarshal accounts: %w", err) - } - results := make([]actor.Profile, 0, len(accounts)) - for _, acc := range accounts { - results = append(results, actor.Profile{ - Platform: "ActivityPub", - Handle: acc.Acct, - DisplayName: acc.DisplayName, - AvatarURL: acc.Avatar, - Bio: acc.Note, - FollowersCount: acc.FollowersCount, - FollowingCount: acc.FollowingCount, - PostCount: acc.StatusesCount, - URL: acc.URL, - }) - } - return results, nil + } } diff --git a/internal/config/config.go b/internal/config/config.go index afe01b9..c60c9f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -86,9 +86,9 @@ func Load() *Config { BlueskyAppPassword: getEnv("BLUESKY_APP_PASSWORD", ""), BlueskyHost: getEnv("BLUESKY_HOST", "https://bsky.social"), NostrRelays: splitComma(getEnv("NOSTR_RELAYS", "wss://relay.damus.io,wss://nos.lol")), - TwitterBearerToken: getEnv("TWITTER_BEARER_TOKEN", ""), - PrewarmAuthors: splitComma(getEnv("PREWARM_AUTHORS", "")), - PrewarmFeedURI: getEnv("PREWARM_FEED_URI", ""), + TwitterBearerToken: getEnvOptional("TWITTER_BEARER_TOKEN"), + PrewarmAuthors: splitComma(getEnvOptional("PREWARM_AUTHORS")), + PrewarmFeedURI: getEnvOptional("PREWARM_FEED_URI"), EnrichCrossPlatform: getEnvBool("ENRICH_CROSS_PLATFORM", "true"), OllamaHost: getEnv("OLLAMA_HOST", "http://localhost:11434"), MigrateOnStart: getEnvBool("MIGRATE_ON_START", "true"), @@ -133,6 +133,14 @@ func getEnv(key, fallback string) string { return fallback } +func getEnvOptional(key string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + log.Warn("Environment variable is not set (optional)", "key", key) + return "" +} + func getEnvBool(key, fallback string) bool { v := getEnv(key, fallback) switch strings.ToLower(strings.TrimSpace(v)) { diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 5c22907..686cd5f 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -30,37 +30,93 @@ type Post struct { CreatedAt time.Time } -func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { - pubkey, err := resolvePubkey(actor) +func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { + eid, err := resolveEventID(id) + if err != nil { + return nil, err + } + + for _, url := range c.relays { + post, found, err := func() (*Post, bool, error) { + relayCtx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) + if err != nil { + return nil, false, nil // Try next relay + } + defer relay.Close() + + sub, err := relay.Subscribe(relayCtx, nostr.Filter{ + IDs: []nostr.ID{eid}, + Kinds: []nostr.Kind{1}, + }, nostr.SubscriptionOptions{}) + if err != nil { + return nil, false, nil + } + defer sub.Unsub() + + select { + case <-sub.EndOfStoredEvents: + return nil, false, nil + case ev, ok := <-sub.Events: + if !ok { + return nil, false, nil + } + return &Post{ + ID: ev.ID.Hex(), + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }, true, nil + case <-ctx.Done(): + return nil, false, ctx.Err() + } + }() + + if err != nil { + return nil, err + } + if found { + return post, nil + } + } + + return nil, fmt.Errorf("post %q not found across configured relays", id) +} + +func (c *Client) GetLatestNPosts(ctx context.Context, author string, limit int) ([]*Post, error) { + pubkey, err := resolvePubkey(author) if err != nil { return nil, err } posts := make([]*Post, 0, limit) seen := make(map[string]struct{}) - connected := false for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue - } - connected = true - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - func() { - defer sub.Unsub() + relayCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) + if err != nil { + return + } + connected = true defer relay.Close() + sub, err := relay.Subscribe(relayCtx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Limit: limit, + }, nostr.SubscriptionOptions{}) + if err != nil { + return + } + defer sub.Unsub() + collect: for { select { @@ -81,25 +137,31 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( Content: ev.Content, CreatedAt: ev.CreatedAt.Time(), }) + + if len(posts) >= limit { + break collect + } case <-ctx.Done(): break collect + case <-relayCtx.Done(): + break collect } } }() - if len(posts) >= limit { + if len(posts) >= limit || ctx.Err() != nil { break } } - if !connected { + if !connected && len(posts) == 0 { return nil, fmt.Errorf("could not connect to any relay") } return posts, nil } -func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { - pubkey, err := resolvePubkey(actor) +func (c *Client) GetPostsInTimeframe(ctx context.Context, author string, start, end time.Time, limit int) ([]*Post, error) { + pubkey, err := resolvePubkey(author) if err != nil { return nil, err } @@ -109,31 +171,32 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e posts := make([]*Post, 0, limit) seen := make(map[string]struct{}) - connected := false for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue - } - connected = true - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Since: since, - Until: until, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - func() { - defer sub.Unsub() + relayCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) + if err != nil { + return + } + connected = true defer relay.Close() + sub, err := relay.Subscribe(relayCtx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Since: since, + Until: until, + Limit: limit, + }, nostr.SubscriptionOptions{}) + if err != nil { + return + } + defer sub.Unsub() + collect: for { select { @@ -154,68 +217,30 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e Content: ev.Content, CreatedAt: ev.CreatedAt.Time(), }) + + if len(posts) >= limit { + break collect + } case <-ctx.Done(): break collect + case <-relayCtx.Done(): + break collect } } }() - if len(posts) >= limit { + if len(posts) >= limit || ctx.Err() != nil { break } } - if !connected { + if !connected && len(posts) == 0 { return nil, fmt.Errorf("could not connect to any relay") } return posts, nil } -func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { - eid, err := resolveEventID(id) - if err != nil { - return nil, err - } - - for _, url := range c.relays { - relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) - if err != nil { - continue - } - sub, err := relay.Subscribe(ctx, nostr.Filter{ - IDs: []nostr.ID{eid}, - Kinds: []nostr.Kind{1}, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - defer sub.Unsub() - defer relay.Close() - - for { - select { - case <-sub.EndOfStoredEvents: - return nil, fmt.Errorf("post %q not found", id) - case ev, ok := <-sub.Events: - if !ok { - return nil, fmt.Errorf("post %q not found", id) - } - return &Post{ - ID: ev.ID.Hex(), - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }, nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } - } - return nil, fmt.Errorf("could not connect to any relay") -} - func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, error) { pubkey, err := resolvePubkey(author) if err != nil { @@ -224,30 +249,32 @@ func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, e npub := nip19.EncodeNpub(pubkey) for _, u := range c.relays { - relay, err := nostr.RelayConnect(ctx, u, nostr.RelayOptions{}) - if err != nil { - continue - } - sub, err := relay.Subscribe(ctx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{0}, - Limit: 1, - }, nostr.SubscriptionOptions{}) - if err != nil { - relay.Close() - continue - } - defer sub.Unsub() - defer relay.Close() + profile, found, err := func() (*actor.Profile, bool, error) { + relayCtx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + relay, err := nostr.RelayConnect(relayCtx, u, nostr.RelayOptions{}) + if err != nil { + return nil, false, nil + } + defer relay.Close() + + sub, err := relay.Subscribe(relayCtx, nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{0}, + Limit: 1, + }, nostr.SubscriptionOptions{}) + if err != nil { + return nil, false, nil + } + defer sub.Unsub() - collect: - for { select { case <-sub.EndOfStoredEvents: - break collect + return nil, false, nil case ev, ok := <-sub.Events: if !ok { - break collect + return nil, false, nil } var meta struct { Name string `json:"name"` @@ -257,49 +284,63 @@ func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, e Website string `json:"website"` } if err := json.Unmarshal([]byte(ev.Content), &meta); err == nil { + displayName := meta.DisplayName + if displayName == "" { + displayName = meta.Name + } return &actor.Profile{ Platform: "Nostr", Handle: npub, - DisplayName: meta.DisplayName, + DisplayName: displayName, AvatarURL: meta.Picture, Bio: meta.About, URL: "https://njump.me/" + npub, - }, nil + }, true, nil } - break collect + return nil, false, nil case <-ctx.Done(): - return nil, ctx.Err() + return nil, false, ctx.Err() } + }() + + if err != nil { + return nil, err + } + if found { + return profile, nil } } + return nil, fmt.Errorf("no profile metadata found for %q", author) } -func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]actor.Profile, error) { - // Nostr has no global actor search; best-effort: if the query resolves to a - // pubkey/npub, return that single profile. +func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { if _, err := resolvePubkey(query); err != nil { - return []actor.Profile{}, nil + return []*actor.Profile{}, nil } prof, err := c.GetActor(ctx, query) if err != nil { - return []actor.Profile{}, nil + return []*actor.Profile{}, nil } - return []actor.Profile{*prof}, nil + return []*actor.Profile{prof}, nil } func resolvePubkey(actor string) (nostr.PubKey, error) { prefix, value, err := nip19.Decode(actor) if err == nil { - if prefix != "npub" { - return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) + switch prefix { + case "npub": + if pk, ok := value.(nostr.PubKey); ok { + return pk, nil + } + case "nprofile": + if pp, ok := value.(nostr.ProfilePointer); ok { + return pp.PublicKey, nil + } } - pk, ok := value.(nostr.PubKey) - if !ok { - return nostr.PubKey{}, fmt.Errorf("unexpected decoded value type %T", value) - } - return pk, nil + return nostr.PubKey{}, fmt.Errorf("unsupported nostr entity %q", prefix) } + if len(actor) == 64 { pk, e := nostr.PubKeyFromHex(actor) if e != nil { @@ -313,15 +354,19 @@ func resolvePubkey(actor string) (nostr.PubKey, error) { func resolveEventID(id string) (nostr.ID, error) { prefix, value, err := nip19.Decode(id) if err == nil { - if prefix != "note" { - return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) + switch prefix { + case "note": + if eid, ok := value.(nostr.ID); ok { + return eid, nil + } + case "nevent": + if ep, ok := value.(nostr.EventPointer); ok { + return ep.ID, nil + } } - eid, ok := value.(nostr.ID) - if !ok { - return nostr.ID{}, fmt.Errorf("unexpected decoded value type %T", value) - } - return eid, nil + return nostr.ID{}, fmt.Errorf("unsupported nostr entity %q", prefix) } + if len(id) == 64 { eid, e := nostr.IDFromHex(id) if e != nil { diff --git a/internal/twitter/client.go b/internal/twitter/client.go index 08e97e2..4adb85e 100644 --- a/internal/twitter/client.go +++ b/internal/twitter/client.go @@ -105,13 +105,17 @@ type singleTweetResponse struct { } `json:"errors"` } -// doGet issues a GET and retries with exponential backoff when Twitter returns HTTP 429 -// (Too Many Requests). It honors the x-rate-limit-* headers and caps retries so a -// sustained rate limit still surfaces as a real error to the caller (and thus isn't billed). +const ( + maxRetries = 4 + maxRateLimitWait = 30 * time.Second // Never stall a request longer than 30s +) + +var defaultBackoff = 500 * time.Millisecond + func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { - const maxRetries = 4 - backoff := 500 * time.Millisecond - for attempt := 0; ; attempt++ { + backoff := defaultBackoff + + for attempt := 0; attempt <= maxRetries; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, fmt.Errorf("failed to build request: %w", err) @@ -122,62 +126,85 @@ func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("failed to do request: %w", err) } - body, err := io.ReadAll(resp.Body) - resp.Body.Close() + + // Handle 429 Too Many Requests + if resp.StatusCode == http.StatusTooManyRequests { + // Drain & close body using Discard (no allocation) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + if attempt >= maxRetries { + return nil, fmt.Errorf("twitter rate limit exceeded, max retries reached") + } + + wait := parseRateLimitWait(resp.Header, backoff) + + // Protect against 15-minute thread freezes + if wait > maxRateLimitWait { + return nil, fmt.Errorf("twitter rate limited: reset wait (%s) exceeds maximum allowed (%s)", wait, maxRateLimitWait) + } + + log.WarnWithContext(ctx, "twitter rate limited, waiting to retry", "attempt", attempt+1, "wait", wait.String()) + + // Safe timer that frees resources on context cancellation + timer := time.NewTimer(wait) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + } + + backoff *= 2 // Exponential backoff for subsequent attempts + continue + } + + // Read response body once + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) if err != nil { return nil, fmt.Errorf("failed to read response: %w", err) } - if resp.StatusCode == http.StatusTooManyRequests { - if attempt >= maxRetries { - return nil, fmt.Errorf("twitter rate limited after retries: %s", resp.Status) + if resp.StatusCode != http.StatusOK { + errMsg := strings.TrimSpace(string(body)) + if len(errMsg) > 512 { + errMsg = errMsg[:512] + "... (truncated)" } - // Honor x-rate-limit-* headers first - remaining := resp.Header.Get("x-rate-limit-remaining") - reset := resp.Header.Get("x-rate-limit-reset") - if ra := remaining; ra != "" { - if rem, perr := strconv.Atoi(ra); perr == nil { - if rem <= 0 && reset != "" { - if rst, perr2 := strconv.Atoi(reset); perr2 == nil { - // Wait until the rate limit resets (Unix timestamp) - now := time.Now().Unix() - rst64 := int64(rst) - wait := time.Duration(rst64-now) * time.Second - if wait > 0 { - log.WarnWithContext(ctx, "twitter rate limited, waiting until reset", "reset", reset, "wait", wait.String()) - select { - case <-time.After(wait): - case <-ctx.Done(): - return nil, ctx.Err() - } - backoff = 500 * time.Millisecond - continue - } - } + return nil, fmt.Errorf("twitter error (%s): %s", resp.Status, errMsg) + } + + return body, nil + } + + return nil, fmt.Errorf("twitter request failed after %d attempts", maxRetries) +} + +// parseRateLimitWait extracts x-rate-limit-* or Retry-After, falling back to backoff. +func parseRateLimitWait(header http.Header, fallback time.Duration) time.Duration { + // 1. Check x-rate-limit-reset (Unix timestamp) + if remStr := header.Get("x-rate-limit-remaining"); remStr != "" { + if rem, err := strconv.Atoi(remStr); err == nil && rem <= 0 { + if rstStr := header.Get("x-rate-limit-reset"); rstStr != "" { + if rst, err := strconv.ParseInt(rstStr, 10, 64); err == nil { + // Add 1s clock skew buffer + wait := time.Until(time.Unix(rst, 0)) + (1 * time.Second) + if wait > 0 { + return wait } } } - // Fall back to Retry-After if headers not useful - if ra := resp.Header.Get("Retry-After"); ra != "" { - if secs, perr := strconv.Atoi(ra); perr == nil && secs > 0 { - backoff = time.Duration(secs) * time.Second - } - } - log.WarnWithContext(ctx, "twitter rate limited, backing off", "attempt", attempt+1, "wait", backoff.String()) - select { - case <-time.After(backoff): - case <-ctx.Done(): - return nil, ctx.Err() - } - backoff *= 2 - continue } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("twitter error: %s: %s", resp.Status, string(body)) - } - return body, nil } + + // 2. Check standard Retry-After (seconds) + if ra := header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + return time.Duration(secs)*time.Second + (1 * time.Second) + } + } + + return fallback } func (c *Client) resolveUser(ctx context.Context, username string) (string, error) { @@ -332,16 +359,3 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { Replies: tr.Data.PublicMetrics.ReplyCount, }, nil } - -type userFields struct { - ID string `json:"id"` - Name string `json:"name"` - Username string `json:"username"` - Description string `json:"description"` - ProfileImageURL string `json:"profile_image_url"` - PublicMetrics struct { - FollowersCount int `json:"followers_count"` - FollowingCount int `json:"following_count"` - TweetCount int `json:"tweet_count"` - } `json:"public_metrics"` -} From 7aee2fe88c5666bd0db1474774e1a306f06483fa Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:15:31 +0000 Subject: [PATCH 05/30] feat: x402 actual-cost settlement for getStyleProfile + custom MCP wrapper --- internal/server/server_init.go | 2 +- internal/server/tools.go | 8 +- internal/x402/mcpwrapper.go | 156 ++++++++++++++++++++++++ internal/x402/x402.go | 137 +++++++++++++++++---- internal/x402/x402_test.go | 210 ++++++++++++++++++++++++++++++--- 5 files changed, 469 insertions(+), 44 deletions(-) create mode 100644 internal/x402/mcpwrapper.go diff --git a/internal/server/server_init.go b/internal/server/server_init.go index 88f801b..0cef798 100644 --- a/internal/server/server_init.go +++ b/internal/server/server_init.go @@ -65,7 +65,7 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } srv, err := x402.NewX402Server( - "x402 enabled Bluesky lookingglass for autonomous Agents", + "x402 enabled social media lookingglass for autonomous agents", "0.2.0", cfg.X402FacilitatorURL, auth, diff --git a/internal/server/tools.go b/internal/server/tools.go index ebf7d22..e6cecb9 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -93,8 +93,12 @@ func (s *Server) registerTools() { ), }, { + // 0.76 is the maximum authorization for a style profile: $0.25 base + $0.01 + // Twitter lookup + $0.005/post at the 100-post cap (see maxProfileCostMicro in + // internal/x402). The actual charge is computed from posts read and settled via + // an override, so this must stay in sync with that constant. Name: "getStyleProfile", Desc: "Get an actor's style Profile", - Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.36", Scheme: "upto", + Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.76", Scheme: "upto", Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -109,7 +113,7 @@ func (s *Server) registerTools() { }, { Name: "findSimilarUsers", Desc: "Find authors with a similar writing style to the given author, across all platforms", - Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.05", + Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.10", Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, diff --git a/internal/x402/mcpwrapper.go b/internal/x402/mcpwrapper.go new file mode 100644 index 0000000..8d69b01 --- /dev/null +++ b/internal/x402/mcpwrapper.go @@ -0,0 +1,156 @@ +package x402 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/modelcontextprotocol/go-sdk/mcp" + log "github.com/nacorid/logger" + x402 "github.com/x402-foundation/x402/go/v2" + "github.com/x402-foundation/x402/go/v2/types" +) + +// Payment meta keys exchanged over MCP, mirroring the x402 SDK contract. +const ( + PaymentMetaKey = "x402/payment" + PaymentResponseMetaKey = "x402/payment-response" +) + +// SettleAmountResolver computes the actual settlement amount for a completed +// tool call. It returns ("", false) to settle the amount advertised in the +// payment requirements as-is. +type SettleAmountResolver func(result *mcp.CallToolResult) (string, bool) + +// PaymentWrapperConfig configures a PaymentWrapper. Unlike the SDK's wrapper, +// it accepts a SettleAmountResolver so upto-scheme tools can charge the actual +// computed cost after the handler runs (the same behavior the REST path gets +// via nethttp.SetSettlementOverrides). +type PaymentWrapperConfig struct { + Accepts []x402.PaymentRequirements + Resource *types.ResourceInfo + Extensions map[string]interface{} + // SettleAmountResolver is invoked after a successful handler call. When it + // returns a non-empty amount, that amount is charged instead of the + // advertised maximum (which must therefore be <= the accept amount). + SettleAmountResolver SettleAmountResolver +} + +// PaymentWrapper gates an MCP tool behind x402 payments and settles them. +type PaymentWrapper struct { + server *x402.X402ResourceServer + config PaymentWrapperConfig +} + +// NewPaymentWrapper creates a PaymentWrapper for the given resource server. +func NewPaymentWrapper(server *x402.X402ResourceServer, config PaymentWrapperConfig) *PaymentWrapper { + if len(config.Accepts) == 0 { + panic("PaymentWrapperConfig.Accepts must have at least one payment requirement") + } + return &PaymentWrapper{server: server, config: config} +} + +// Wrap returns a ToolHandler that requires a valid x402 payment before invoking +// the wrapped handler and settles it afterwards. +func (w *PaymentWrapper) Wrap(handler mcp.ToolHandler) mcp.ToolHandler { + return func(ctx context.Context, request *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + paymentData := extractPaymentFromRequest(request) + if paymentData == nil { + return w.paymentRequiredResult("Payment Required"), nil + } + + payloadBytes, err := json.Marshal(paymentData) + if err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil + } + var payload types.PaymentPayload + if err := json.Unmarshal(payloadBytes, &payload); err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Invalid payment payload: %v", err)), nil + } + + matched := w.server.FindMatchingRequirements(w.config.Accepts, payload) + if matched == nil { + return w.paymentRequiredResult("No matching payment requirements found"), nil + } + requirements := *matched + + verifyResp, err := w.server.VerifyPayment(ctx, payload, requirements) + if err != nil { + return w.paymentRequiredResult(fmt.Sprintf("Payment verification error: %v", err)), nil + } + if !verifyResp.IsValid { + return w.paymentRequiredResult(fmt.Sprintf("Payment verification failed: %s", verifyResp.InvalidReason)), nil + } + + result, err := handler(ctx, request) + if err != nil { + return nil, err + } + if result.IsError { + return result, nil + } + + var overrides *x402.SettlementOverrides + if w.config.SettleAmountResolver != nil { + if amount, ok := w.config.SettleAmountResolver(result); ok && amount != "" { + overrides = &x402.SettlementOverrides{Amount: amount} + log.Debug("MCP settlement override", "amount", amount) + } + } + + settleResp, err := w.server.SettlePayment(ctx, payload, requirements, overrides) + if err != nil { + return w.settlementFailedResult(fmt.Sprintf("Settlement error: %v", err)), nil + } + if !settleResp.Success { + return w.settlementFailedResult(fmt.Sprintf("Settlement failed: %s", settleResp.ErrorReason)), nil + } + + if result.Meta == nil { + result.Meta = mcp.Meta{} + } + result.Meta[PaymentResponseMetaKey] = settleResp + return result, nil + } +} + +func extractPaymentFromRequest(request *mcp.CallToolRequest) interface{} { + if request == nil || request.Params.Meta == nil { + return nil + } + return request.Params.Meta[PaymentMetaKey] +} + +func (w *PaymentWrapper) paymentRequiredResult(errorMessage string) *mcp.CallToolResult { + var resource *types.ResourceInfo + if w.config.Resource != nil { + resource = &types.ResourceInfo{ + URL: w.config.Resource.URL, + Description: w.config.Resource.Description, + } + } + payment := types.PaymentRequired{ + X402Version: 2, + Error: errorMessage, + Resource: resource, + Accepts: w.config.Accepts, + Extensions: w.config.Extensions, + } + b, err := json.Marshal(payment) + if err != nil { + panic(err) + } + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}, + } +} + +func (w *PaymentWrapper) settlementFailedResult(errorMessage string) *mcp.CallToolResult { + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{ + &mcp.TextContent{Text: fmt.Sprintf("{\"x402Version\":2,\"error\":%q}", errorMessage)}, + }, + } +} diff --git a/internal/x402/x402.go b/internal/x402/x402.go index e1cb2a6..c2d5525 100644 --- a/internal/x402/x402.go +++ b/internal/x402/x402.go @@ -10,7 +10,9 @@ import ( "net" "net/http" "os" + "os/user" "reflect" + "slices" "strconv" "strings" "sync" @@ -22,9 +24,9 @@ import ( x402 "github.com/x402-foundation/x402/go/v2" x402http "github.com/x402-foundation/x402/go/v2/http" nethttp "github.com/x402-foundation/x402/go/v2/http/nethttp" - x402mcp "github.com/x402-foundation/x402/go/v2/mcp" evm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/exact/server" uptoevm "github.com/x402-foundation/x402/go/v2/mechanisms/evm/upto/server" + "github.com/x402-foundation/x402/go/v2/types" ) type X402Middlewares struct { @@ -55,6 +57,9 @@ type X402Server struct { resourceInitMu sync.Mutex resourceInitDone bool + + httpServer *http.Server + mu sync.Mutex } const VERSION1 = "v1/" @@ -128,7 +133,7 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au s.mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) + _, _ = w.Write([]byte("ok")) }) s.mux.HandleFunc("GET /openapi.json", s.serveOpenAPI) @@ -151,9 +156,9 @@ const facilitatorRetryTimeout = 10 * time.Second // facilitator outage at boot (or afterwards) no longer requires a restart. func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { s.resourceInitMu.Lock() - done := s.resourceInitDone - s.resourceInitMu.Unlock() - if done { + defer s.resourceInitMu.Unlock() + + if s.resourceInitDone { return } @@ -164,9 +169,7 @@ func (s *X402Server) ensureFacilitatorReady(ctx context.Context) { return } - s.resourceInitMu.Lock() s.resourceInitDone = true - s.resourceInitMu.Unlock() } func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { @@ -194,7 +197,7 @@ func (s *X402Server) AddPayableTool( ) { if len(requirements) == 0 { - log.Fatalf("tool %s requires at least one payment requirement", tool.Name) + panic(fmt.Sprintf("tool %s requires at least one payment requirement", tool.Name)) } // Convert each raw dollar price into the asset's smallest-unit integer amount using the @@ -212,7 +215,7 @@ func (s *X402Server) AddPayableTool( Extra: req.Extra, }) if err != nil { - log.Fatalf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err) + panic(fmt.Sprintf("tool %s failed to build MCP payment requirements for scheme %s: %v", tool.Name, req.Scheme, err)) } mcpAccepts = append(mcpAccepts, built...) } @@ -220,9 +223,31 @@ func (s *X402Server) AddPayableTool( tool.InputSchema = toJSONSchema(tool.InputSchema) tool.OutputSchema = toJSONSchema(tool.OutputSchema) - wrapper := x402mcp.NewPaymentWrapper(s.x402ResourceServer, x402mcp.PaymentWrapperConfig{ - Accepts: mcpAccepts, - Resource: &x402mcp.ResourceInfo{ + // getStyleProfile bills by actual posts read (see computeTwitterStyleProfileCost). + // The MCP wrapper then settles the computed cost instead of the advertised maximum, + // mirroring the REST path's nethttp.SetSettlementOverrides behavior. + var settleResolver SettleAmountResolver + if tool.Name == "getStyleProfile" { + settleResolver = func(result *mcp.CallToolResult) (string, bool) { + if len(result.Content) != 1 { + return "", false + } + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + return "", false + } + var js json.RawMessage + if json.Unmarshal([]byte(textContent.Text), &js) != nil { + return "", false + } + return computeTwitterStyleProfileCost(nil, textContent.Text), true + } + } + + wrapper := NewPaymentWrapper(s.x402ResourceServer, PaymentWrapperConfig{ + Accepts: mcpAccepts, + SettleAmountResolver: settleResolver, + Resource: &types.ResourceInfo{ URL: "mcp://tool/" + tool.Name, Description: tool.Description, }, @@ -356,7 +381,8 @@ func (s *X402Server) AddPayableTool( } func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, toolName string, handler mcp.ToolHandler) { - bodyBytes, err := io.ReadAll(r.Body) + defer r.Body.Close() + bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 5*1024*1024)) if err != nil { http.Error(w, "Failed to read body", http.StatusBadRequest) return @@ -378,13 +404,11 @@ func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, tool } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) if len(resp.Content) == 1 { if textContent, ok := resp.Content[0].(*mcp.TextContent); ok { var js json.RawMessage if json.Unmarshal([]byte(textContent.Text), &js) == nil { - // For upto scheme tools, compute actual cost and apply settlement override if toolName == "getStyleProfile" { actualCost := computeTwitterStyleProfileCost(r, textContent.Text) log.DebugWithContext(r.Context(), "Setting settlement override", @@ -395,13 +419,15 @@ func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, tool Amount: actualCost, }) } - w.Write([]byte(textContent.Text)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(textContent.Text)) return } } } - json.NewEncoder(w).Encode(resp) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) } func (s *X402Server) OnAfterSettle(hook func(ctx x402.SettleResultContext) error) { @@ -492,6 +518,23 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to set socket permissions: %w", err) } + grp, err := user.LookupGroup("mcpserver") + if err != nil { + return fmt.Errorf("failed to lookup group mcpserver: %w", err) + } + gid, err := strconv.Atoi(grp.Gid) + if err != nil { + return fmt.Errorf("invalid gid %q: %w", grp.Gid, err) + } + + if err := os.Chown(socketPath, -1, gid); err != nil { + return fmt.Errorf("failed to set socket group ownership: %w", err) + } + + if err := os.Chmod(socketPath, permissions); err != nil { + return fmt.Errorf("failed to set socket permissions: %w", err) + } + log.Info("Starting X402 MCP Server on unix socket", "socketPath", socketPath) } else { log.Info("Starting X402 MCP Server on TCP", "addr", addr) @@ -501,15 +544,33 @@ func (s *X402Server) Start(addr string, permissions os.FileMode) error { return fmt.Errorf("failed to listen on tcp: %w", err) } } - defer listener.Close() srv := &http.Server{ - Handler: s.globalReqLog(s.mux), - ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), + Handler: s.globalReqLog(s.mux), + ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelDebug), + ReadTimeout: 15 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, } + + s.mu.Lock() + s.httpServer = srv + s.mu.Unlock() + return srv.Serve(listener) } +func (s *X402Server) Shutdown(ctx context.Context) error { + s.mu.Lock() + srv := s.httpServer + s.mu.Unlock() + + if srv == nil { + return nil + } + return srv.Shutdown(ctx) +} + func toJSONSchema(val any) json.RawMessage { if val == nil { return json.RawMessage(`{"type": "object"}`) @@ -570,8 +631,8 @@ func parseSchema(schemaAny any) any { func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { - for i := len(middlewares) - 1; i >= 0; i-- { - next = middlewares[i](next) + for _, middleware := range slices.Backward(middlewares) { + next = middleware(next) } return next } @@ -586,21 +647,38 @@ const ( // baseProfileCostMicro is the base cost of the style-profile endpoint for // the non-Twitter path. It doubles as the profit margin added to Twitter // requests, expressed in micro-dollars (USDC has 6 decimals). - baseProfileCostMicro int64 = 100_000 // $0.10 + baseProfileCostMicro int64 = 250_000 // $0.25 // twitterUserLookupMicro is the per-request Twitter user lookup API cost. twitterUserLookupMicro int64 = 10_000 // $0.01 // twitterPostReadMicro is the per-post Twitter read API cost. twitterPostReadMicro int64 = 5_000 // $0.005 per post + // maxProfileCountedPosts caps the post count a style profile can ever + // report. It is NOT applied inside computeTwitterStyleProfileCost (charges + // follow the posts actually read, and the server fetches at most this + // many); it only bounds the largest settlement the endpoint can demand. + maxProfileCountedPosts int64 = 100 ) +// maxProfileCostMicro is the largest price getStyleProfile can ever command: +// the flat base cost plus the Twitter lookup and per-post costs at the +// post-count cap. The price advertised for getStyleProfile in tools.go must +// equal formatDollarAmount(maxProfileCostMicro) ("$0.76"). +const maxProfileCostMicro = baseProfileCostMicro + twitterUserLookupMicro + maxProfileCountedPosts*twitterPostReadMicro + // computeTwitterStyleProfileCost calculates the actual charge for a Twitter // getStyleProfile request based on the number of posts fetched and the user -// lookup, and returns it as a dollar price string (e.g. "$0.16") suitable for a +// lookup, and returns it as a dollar price string (e.g. "$0.31") suitable for a // x402 settlement override. // -// Pricing: guaranteed base cost of $0.10 (the endpoint cost for other +// Pricing: guaranteed base cost of $0.25 (the endpoint cost for other // platforms), plus user lookup $0.01, plus $0.005 per post read. For example, -// 10 posts = $0.16. Non-Twitter platforms are charged the flat $0.10 base. +// 10 posts = $0.31. Non-Twitter platforms are charged the flat $0.25 base. +// +// The price always reflects the posts actually read - it is never clamped to a +// cap (the server only ever fetches at most maxProfileCountedPosts, so the +// derived max always stays at or below formatDollarAmount(maxProfileCostMicro)). +// A PostCount of 0 means no posts were read (server-side defaults are applied +// before the profile is built, so responses carry the real count). func computeTwitterStyleProfileCost(_ *http.Request, responseText string) string { // Parse the response to get PostCount and SourcePlatform var resp GetStyleProfileResponse @@ -660,6 +738,13 @@ func (c *verifyErrorCapture) WriteHeader(code int) { c.ResponseWriter.WriteHeader(code) } +func (c *verifyErrorCapture) Write(b []byte) (int, error) { + if c.statusCode == 0 { + c.statusCode = http.StatusOK + } + return c.ResponseWriter.Write(b) +} + func decodePaymentRequiredHeader(header string) (x402.PaymentRequired, error) { data, err := base64.StdEncoding.DecodeString(header) if err != nil { diff --git a/internal/x402/x402_test.go b/internal/x402/x402_test.go index 2598be8..56c35e1 100644 --- a/internal/x402/x402_test.go +++ b/internal/x402/x402_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "sync/atomic" "testing" @@ -136,14 +137,14 @@ func TestX402Server_MCPBoundaries(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer serverSession.Close() + defer func() { _ = serverSession.Close() }() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer clientSession.Close() + defer func() { _ = clientSession.Close() }() toolsResponse, err := clientSession.ListTools(ctx, nil) if err != nil { @@ -221,7 +222,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { x402.PaymentRequirements{ Scheme: "upto", Network: "eip155:84532", - Amount: "0.36", + Amount: "0.51", PayTo: "0x0000000000000000000000000000000000000000", }, ) @@ -230,14 +231,14 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { if err != nil { t.Fatalf("failed to connect server session: %v", err) } - defer serverSession.Close() + defer func() { _ = serverSession.Close() }() client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) clientSession, err := client.Connect(ctx, clientTransport, nil) if err != nil { t.Fatalf("failed to connect client session: %v", err) } - defer clientSession.Close() + defer func() { _ = clientSession.Close() }() cases := []struct { tool string @@ -245,7 +246,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { want string }{ {tool: "getExact", scheme: "exact", want: "10000"}, - {tool: "getUpto", scheme: "upto", want: "360000"}, + {tool: "getUpto", scheme: "upto", want: "510000"}, } for _, tc := range cases { @@ -439,11 +440,13 @@ func TestComputeTwitterStyleProfileCost(t *testing.T) { postCount int want string }{ - {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.10"}, - {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.16"}, - {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.11"}, - {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.36"}, - {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.125"}, + {name: "non-twitter flat base", platform: "Bluesky", postCount: 10, want: "$0.25"}, + {name: "twitter example from spec", platform: "Twitter", postCount: 10, want: "$0.31"}, + {name: "twitter zero posts", platform: "Twitter", postCount: 0, want: "$0.26"}, + {name: "twitter max posts", platform: "Twitter", postCount: 50, want: "$0.51"}, + {name: "twitter odd post count half cents", platform: "Twitter", postCount: 3, want: "$0.275"}, + {name: "posts at the max counted cap", platform: "Twitter", postCount: 100, want: "$0.76"}, + {name: "beyond cap is not clamped", platform: "Twitter", postCount: 200, want: "$1.26"}, } for _, tc := range cases { @@ -691,10 +694,11 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { postCount int want string }{ - {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.10"}, // not "Twitter" so flat base - {name: "empty platform", platform: "", postCount: 10, want: "$0.10"}, - {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.11"}, // should clamp to 0 - {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.11"}, + {name: "case insensitive twitter", platform: "TWITTER", postCount: 10, want: "$0.25"}, // not "Twitter" so flat base + {name: "empty platform", platform: "", postCount: 10, want: "$0.25"}, + {name: "negative post count", platform: "Twitter", postCount: -5, want: "$0.26"}, // should clamp to 0 + {name: "few posts", platform: "Twitter", postCount: 1, want: "$0.265"}, + {name: "very large post count", platform: "Twitter", postCount: 1000, want: "$5.26"}, } for _, tc := range cases { @@ -710,3 +714,179 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { }) } } + +// TestStyleProfileMaxPriceStaysAdvertised guards the advertised getStyleProfile +// price in internal/server/tools.go against silent drift from the x402 pricing +// model. If this fails, "0.76" in tools.go must be updated (they cannot share a +// constant because internal/server cannot import internal/x402). +func TestStyleProfileMaxPriceStaysAdvertised(t *testing.T) { + if got := formatDollarAmount(maxProfileCostMicro); got != "$0.76" { + t.Fatalf("maxProfileCostMicro = %d (%s), want $0.76; update getStyleProfile Price in internal/server/tools.go if the pricing model changed", maxProfileCostMicro, got) + } +} + +// TestX402Server_MCPStyleProfileSettlementOverride drives the full MCP flow for +// the getStyleProfile upto tool and asserts the facilitator is told to settle +// the computed per-request cost (not the advertised $0.76 maximum). +func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { + ctx := context.Background() + + var mu sync.Mutex + var settleAmounts []string + mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == "GET" && r.URL.Path == "/supported": + _, _ = w.Write([]byte(`{ + "kinds": [ + {"x402Version": 2, "scheme": "upto", "network": "eip155:84532"} + ] + }`)) + case r.Method == "POST" && r.URL.Path == "/verify": + _, _ = w.Write([]byte(`{"isValid": true, "invalidReason": ""}`)) + case r.Method == "POST" && r.URL.Path == "/settle": + var body struct { + PaymentRequirements map[string]json.RawMessage `json:"paymentRequirements"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if raw := body.PaymentRequirements["amount"]; len(raw) > 0 { + var s string + var f float64 + mu.Lock() + if json.Unmarshal(raw, &s) == nil { + settleAmounts = append(settleAmounts, s) + } else if json.Unmarshal(raw, &f) == nil { + settleAmounts = append(settleAmounts, strconv.FormatFloat(f, 'f', -1, 64)) + } + mu.Unlock() + } + _, _ = w.Write([]byte(`{ + "success": true, + "transaction": "0x1234abcd", + "network": "eip155:84532", + "payer": "0xabcd1234" + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer mock.Close() + + serverTransport, clientTransport := mcp.NewInMemoryTransports() + + srv, err := NewX402Server( + "Test Agent API", + "1.0.0", + mock.URL, + &DummyAuthProvider{}, + X402Middlewares{}, + func(next http.Handler) http.Handler { return next }, + func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) {}, + []string{"eip155:84532"}, + ) + if err != nil { + t.Fatalf("failed to create server: %v", err) + } + + responses := []GetStyleProfileResponse{ + {Author: "@a", TopKeywords: []string{}, StyleSummary: "s", PostCount: 10, SourcePlatform: "Bluesky"}, + {Author: "@b", TopKeywords: []string{}, StyleSummary: "s", PostCount: 50, SourcePlatform: "Twitter"}, + } + var idx int + srv.AddPayableTool( + mcp.Tool{Name: "getStyleProfile", Description: "get a style profile"}, + func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + resp := responses[idx%len(responses)] + idx++ + b, _ := json.Marshal(resp) + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}}, nil + }, + x402.PaymentRequirements{ + Scheme: "upto", + Network: "eip155:84532", + Amount: "0.76", + PayTo: "0x0000000000000000000000000000000000000000", + }, + ) + + serverSession, err := srv.mcpServer.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatalf("failed to connect server session: %v", err) + } + defer func() { _ = serverSession.Close() }() + + client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1.0.0"}, nil) + clientSession, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatalf("failed to connect client session: %v", err) + } + defer func() { _ = clientSession.Close() }() + + paidCall := func() (*mcp.CallToolResult, error) { + challenge, err := clientSession.CallTool(ctx, &mcp.CallToolParams{Name: "getStyleProfile", Arguments: json.RawMessage(`{}`)}) + if err != nil { + return nil, fmt.Errorf("challenge call failed: %w", err) + } + if !challenge.IsError { + return nil, fmt.Errorf("expected a payment challenge, got %+v", challenge) + } + text, ok := challenge.Content[0].(*mcp.TextContent) + if !ok { + return nil, fmt.Errorf("unexpected challenge content type %T", challenge.Content[0]) + } + var pr types.PaymentRequired + if err := json.Unmarshal([]byte(text.Text), &pr); err != nil { + return nil, fmt.Errorf("failed to decode payment challenge: %w", err) + } + if len(pr.Accepts) == 0 { + return nil, fmt.Errorf("expected at least one accept, got %d", len(pr.Accepts)) + } + payload := types.PaymentPayload{ + X402Version: 2, + Payload: map[string]interface{}{"chainId": 84532}, + Accepted: pr.Accepts[0], + } + return clientSession.CallTool(ctx, &mcp.CallToolParams{ + Name: "getStyleProfile", + Arguments: json.RawMessage(`{}`), + Meta: mcp.Meta{PaymentMetaKey: payload}, + }) + } + + for i := 0; i < 2; i++ { + result, err := paidCall() + if err != nil { + t.Fatalf("paid call: %v", err) + } + if result.IsError { + text, _ := result.Content[0].(*mcp.TextContent) + t.Fatalf("expected successful paid call, got error result: %+v", text) + } + if resp := result.Meta[PaymentResponseMetaKey]; resp == nil { + t.Fatalf("expected %q in result meta", PaymentResponseMetaKey) + } + } + + mu.Lock() + got := append([]string(nil), settleAmounts...) + mu.Unlock() + // The "$0.25" / "$0.51" overrides resolve to 250000 / 510000 atomic units, + // with up to 1 micro tolerance for float rounding in the SDK's dollar parser. + wants := []int64{250000, 510000} + if len(got) != len(wants) { + t.Fatalf("settle amount count = %d (%v), want %d: %v", len(got), got, len(wants), wants) + } + for i, want := range wants { + v, err := strconv.ParseInt(got[i], 10, 64) + if err != nil { + t.Fatalf("settle amount[%d] = %q is not an integer: %v", i, got[i], err) + } + diff := v - want + if diff < 0 { + diff = -diff + } + if diff > 1 { + t.Errorf("settle amount[%d] = %d, want ~%d", i, v, want) + } + } +} From 22b92cb6cd5ac0ca0608e8c5ab550c1c5722187a Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:23:08 +0000 Subject: [PATCH 06/30] fix: skip empty embeddings/text posts; 1-month profile refresh window --- internal/server/mcp_test.go | 4 +- internal/server/posts_convert.go | 3 ++ internal/server/style.go | 23 +++++++++-- internal/server/styleprofile_test.go | 60 +++++++++++++++++++++++++++- internal/utils/styleProfiles.go | 29 +++++++++++--- internal/utils/styleProfiles_test.go | 47 ++++++++++++++++++++++ 6 files changed, 153 insertions(+), 13 deletions(-) create mode 100644 internal/utils/styleProfiles_test.go diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 85cac53..36a8dc6 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -823,7 +823,7 @@ func TestPostCacheDifferentKeys(t *testing.T) { func TestCacheVersionStaleness(t *testing.T) { st := newFakeStore() - // Add a stale profile (older than 7 days) + // Add a stale profile (older than 30 days) st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", Platform: "ActivityPub", @@ -831,7 +831,7 @@ func TestCacheVersionStaleness(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}}} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index 2475c10..c9bc0db 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -154,6 +154,9 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I if !ok { continue } + if fp.Text == "" { + continue + } out = append(out, clustering.Item{ID: p.Post.Uri, Text: fp.Text}) } return out diff --git a/internal/server/style.go b/internal/server/style.go index 5f9e217..4001d00 100644 --- a/internal/server/style.go +++ b/internal/server/style.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "strings" + "github.com/modelcontextprotocol/go-sdk/mcp" log "github.com/nacorid/logger" "github.com/nacorid/naco-api/internal/clustering" @@ -52,7 +54,7 @@ func (s *Server) getStyleProfile(ctx context.Context, request *mcp.CallToolReque } // requestStyleProfile returns a style profile for author. It computes and persists the -// profile only when a request is made AND the cached profile is older than a week (or +// profile only when a request is made AND the cached profile is older than a month (or // missing); otherwise it serves the cached copy. The prewarmer calls this same path, so a // pre-fetch is treated as a system-initiated request and obeys the identical update policy. func (s *Server) requestStyleProfile(ctx context.Context, author string) (*GetStyleProfileResponse, error) { @@ -86,7 +88,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor Platform: platform.String(), }) if err == nil { - isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*7*time.Hour + isRecent := styleProfile.ModifiedAt.Valid && time.Since(styleProfile.ModifiedAt.Time) < 24*30*time.Hour if isRecent { embedding, perr := utils.ParseEmbedding(styleProfile.Embedding) if perr != nil { @@ -110,7 +112,7 @@ func (s *Server) requestStyleProfileForced(ctx context.Context, platform platfor // enrichCrossPlatform looks for author on every platform other than sourcePlatform, in the // background. It uses the same identifier on each platform (matching only where that handle -// exists; mismatches fail gracefully and are skipped) and obeys the same 1-week update gate +// exists; mismatches fail gracefully and are skipped) and obeys the same 1-month update gate // as a live request. Concurrency is bounded to the other three platforms. func (s *Server) enrichCrossPlatform(ctx context.Context, sourcePlatform platform, author string) { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) @@ -211,6 +213,21 @@ func (s *Server) buildStyleProfile(ctx context.Context, platform platform, autho return nil, fmt.Errorf("no posts found for author %s", author) } + // Drop posts without any text (e.g. Bluesky image/reply-only posts). Embedding them + // yields empty vectors, which would poison the persona-average and the persisted + // profile. + filtered := posts[:0] + for _, p := range posts { + if strings.TrimSpace(p.Text) == "" { + continue + } + filtered = append(filtered, p) + } + if len(filtered) == 0 { + return nil, fmt.Errorf("no posts found for author %s", author) + } + posts = filtered + // Delete old posts for non-Twitter platforms before rebuild if platform != Twitter { if err := s.db.DeleteStylePosts(ctx, store.DeleteStylePostsParams{ diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index ffc8079..49631c8 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -3,6 +3,7 @@ package server import ( "database/sql" "fmt" + "reflect" "strings" "testing" "time" @@ -82,7 +83,7 @@ func TestGetStyleProfileServesFreshCache(t *testing.T) { } func TestGetStyleProfileRebuildsStale(t *testing.T) { - // A stale (older than a week) cached profile triggers a rebuild. + // A stale (older than a month) cached profile triggers a rebuild. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ UserID: "@a@mastodon.social", @@ -91,7 +92,7 @@ func TestGetStyleProfileRebuildsStale(t *testing.T) { TopKeywords: []string{"old"}, StyleSummary: "stale summary", PostCount: 5, - ModifiedAt: sql.NullTime{Time: time.Now().Add(-10 * 24 * time.Hour), Valid: true}, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } ap := &fakeAP{posts: []*activitypub.Post{ {Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}, @@ -165,6 +166,61 @@ func TestGetStyleProfileBlueskyResolution(t *testing.T) { } } +func TestGetStyleProfileFirstEmbeddingEmpty(t *testing.T) { + // Regression: an empty embedding returned by Ollama for a non-first post used to + // panic AverageVector when the first vector was also empty (accumulator sized to + // length 0, later non-empty vector indexed it). Empty vectors are skipped. + bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ + {Post: bskyPostView("first post")}, + {Post: bskyPostView("second post")}, + }} + oll := &fakeOllama{embeddings: [][]float64{{}, {0.1, 0.2}}} + st := newFakeStore() + s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) + + res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) + if err != nil { + t.Fatalf("getStyleProfile error: %v", err) + } + var out GetStyleProfileResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(out.PersonaEmbedding, []float64{0.1, 0.2}) { + t.Fatalf("expected averaged embedding {0.1,0.2}, got %v", out.PersonaEmbedding) + } + if out.PostCount != 2 { + t.Fatalf("expected post count 2, got %d", out.PostCount) + } +} + +func TestGetStyleProfileSkipsEmptyTextPosts(t *testing.T) { + // A Bluesky post with no text (image/reply-only) must not reach Ollama: its empty + // embedding would poison the persona average. It is dropped before embedding. + bsky := &fakeBluesky{feedViews: []*bsky.FeedDefs_FeedViewPost{ + {Post: bskyPostView("")}, + {Post: bskyPostView("real post")}, + }} + oll := &fakeOllama{} + st := newFakeStore() + s := newServerWith(bsky, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, oll) + + res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "did:plc:abc"}) + if err != nil { + t.Fatalf("getStyleProfile error: %v", err) + } + var out GetStyleProfileResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.PostCount != 1 { + t.Fatalf("expected filtered post count 1, got %d", out.PostCount) + } + if len(out.PersonaEmbedding) != 2 { + t.Fatalf("expected 2-dim embedding, got %v", out.PersonaEmbedding) + } +} + // ---- findSimilarUsers ---- func TestFindSimilarUsers(t *testing.T) { diff --git a/internal/utils/styleProfiles.go b/internal/utils/styleProfiles.go index 7a6a587..cb475a4 100644 --- a/internal/utils/styleProfiles.go +++ b/internal/utils/styleProfiles.go @@ -56,17 +56,34 @@ func FormatVector(v []float64) string { return "[" + strings.Join(parts, ",") + "]" } +// AverageVector returns the element-wise mean of vs. Zero-length vectors (e.g. an +// embedding model returning an empty vector for a post without text) are skipped, and +// the accumulator is sized from the first non-empty vector so a leading empty vector +// cannot panic later in the loop. Returns nil when no usable vector is present. func AverageVector(vs [][]float64) []float64 { - dim := len(vs[0]) - avg := make([]float64, dim) - + dim := 0 + var valid [][]float64 for _, v := range vs { - for i, val := range v { - avg[i] += val + if len(v) == 0 { + continue + } + if dim == 0 { + dim = len(v) + } + valid = append(valid, v) + } + if len(valid) == 0 { + return nil + } + + avg := make([]float64, dim) + for _, v := range valid { + for i := 0; i < dim && i < len(v); i++ { + avg[i] += v[i] } } for i := range avg { - avg[i] /= float64(len(vs)) + avg[i] /= float64(len(valid)) } return avg } diff --git a/internal/utils/styleProfiles_test.go b/internal/utils/styleProfiles_test.go new file mode 100644 index 0000000..4692b95 --- /dev/null +++ b/internal/utils/styleProfiles_test.go @@ -0,0 +1,47 @@ +package utils + +import ( + "math" + "testing" +) + +func assertVectorsEqual(t *testing.T, got, want []float64) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("length %d, want %d", len(got), len(want)) + } + for i := range want { + if math.Abs(got[i]-want[i]) > 1e-9 { + t.Fatalf("index %d = %v, want %v", i, got, want) + } + } +} + +func TestAverageVector(t *testing.T) { + tests := []struct { + name string + in [][]float64 + want []float64 + }{ + {name: "empty input", in: nil, want: nil}, + {name: "empty slice", in: [][]float64{}, want: nil}, + {name: "all empty vectors", in: [][]float64{{}, {}, {}}, want: nil}, + {name: "single vector", in: [][]float64{{0.4, -0.2}}, want: []float64{0.4, -0.2}}, + {name: "multiple vectors", in: [][]float64{{0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "first vector empty", in: [][]float64{{}, {0.1, 0.2}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "trailing empty vectors", in: [][]float64{{0.1, 0.2}, {}, {0.3, 0.4}}, want: []float64{0.2, 0.3}}, + {name: "only first valid", in: [][]float64{{}, {}, {0.1, 0.2}}, want: []float64{0.1, 0.2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := AverageVector(tt.in) + if tt.want == nil { + if got != nil { + t.Fatalf("AverageVector(%v) = %v, want nil", tt.in, got) + } + return + } + assertVectorsEqual(t, got, tt.want) + }) + } +} From 5cd1d128d6af9e76a316ab5ac93a27fde28457fd Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:25:44 +0000 Subject: [PATCH 07/30] test: expand Ollama client tests and store integration coverage --- internal/ollama/client_test.go | 434 +++++++++++++++++++---------- internal/store/integration_test.go | 174 ++++++++---- 2 files changed, 409 insertions(+), 199 deletions(-) diff --git a/internal/ollama/client_test.go b/internal/ollama/client_test.go index 16b18f3..892ac73 100644 --- a/internal/ollama/client_test.go +++ b/internal/ollama/client_test.go @@ -2,27 +2,87 @@ package ollama import ( "context" + "encoding/json" "net/http" "net/http/httptest" "net/url" + "strings" + "sync" "testing" + "time" "github.com/nacorid/naco-api/internal/clustering" ) +// setupTestServer creates a test server and an initialized client. +func setupTestServer(t *testing.T, handler http.Handler) (*httptest.Server, *OllamaClient) { + t.Helper() + server := httptest.NewServer(handler) + + host, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server URL: %v", err) + } + + client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + return server, client +} + +// ============================================================================ +// EmbedTexts Tests +// ============================================================================ + func TestEmbedTexts(t *testing.T) { + type embedRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + } + + var mu sync.Mutex + var receivedRequests []embedRequest + mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + // 1. Validate HTTP Method + if r.Method != http.MethodPost { + t.Errorf("expected method POST, got %s", r.Method) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // 2. Validate Headers + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) + http.Error(w, "invalid content-type", http.StatusBadRequest) + return + } + + // 3. Validate Request Payload + var req embedRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + if req.Model != "test-embed" { + t.Errorf("expected model 'test-embed', got %q", req.Model) + } + + mu.Lock() + receivedRequests = append(receivedRequests, req) + mu.Unlock() + + // 4. Return valid response w.Header().Set("Content-Type", "application/json") - w.Write([]byte(`{"embedding":[0.1,0.2,0.3]}`)) + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{0.1, 0.2, 0.3}, + }) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - posts := []clustering.Item{ {ID: "1", Text: "hello", Embedding: nil}, {ID: "2", Text: "world", Embedding: nil}, @@ -33,177 +93,194 @@ func TestEmbedTexts(t *testing.T) { t.Fatalf("EmbedTexts failed: %v", err) } + // Verify outgoing requests: ensure both prompts were sent + mu.Lock() + if len(receivedRequests) != 2 { + t.Errorf("expected 2 HTTP requests, got %d", len(receivedRequests)) + } + promptsReceived := map[string]bool{} + for _, req := range receivedRequests { + promptsReceived[req.Prompt] = true + } + mu.Unlock() + + if !promptsReceived["hello"] || !promptsReceived["world"] { + t.Errorf("expected prompts 'hello' and 'world' to be sent, got: %+v", receivedRequests) + } + + // Verify parsed results if len(embeddings) != 2 { t.Fatalf("expected 2 embeddings, got %d", len(embeddings)) } - for i, emb := range embeddings { if len(emb) != 3 { - t.Errorf("embedding %d has wrong dim: %d", i, len(emb)) + t.Errorf("embedding %d has wrong dimension: %d", i, len(emb)) } if emb[0] != 0.1 || emb[1] != 0.2 || emb[2] != 0.3 { - t.Errorf("embedding %d wrong values: %v", i, emb) + t.Errorf("embedding %d has wrong values: %v", i, emb) } } } -func TestEmbedTexts_Error(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} - - _, err := client.EmbedTexts(context.Background(), posts) - if err == nil { - t.Fatal("expected error from failing server") - } -} - -func TestSummarizeStyle(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - flusher, ok := w.(http.Flusher) - if !ok { - t.Fatal("response writer doesn't support flushing") - } - w.Header().Set("Content-Type", "application/x-ndjson") - // Stream response as newline-delimited JSON - w.Write([]byte(`{"response":"Style summary: "}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"concise and clear."}` + "\n")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{ - {ID: "1", Text: "First post", Embedding: nil}, - {ID: "2", Text: "Second post", Embedding: nil}, - } - - summary, err := client.SummarizeStyle(context.Background(), posts) - if err != nil { - t.Fatalf("SummarizeStyle failed: %v", err) - } - - if summary != "Style summary: concise and clear." { - t.Errorf("unexpected summary: %q", summary) - } -} - -func TestSummarizeStyle_Error(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "hello", Embedding: nil}} - - _, err := client.SummarizeStyle(context.Background(), posts) - if err == nil { - t.Fatal("expected error from failing server") - } -} - -func TestSummarizeStyle_Streaming(t *testing.T) { - callCount := 0 - mux := http.NewServeMux() - mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - callCount++ - flusher, ok := w.(http.Flusher) - if !ok { - t.Fatal("response writer doesn't support flushing") - } - w.Header().Set("Content-Type", "application/x-ndjson") - // Simulate streaming response - w.Write([]byte(`{"response":"part1"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"part2"}` + "\n")) - if flusher != nil { - flusher.Flush() - } - w.Write([]byte(`{"response":"part3","done":true}` + "\n")) - }) - - server := httptest.NewServer(mux) - defer server.Close() - - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - - posts := []clustering.Item{{ID: "1", Text: "test", Embedding: nil}} - - summary, err := client.SummarizeStyle(context.Background(), posts) - if err != nil { - t.Fatalf("SummarizeStyle failed: %v", err) - } - - if summary != "part1part2part3" { - t.Errorf("streaming summary = %q, want %q", summary, "part1part2part3") - } - - if callCount != 1 { - t.Errorf("generate endpoint called %d times, want 1", callCount) - } -} - func TestEmbedTexts_EmptyInput(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { - t.Fatal("embeddings endpoint should not be called for empty input") + t.Error("embeddings endpoint should not be called when input is empty") + http.Error(w, "should not be called", http.StatusInternalServerError) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - embeddings, err := client.EmbedTexts(context.Background(), []clustering.Item{}) if err != nil { t.Fatalf("EmbedTexts with empty input failed: %v", err) } if len(embeddings) != 0 { - t.Errorf("expected empty embeddings, got %d", len(embeddings)) + t.Errorf("expected 0 embeddings, got %d", len(embeddings)) + } +} + +func TestEmbedTexts_Error(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error": "model test-embed not found"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.EmbedTexts(context.Background(), posts) + if err == nil { + t.Fatal("expected error from failing server, got nil") + } +} + +func TestEmbedTexts_ContextCanceled(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + // Hang until client cancels the request + <-r.Context().Done() + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + _, err := client.EmbedTexts(ctx, posts) + if err == nil { + t.Fatal("expected error due to canceled context, got nil") + } +} + +// ============================================================================ +// SummarizeStyle Tests +// ============================================================================ + +func TestSummarizeStyle(t *testing.T) { + type generateRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + } + + var reqBody generateRequest + callCount := 0 + + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + callCount++ + + // 1. Validate HTTP Method + if r.Method != http.MethodPost { + t.Errorf("expected method POST, got %s", r.Method) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + // 2. Validate Headers + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + t.Errorf("expected Content-Type application/json, got %q", r.Header.Get("Content-Type")) + http.Error(w, "invalid content-type", http.StatusBadRequest) + return + } + + // 3. Validate Request Payload + if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + if reqBody.Model != "test-style" { + t.Errorf("expected model 'test-style', got %q", reqBody.Model) + } + + // Ensure the prompt includes input text + if !strings.Contains(reqBody.Prompt, "First post") || !strings.Contains(reqBody.Prompt, "Second post") { + t.Errorf("prompt missing post contents: %q", reqBody.Prompt) + } + + // 4. Stream NDJSON Response + flusher, ok := w.(http.Flusher) + if !ok { + t.Error("ResponseWriter is not a Flusher") + return + } + + w.Header().Set("Content-Type", "application/x-ndjson") + + chunks := []string{ + `{"response":"Style summary: "}` + "\n", + `{"response":"concise and "}` + "\n", + `{"response":"clear.","done":true}` + "\n", + } + + for _, chunk := range chunks { + _, _ = w.Write([]byte(chunk)) + flusher.Flush() + } + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{ + {ID: "1", Text: "First post"}, + {ID: "2", Text: "Second post"}, + } + + summary, err := client.SummarizeStyle(context.Background(), posts) + if err != nil { + t.Fatalf("SummarizeStyle failed: %v", err) + } + + expected := "Style summary: concise and clear." + if summary != expected { + t.Errorf("got summary %q, want %q", summary, expected) + } + + if callCount != 1 { + t.Errorf("expected 1 call to /api/generate, got %d", callCount) } } func TestSummarizeStyle_EmptyInput(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { - t.Fatal("generate endpoint should not be called for empty input") + t.Error("generate endpoint should not be called when input is empty") + http.Error(w, "should not be called", http.StatusInternalServerError) }) - server := httptest.NewServer(mux) + server, client := setupTestServer(t, mux) defer server.Close() - host, _ := url.Parse(server.URL) - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") - summary, err := client.SummarizeStyle(context.Background(), []clustering.Item{}) if err != nil { t.Fatalf("SummarizeStyle with empty input failed: %v", err) @@ -213,3 +290,76 @@ func TestSummarizeStyle_EmptyInput(t *testing.T) { t.Errorf("expected empty summary, got %q", summary) } } + +func TestSummarizeStyle_Error(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"internal model failure"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error from failing server, got nil") + } +} + +func TestSummarizeStyle_MalformedJSONStream(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + + // First line is valid JSON, second is corrupt + _, _ = w.Write([]byte(`{"response":"valid part"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + _, _ = w.Write([]byte(`{corrupt_json` + "\n")) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error on malformed JSON stream, got nil") + } +} + +func TestSummarizeStyle_ContextCanceled(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + + // Send initial chunk + _, _ = w.Write([]byte(`{"response":"waiting..."}` + "\n")) + if flusher != nil { + flusher.Flush() + } + + // Wait until client context cancels + <-r.Context().Done() + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(ctx, posts) + if err == nil { + t.Fatal("expected error on canceled context, got nil") + } +} diff --git a/internal/store/integration_test.go b/internal/store/integration_test.go index 2adc0b4..5c9aeaf 100644 --- a/internal/store/integration_test.go +++ b/internal/store/integration_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "os" + "reflect" "strings" "testing" "time" @@ -12,93 +13,140 @@ import ( _ "github.com/lib/pq" ) -// TestStoreRoundTrip exercises the sqlc queries against a real Postgres+pgvector database. -// It is skipped unless TEST_DATABASE_URL is set (e.g. a local instance with the vector -// extension available), so it never fails in environments without a database. -func TestStoreRoundTrip(t *testing.T) { +// setupTestDB handles connection pooling, migrations, and cleanup per test run. +func setupTestDB(t *testing.T) (*sql.DB, *Queries) { + t.Helper() + dsn := os.Getenv("TEST_DATABASE_URL") if dsn == "" { t.Skip("TEST_DATABASE_URL not set; skipping store integration test") } + ctx := context.Background() conn, err := sql.Open("postgres", dsn) if err != nil { - t.Fatal(err) + t.Fatalf("failed to open db: %v", err) } - defer conn.Close() + if err := conn.PingContext(ctx); err != nil { + _ = conn.Close() t.Fatalf("cannot reach test database: %v", err) } + + t.Cleanup(func() { + _ = conn.Close() + }) + if err := Migrate(ctx, conn, "sql/schema"); err != nil { t.Fatalf("migrate: %v", err) } - q := New(conn) + + return conn, New(conn) +} + +func makeEmbedding(dim int, val string) string { + return "[" + strings.TrimSuffix(strings.Repeat(val+",", dim), ",") + "]" +} + +func TestStoreRoundTrip(t *testing.T) { + conn, q := setupTestDB(t) + ctx := context.Background() uid := fmt.Sprintf("test_%d", time.Now().UnixNano()) - emb := "[" + strings.TrimSuffix(strings.Repeat("0.0,", 1024), ",") + "]" + platform := "Bluesky" + emb1 := makeEmbedding(1024, "0.1") - if err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ + // Clean up after the test so leftover records do not pollute subsequent runs + t.Cleanup(func() { + _, _ = conn.ExecContext(context.Background(), "DELETE FROM style_profiles WHERE user_id = $1", uid) + }) + + // 1. Test INSERT path of Upsert + err := q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ UserID: uid, - Platform: "Bluesky", - Embedding: emb, + Platform: platform, + Embedding: emb1, TopKeywords: []string{"alpha", "beta"}, - StyleSummary: "summary", + StyleSummary: "original summary", PostCount: 5, - }); err != nil { - t.Fatalf("upsert: %v", err) + }) + if err != nil { + t.Fatalf("initial upsert failed: %v", err) } - p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: "Bluesky"}) + // 2. Test GET and verify fields + p, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) if err != nil { - t.Fatalf("get: %v", err) - } - if p.UserID != uid || p.Platform != "Bluesky" || p.PostCount != 5 { - t.Fatalf("unexpected profile: %+v", p) + t.Fatalf("get failed: %v", err) } - rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{QueryEmbedding: emb, LimitVal: 10}) - if err != nil { - t.Fatalf("find similar: %v", err) + if p.UserID != uid || p.Platform != platform || p.PostCount != 5 { + t.Errorf("mismatched basic fields: %+v", p) } + if p.StyleSummary != "original summary" { + t.Errorf("got summary %q, want %q", p.StyleSummary, "original summary") + } + if !reflect.DeepEqual(p.TopKeywords, []string{"alpha", "beta"}) { + t.Errorf("got keywords %v, want %v", p.TopKeywords, []string{"alpha", "beta"}) + } + + // 3. Test UPDATE path of Upsert (Verify conflict resolution works) + emb2 := makeEmbedding(1024, "0.2") + err = q.UpsertStyleProfile(ctx, UpsertStyleProfileParams{ + UserID: uid, + Platform: platform, + Embedding: emb2, + TopKeywords: []string{"gamma"}, + StyleSummary: "updated summary", + PostCount: 10, + }) + if err != nil { + t.Fatalf("secondary upsert (update) failed: %v", err) + } + + updated, err := q.GetStyleProfile(ctx, GetStyleProfileParams{UserID: uid, Platform: platform}) + if err != nil { + t.Fatalf("get updated profile failed: %v", err) + } + if updated.PostCount != 10 || updated.StyleSummary != "updated summary" { + t.Errorf("upsert failed to update existing record: %+v", updated) + } + + // 4. Test pgvector similarity search + rows, err := q.FindSimilarUsers(ctx, FindSimilarUsersParams{ + QueryEmbedding: emb2, + LimitVal: 10, + }) + if err != nil { + t.Fatalf("find similar failed: %v", err) + } + found := false for _, r := range rows { if r.UserID == uid { found = true + break } } if !found { - t.Fatalf("expected newly inserted profile to be returned by similarity search") + t.Errorf("expected user %s in similarity results, but was not found", uid) } } -// TestStoreSaveTransaction exercises SaveTransaction against a real Postgres database. -// It is skipped unless TEST_DATABASE_URL is set. func TestStoreSaveTransaction(t *testing.T) { - dsn := os.Getenv("TEST_DATABASE_URL") - if dsn == "" { - t.Skip("TEST_DATABASE_URL not set; skipping store integration test") - } + conn, q := setupTestDB(t) ctx := context.Background() - conn, err := sql.Open("postgres", dsn) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - if err := conn.PingContext(ctx); err != nil { - t.Fatalf("cannot reach test database: %v", err) - } - if err := Migrate(ctx, conn, "sql/schema"); err != nil { - t.Fatalf("migrate: %v", err) - } - q := New(conn) - txHash := fmt.Sprintf("0x_%d", time.Now().Unix()) + txHash := fmt.Sprintf("0x_%d", time.Now().UnixNano()) amount := "10.0" asset := "ETH" network := "eip155:1" - now := time.Now().Unix() - recipient := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000), "0") - sender := "0x" + strings.TrimLeft(fmt.Sprintf("%x", now%100000000+0xab), "0") + recipient := "0x1234567890abcdef" + sender := "0xabcdef1234567890" + + t.Cleanup(func() { + _, _ = conn.ExecContext(context.Background(), "DELETE FROM transactions WHERE x402_transaction_hash = $1", txHash) + }) ret, err := q.SaveTransaction(ctx, SaveTransactionParams{ X402TransactionHash: txHash, @@ -113,22 +161,34 @@ func TestStoreSaveTransaction(t *testing.T) { } if len(ret) != 1 { - t.Fatalf("expected 1 result, got %d", len(ret)) + t.Fatalf("expected 1 result row returned, got %d", len(ret)) } - id := ret[0].ID - if id <= 0 { - t.Fatalf("expected positive ID, got %d", id) + saved := ret[0] + if saved.ID <= 0 { + t.Errorf("expected positive ID, got %d", saved.ID) + } + if !saved.CreatedAt.Valid { + t.Error("expected CreatedAt to be set") } - // Verify the returned row has the expected fields - if !ret[0].CreatedAt.Valid { - t.Fatal("expected CreatedAt to be set") + // Since the sqlc query only returns `id` and `created_at`, + // verify directly that PostgreSQL stored the remaining columns accurately: + var ( + gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender string + ) + err = conn.QueryRowContext(ctx, ` + SELECT x402_transaction_hash, amount, asset, network, recipient, sender + FROM transactions + WHERE id = $1 + `, saved.ID).Scan(&gotHash, &gotAmount, &gotAsset, &gotNetwork, &gotRecipient, &gotSender) + if err != nil { + t.Fatalf("failed to query inserted transaction from db: %v", err) } - // Verify we can retrieve the transaction back - // Note: we'd need a SELECT query for full verification, but the insert - // returning id and created_at is the critical path tested here - - t.Logf("SaveTransaction successful: id=%d, hash=%s", id, txHash) + if gotHash != txHash || gotAmount != amount || gotAsset != asset || + gotNetwork != network || gotRecipient != recipient || gotSender != sender { + t.Errorf("stored values do not match input params: got (%s, %s, %s, %s, %s, %s)", + gotHash, gotAmount, gotAsset, gotNetwork, gotRecipient, gotSender) + } } From debd49d6f683637ae651aaa6248d2b0356cdc626 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:29:28 +0000 Subject: [PATCH 08/30] chore: quiet errcheck/staticcheck across server, platform, and test clients --- internal/server/analytics.go | 8 ++++---- internal/server/platform.go | 2 +- testClient/mcp/test-mcp-endpoints.go | 4 ++-- testClient/rest/test-rest-endpoints.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/server/analytics.go b/internal/server/analytics.go index ea0faf7..cc95d61 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -152,7 +152,7 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to return nil, fmt.Errorf("invalid time range: %w", err) } - platforms := []string{} + var platforms []string if args.Platform != "" { platforms = []string{args.Platform} } else { @@ -175,7 +175,7 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if cached, ok := s.getAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash); ok { var cachedResult json.RawMessage - json.Unmarshal(cached, &cachedResult) + _ = json.Unmarshal(cached, &cachedResult) return s.toToolResult(cachedResult) } @@ -206,7 +206,7 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to // Cache the result under the matching scope key. ttl := s.getCacheTTL(toolName) - s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) + _ = s.setAnalyticsCache(ctx, args.Author, cachePlatform, toolName, timeRangeHash, result, ttl) return s.toToolResult(result) } @@ -911,7 +911,7 @@ Texts: Neutral float64 `json:"neutral"` Emotions map[string]int `json:"emotions"` } - json.Unmarshal([]byte(resp), &result) + _ = json.Unmarshal([]byte(resp), &result) // Normalize sum := result.Positive + result.Negative + result.Neutral diff --git a/internal/server/platform.go b/internal/server/platform.go index 6dcb2bf..a0330cd 100644 --- a/internal/server/platform.go +++ b/internal/server/platform.go @@ -26,7 +26,7 @@ func determinePlatform(author string) (platform, error) { if len(author) == 64 { isHex := true for _, c := range author { - if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { isHex = false break } diff --git a/testClient/mcp/test-mcp-endpoints.go b/testClient/mcp/test-mcp-endpoints.go index b6668e8..79b35cb 100644 --- a/testClient/mcp/test-mcp-endpoints.go +++ b/testClient/mcp/test-mcp-endpoints.go @@ -87,7 +87,7 @@ func getEnv(key, fallback string) string { } func main() { - godotenv.Load() + _ = godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080") @@ -125,7 +125,7 @@ func main() { if err != nil { log.Fatal("Failed to connect MCP client:", err) } - defer session.Close() + defer func() { _ = session.Close() }() x402Mcp := x402mcp.NewX402MCPClientFromConfig(session, []x402mcp.SchemeRegistration{ {Network: x402.Network("eip155:84532"), Client: evm.NewExactEvmScheme(signer, nil)}, diff --git a/testClient/rest/test-rest-endpoints.go b/testClient/rest/test-rest-endpoints.go index f048e29..c9e9f7b 100644 --- a/testClient/rest/test-rest-endpoints.go +++ b/testClient/rest/test-rest-endpoints.go @@ -70,7 +70,7 @@ func callEndpoint(client *http.Client, method, endpoint, body string) error { if err != nil { return fmt.Errorf("%s HTTP request failed: %w", endpoint, err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() respBody, _ := io.ReadAll(resp.Body) fmt.Printf("%s Response Status: %s\n", endpoint, resp.Status) @@ -124,7 +124,7 @@ func getEnv(key, fallback string) string { } func main() { - godotenv.Load() + _ = godotenv.Load() privateKey := os.Getenv("PRIVATE_KEY") serverURL := getEnv("TEST_SERVER_URL", "http://localhost:8080") From 8c561a83a20b9131ef0322609df5c11270719c46 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:31:50 +0000 Subject: [PATCH 09/30] chore: idempotent migrations (IF NOT EXISTS) + systemd binary name fix --- mcpServer.service | 2 +- sql/schema/003_add_cache_version.sql | 6 +++--- sql/schema/004_add_style_posts.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mcpServer.service b/mcpServer.service index d9e183e..d852b7b 100644 --- a/mcpServer.service +++ b/mcpServer.service @@ -7,7 +7,7 @@ User=mcpserver Group=mcpserver WorkingDirectory=/home/ubuntu/gitSources/mcpServer -ExecStart=/home/ubuntu/gitSources/mcpServer/bin/main +ExecStart=/home/ubuntu/gitSources/mcpServer/bin/mcpserver RuntimeDirectory=mcpserver RuntimeDirectoryMode=0750 diff --git a/sql/schema/003_add_cache_version.sql b/sql/schema/003_add_cache_version.sql index f5d2496..4fdbf1e 100644 --- a/sql/schema/003_add_cache_version.sql +++ b/sql/schema/003_add_cache_version.sql @@ -1,8 +1,8 @@ -- +goose up ALTER TABLE user_style_profiles - ADD COLUMN model_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN schema_version INTEGER NOT NULL DEFAULT 1, - ADD COLUMN algorithm_version INTEGER NOT NULL DEFAULT 1; + ADD COLUMN IF NOT EXISTS model_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS schema_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS algorithm_version INTEGER NOT NULL DEFAULT 1; CREATE INDEX IF NOT EXISTS idx_user_style_profiles_versions ON user_style_profiles (model_version, schema_version, algorithm_version); diff --git a/sql/schema/004_add_style_posts.sql b/sql/schema/004_add_style_posts.sql index cc020c1..ae5bf08 100644 --- a/sql/schema/004_add_style_posts.sql +++ b/sql/schema/004_add_style_posts.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS user_style_posts ( UNIQUE (user_id, platform, post_id) ); -CREATE INDEX idx_user_style_posts_user_platform +CREATE INDEX IF NOT EXISTS idx_user_style_posts_user_platform ON user_style_posts (user_id, platform); -- +goose down From 1e32bb752fd0a349bdd81ebfa17db5d6b690dd31 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:35:55 +0000 Subject: [PATCH 10/30] chore: remove dead post-pruner goroutine and rescueFunds.py script --- cdpScripts/rescueFunds.py | 25 ------------------------- internal/server/prewarm.go | 10 ---------- internal/server/server.go | 2 +- 3 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 cdpScripts/rescueFunds.py diff --git a/cdpScripts/rescueFunds.py b/cdpScripts/rescueFunds.py deleted file mode 100644 index 18ef5fd..0000000 --- a/cdpScripts/rescueFunds.py +++ /dev/null @@ -1,25 +0,0 @@ -from cdp import cdp_client -import asyncio -import inspect -from dotenv import load_dotenv - -load_dotenv() - -OLD = "0x3C17021dF1071b7F45cd3D1e4Db0bcBd03C6ECF4" -NETWORK = "base-sepolia" - - -async def main(): - cdp = cdp_client.CdpClient() - - try: - private_key = await cdp.evm.export_account( - name="api-naco-li" - ) - print(f"Private key: {private_key}") - - finally: - await cdp.close() - - -asyncio.run(main()) \ No newline at end of file diff --git a/internal/server/prewarm.go b/internal/server/prewarm.go index 55caf69..1b5cdf5 100644 --- a/internal/server/prewarm.go +++ b/internal/server/prewarm.go @@ -89,13 +89,3 @@ func (s *Server) prewarmOne(ctx context.Context, author string) { } log.InfoWithContext(ctx, "prewarmed style profile", "author", author) } - -func (s *Server) startPostPruner() { - ticker := time.NewTicker(24 * time.Hour) - defer ticker.Stop() - for range ticker.C { - if err := s.db.PruneStylePosts(context.Background()); err != nil { - log.Error("failed to prune style posts", "error", err) - } - } -} diff --git a/internal/server/server.go b/internal/server/server.go index ef4508b..4b0cbfb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -104,6 +104,6 @@ func (s *Server) MetricsText() string { } func (s *Server) Start() error { - go s.startPostPruner() + //go s.startPostPruner() return s.mcpHandler.Start(s.listenAddress, s.permissions) } From 9a45dbc2c6287913918c1bcbf1ad13b036589431 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Tue, 8 Sep 2026 01:44:36 +0000 Subject: [PATCH 11/30] refactor: complete pointer-ify in remaining test fakes and analytics converter --- internal/server/mcp_test.go | 12 ++++++------ internal/server/posts_convert.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 36a8dc6..40d15b7 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -763,10 +763,10 @@ func TestSearchActorsLimitClamp(t *testing.T) { func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { // Twitter is not supported for search in the current implementation - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "a"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} - nstr := &fakeNostr{actors: []actor.Profile{{Platform: "Nostr", Handle: "c"}}} - tw := &fakeTwitter{actors: []actor.Profile{{Platform: "Twitter", Handle: "d"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "a"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "b"}}} + nstr := &fakeNostr{actors: []*actor.Profile{{Platform: "Nostr", Handle: "c"}}} + tw := &fakeTwitter{actors: []*actor.Profile{{Platform: "Twitter", Handle: "d"}}} s := newTestServer(bsky, ap, nstr, tw) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "test", Limit: 10}) if err != nil { @@ -784,8 +784,8 @@ func TestSearchActorsTwitterExcludedFromSearch(t *testing.T) { func TestSearchActorsNormalizationAndDedup(t *testing.T) { // Test that handles normalize correctly and deduplicate - bsky := &fakeBluesky{actors: []actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} - ap := &fakeAP{actors: []actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} + bsky := &fakeBluesky{actors: []*actor.Profile{{Platform: "Bluesky", Handle: "user.bsky.social", DisplayName: "User"}}} + ap := &fakeAP{actors: []*actor.Profile{{Platform: "ActivityPub", Handle: "user@mastodon.social", DisplayName: "User"}}} s := newTestServer(bsky, ap, &fakeNostr{}, &fakeTwitter{}) res, err := callTool(s, "searchActors", SearchActorsRequest{Query: "user", Limit: 10}) if err != nil { diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index c9bc0db..959fccb 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -217,7 +217,7 @@ func blueskyFeedViewsToAnalyticsItems(views []*bsky.FeedDefs_FeedViewPost) []clu return out } -func apPostsToAnalyticsItems(posts []activitypub.Post) []clustering.Item { +func apPostsToAnalyticsItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { if p.Text != "" { From e68e02bf20ec7d747681d6dabcf6cb6edadd1b9c Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 13:27:17 +0000 Subject: [PATCH 12/30] feat: data-driven upto scheme registration for twitter-enabled tools Extend x402 actual-cost settlement beyond getStyleProfile to every Twitter-capable tool (findSimilarUsers, getEngagementMetrics, getPostingPatterns, getTopicModeling, getSentimentTrends, getEngagementDrivers). Each carries a twitterPostCount (or source platform + post count) in its response so settlement passes on the realized Twitter API lookup and per-post costs. - tools: payableDef uses UsesTwitter + TwitterPostCap instead of a fixed Scheme; registerTools advertises upto at the cap-derived maximum when Twitter is enabled, otherwise exact at the flat base price. - x402: export pricing constants and MaxTwitterCostMicro / FormatPriceMicro / MicroFromPriceString helpers; add AddPayableToolWithResolver and a per-tool settleResolvers map used by both the MCP wrapper and REST override; buildSchemeConfigs registers only the schemes declared in a tool's requirements. - analytics: default aggregates include Twitter only when enabled; result count recorded before caching and returned as twitterPostCount. - server/twitterEnabled gates the upto switch (config bearer token). --- internal/server/analytics.go | 43 +++++- internal/server/analytics_test.go | 49 +++++++ internal/server/server.go | 7 + internal/server/style.go | 9 +- internal/server/styleprofile_test.go | 55 +++++++ internal/server/tools.go | 60 ++++++-- internal/server/types.go | 24 ++++ internal/x402/x402.go | 207 ++++++++++++++++++++------- internal/x402/x402_test.go | 144 ++++++++++++++++++- 9 files changed, 519 insertions(+), 79 deletions(-) diff --git a/internal/server/analytics.go b/internal/server/analytics.go index cc95d61..6db10e1 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -156,10 +156,15 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if args.Platform != "" { platforms = []string{args.Platform} } else { - // Twitter is included by default - the API costs it incurs are passed - // on to the caller through the paid tool pricing, so aggregating over - // it does not run the server at a loss. - platforms = []string{"Twitter", "Bluesky", "ActivityPub", "Nostr"} + // Twitter is included by default only when the API is enabled: the API + // costs it incurs are passed on to the caller through the upto pricing + // of these tools (see registerTools), so aggregating over it does not + // run the server at a loss. When no bearer token is configured there is + // no Twitter to aggregate over. + platforms = []string{"Bluesky", "ActivityPub", "Nostr"} + if s.twitterEnabled { + platforms = append(platforms, "Twitter") + } } timeRangeHash := hashTimeRange(start, end, limit) @@ -181,6 +186,7 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to // Fetch posts from all platforms var allPosts []clustering.Item + twitterPosts := 0 for _, p := range platforms { plat, ok := platformFromString(p) if !ok { @@ -192,6 +198,9 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to continue } allPosts = append(allPosts, posts...) + if plat == Twitter { + twitterPosts += len(posts) + } } if len(allPosts) == 0 { @@ -203,6 +212,7 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to if err != nil { return nil, err } + result = setAnalyticsTwitterPostCount(result, twitterPosts) // Cache the result under the matching scope key. ttl := s.getCacheTTL(toolName) @@ -211,6 +221,31 @@ func (s *Server) computeAnalytics(ctx context.Context, args AnalyticsRequest, to return s.toToolResult(result) } +// setAnalyticsTwitterPostCount records how many posts in an analytics result +// were sourced from the Twitter API, so upto-scheme settlement can pass the +// realized Twitter costs on to the caller. Unknown result types are returned +// unchanged. +func setAnalyticsTwitterPostCount(result any, count int) any { + switch v := result.(type) { + case EngagementMetricsResponse: + v.TwitterPostCount = count + return v + case PostingPatternsResponse: + v.TwitterPostCount = count + return v + case TopicModelingResponse: + v.TwitterPostCount = count + return v + case SentimentTrendsResponse: + v.TwitterPostCount = count + return v + case EngagementDriversResponse: + v.TwitterPostCount = count + return v + } + return result +} + func (s *Server) fetchAnalyticsPosts(ctx context.Context, platform platform, author string, start, end time.Time, limit int) ([]clustering.Item, error) { posts, err := s.fetchPostsInTimeForAnalytics(ctx, platform, author, start, end, limit) if err != nil { diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index d96a439..dfb8e9a 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -9,6 +9,7 @@ import ( "github.com/nacorid/naco-api/internal/activitypub" "github.com/nacorid/naco-api/internal/clustering" + "github.com/nacorid/naco-api/internal/twitter" ) var errSome = fmt.Errorf("some error") @@ -141,6 +142,54 @@ func TestAnalyticsPartialPlatformFailureTolerated(t *testing.T) { } } +// TestAnalyticsTwitterPostCount verifies the response reports how many posts +// were sourced from the Twitter API, so upto-scheme settlement can charge the +// realized Twitter API costs. +func TestAnalyticsTwitterPostCount(t *testing.T) { + tw := &fakeTwitter{posts: []*twitter.Post{ + {ID: "t1", Content: "first tweet", CreatedAt: time.Date(2024, 1, 11, 12, 0, 0, 0, time.UTC)}, + {ID: "t2", Content: "second tweet", CreatedAt: time.Date(2024, 1, 12, 12, 0, 0, 0, time.UTC)}, + {ID: "t3", Content: "third tweet", CreatedAt: time.Date(2024, 1, 13, 12, 0, 0, 0, time.UTC)}, + }} + st := newFakeStore() + s := newServerWith(&fakeBluesky{}, analyticsAPPosts(), &fakeNostr{}, tw, st, &fakeOllama{}) + + res, err := callTool(s, "getEngagementMetrics", AnalyticsRequest{ + Author: "@alice", + StartTime: "2024-01-01T00:00:00Z", + EndTime: "2024-12-31T00:00:00Z", + Platform: "Twitter", + }) + if err != nil { + t.Fatalf("getEngagementMetrics error: %v", err) + } + var out EngagementMetricsResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 3 { + t.Fatalf("expected TwitterPostCount 3, got %d", out.TwitterPostCount) + } + + // A non-Twitter platform reports 0 and the default aggregate excludes + // Twitter when the API is not enabled (twitterEnabled=false in tests). + res, err = callTool(s, "getEngagementMetrics", AnalyticsRequest{ + Author: "@a@mastodon.social", + StartTime: "2024-01-01T00:00:00Z", + EndTime: "2024-12-31T00:00:00Z", + Platform: "ActivityPub", + }) + if err != nil { + t.Fatalf("getEngagementMetrics error: %v", err) + } + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 0 { + t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out.TwitterPostCount) + } +} + // ---- pure compute functions ---- func TestComputePostingPatterns(t *testing.T) { diff --git a/internal/server/server.go b/internal/server/server.go index 4b0cbfb..d9da9c2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -48,6 +48,12 @@ type Server struct { analyticsTTLTopics time.Duration analyticsTTLSentiment time.Duration analyticsTTLDrivers time.Duration + + // twitterEnabled reports whether the Twitter API is available (a bearer + // token is configured). It flips the Twitter-capable tools to upto + // variable pricing so the realized Twitter API costs are passed on to + // callers (see registerTools). + twitterEnabled bool } func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky *bluesky.BlueskyClient, n *nostr.Client, t *twitter.Client, o *ollama.OllamaClient, cfg *config.Config) *Server { @@ -78,6 +84,7 @@ func New(ctx context.Context, db *store.Queries, ap *activitypub.APClient, bsky analyticsTTLTopics: cfg.AnalyticsTTLTopics, analyticsTTLSentiment: cfg.AnalyticsTTLSentiment, analyticsTTLDrivers: cfg.AnalyticsTTLDrivers, + twitterEnabled: cfg.TwitterBearerToken != "", } // Combine address and port for TCP (not unix socket) if port not already present diff --git a/internal/server/style.go b/internal/server/style.go index 4001d00..8015d50 100644 --- a/internal/server/style.go +++ b/internal/server/style.go @@ -193,7 +193,14 @@ func (s *Server) getSimilarProfiles(ctx context.Context, request *mcp.CallToolRe }) } - respBytes, err := json.Marshal(FindSimilarUsersResponse{Matches: matches}) + twitterPostCount := 0 + if profile.SourcePlatform == "Twitter" { + twitterPostCount = profile.PostCount + } + respBytes, err := json.Marshal(FindSimilarUsersResponse{ + Matches: matches, + TwitterPostCount: twitterPostCount, + }) if err != nil { return nil, err } diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index 49631c8..07fb194 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -304,6 +304,61 @@ func TestFindSimilarUsersEmptyMatches(t *testing.T) { } } +// TestFindSimilarUsersTwitterPostCount verifies the response reports how many +// posts went into the query author's style profile when that author is on +// Twitter, so upto-scheme settlement can charge the realized Twitter API costs. +func TestFindSimilarUsersTwitterPostCount(t *testing.T) { + st := newFakeStore() + st.similarRows = []store.FindSimilarUsersRow{ + {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, + } + // Serve a fresh cached Twitter profile with 42 analyzed posts. + st.styleProfiles["@alice|Twitter"] = store.UserStyleProfile{ + UserID: "@alice", + Platform: "Twitter", + Embedding: "[0.1,0.2]", + PostCount: 42, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + } + s := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) + res, err := callTool(s, "findSimilarUsers", FindSimilarUsersRequest{Author: "@alice"}) + if err != nil { + t.Fatalf("findSimilarUsers error: %v", err) + } + var out FindSimilarUsersResponse + if err := jsonUnmarshal(t, textOf(t, res), &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out.TwitterPostCount != 42 { + t.Fatalf("expected TwitterPostCount 42, got %d", out.TwitterPostCount) + } + + // A non-Twitter query author reports 0. + st2 := newFakeStore() + st2.similarRows = []store.FindSimilarUsersRow{ + {UserID: "did:plc:other", Platform: "Bluesky", Similarity: 80}, + } + st2.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ + UserID: "@a@mastodon.social", + Platform: "ActivityPub", + Embedding: "[0.1,0.2]", + PostCount: 9, + ModifiedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + } + s2 := newServerWith(&fakeBluesky{}, &fakeAP{}, &fakeNostr{}, &fakeTwitter{}, st2, &fakeOllama{}) + res2, err := callTool(s2, "findSimilarUsers", FindSimilarUsersRequest{Author: "@a@mastodon.social"}) + if err != nil { + t.Fatalf("findSimilarUsers error: %v", err) + } + var out2 FindSimilarUsersResponse + if err := jsonUnmarshal(t, textOf(t, res2), &out2); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if out2.TwitterPostCount != 0 { + t.Fatalf("expected TwitterPostCount 0 for ActivityPub, got %d", out2.TwitterPostCount) + } +} + func TestFindSimilarUsersDBError(t *testing.T) { st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{ diff --git a/internal/server/tools.go b/internal/server/tools.go index e6cecb9..9767aeb 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -7,6 +7,7 @@ import ( "time" "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/nacorid/naco-api/internal/x402" ) // registerTools registers all MCP tools with their x402 payment requirements. @@ -42,8 +43,19 @@ func (s *Server) registerTools() { Output any Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) Price string - Scheme string - Extra map[string]any + // UsesTwitter marks tools that may read posts (or set up lookups) + // through the Twitter API and return only transformative/aggregated + // results, as permitted by the X API policy. With Twitter enabled these + // tools charge upto-scheme: base price plus the realized Twitter API + // costs. Tools that resell raw post/profile/feed data (getPost, + // getLatestPosts, ...) never set this. + UsesTwitter bool + // TwitterPostCap is the post count used to compute the advertised upto + // maximum (see x402.MaxTwitterCostMicro). It only bounds the largest + // settlement the tool can demand; the actual charge is settled from the + // response. + TwitterPostCap int64 + Extra map[string]any } payable := []payableDef{ @@ -93,12 +105,15 @@ func (s *Server) registerTools() { ), }, { - // 0.76 is the maximum authorization for a style profile: $0.25 base + $0.01 - // Twitter lookup + $0.005/post at the 100-post cap (see maxProfileCostMicro in - // internal/x402). The actual charge is computed from posts read and settled via - // an override, so this must stay in sync with that constant. + // base price is $0.25 (x402.BaseProfileCostMicro). With Twitter + // enabled the tool is advertised as upto at the maximum possible + // cost - $0.25 base + $0.01 lookup + $0.005/post at the 100-post + // cap = $0.76 (x402.MaxTwitterCostMicro(BaseProfileCostMicro, + // MaxProfileCountedPosts)) - and the actual charge is settled from + // the posts read via x402.NewStyleProfileCostResolver. Name: "getStyleProfile", Desc: "Get an actor's style Profile", - Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.76", Scheme: "upto", + Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.25", + UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -114,6 +129,7 @@ func (s *Server) registerTools() { { Name: "findSimilarUsers", Desc: "Find authors with a similar writing style to the given author, across all platforms", Input: FindSimilarUsersRequest{}, Output: FindSimilarUsersResponse{}, Handler: s.getSimilarProfiles, Price: "0.10", + UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -151,6 +167,7 @@ func (s *Server) registerTools() { { Name: "getEngagementMetrics", Desc: "Get time-series engagement metrics (likes, reposts, replies, quotes) per post", Input: AnalyticsRequest{}, Output: EngagementMetricsResponse{}, Handler: s.getEngagementMetrics, Price: "0.15", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "metrics": Bodyfield{Type: "array", Description: "Engagement metrics per post"}, @@ -159,6 +176,7 @@ func (s *Server) registerTools() { { Name: "getPostingPatterns", Desc: "Get posting heatmap and patterns (hour-of-day x day-of-week)", Input: AnalyticsRequest{}, Output: PostingPatternsResponse{}, Handler: s.getPostingPatterns, Price: "0.10", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "heatmap": Bodyfield{Type: "array", Description: "24x7 matrix: hour (0-23) x day of week (0-6 = Sun-Sat)"}, @@ -172,6 +190,7 @@ func (s *Server) registerTools() { { Name: "getTopicModeling", Desc: "Get LLM-based topic extraction and clustering from posts", Input: AnalyticsRequest{}, Output: TopicModelingResponse{}, Handler: s.getTopicModeling, Price: "0.20", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "topics": Bodyfield{Type: "array", Description: "Identified topic clusters"}, @@ -180,6 +199,7 @@ func (s *Server) registerTools() { { Name: "getSentimentTrends", Desc: "Get sentiment analysis and emotional arcs over time", Input: AnalyticsRequest{}, Output: SentimentTrendsResponse{}, Handler: s.getSentimentTrends, Price: "0.15", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "timeSeries": Bodyfield{Type: "array", Description: "Sentiment over time"}, @@ -191,6 +211,7 @@ func (s *Server) registerTools() { { Name: "getEngagementDrivers", Desc: "Get topics/keywords that correlate with high engagement", Input: AnalyticsRequest{}, Output: EngagementDriversResponse{}, Handler: s.getEngagementDrivers, Price: "0.20", + UsesTwitter: true, TwitterPostCap: x402.MaxAnalyticsCountedPosts, Extra: schemaExtra(analyticsInput, map[string]any{ "author": Bodyfield{Type: "string", Description: "Author identifier"}, "drivers": Bodyfield{Type: "array", Description: "Topics/keywords driving engagement"}, @@ -199,14 +220,29 @@ func (s *Server) registerTools() { } for _, d := range payable { - scheme := d.Scheme - if scheme == "" { - scheme = "exact" + // Twitter-capable tools are advertised as upto (variable-priced) only + // when the Twitter API is actually enabled: the upto maximum adds the + // realized Twitter lookup and per-post costs, and the actual charge is + // settled from the response. Without Twitter, every tool is exact at + // its flat base price. + scheme := "exact" + price := d.Price + var resolver x402.SettleAmountResolver + if d.UsesTwitter && s.twitterEnabled { + scheme = "upto" + baseMicro := x402.MicroFromPriceString(d.Price) + price = x402.FormatPriceMicro(x402.MaxTwitterCostMicro(baseMicro, d.TwitterPostCap)) + if d.Name == "getStyleProfile" { + resolver = x402.NewStyleProfileCostResolver() + } else { + resolver = x402.NewTwitterCountCostResolver(baseMicro) + } } - s.mcpHandler.AddPayableTool( + s.mcpHandler.AddPayableToolWithResolver( mcp.Tool{Name: d.Name, Description: d.Desc, InputSchema: d.Input, OutputSchema: d.Output}, d.Handler, - s.buildRequirements(d.Price, d.Extra, scheme)..., + resolver, + s.buildRequirements(price, d.Extra, scheme)..., ) } diff --git a/internal/server/types.go b/internal/server/types.go index 4e1733e..7d833c6 100644 --- a/internal/server/types.go +++ b/internal/server/types.go @@ -87,6 +87,10 @@ type FindSimilarUsersRequest struct { type FindSimilarUsersResponse struct { Matches []SimilarUser `json:"matches" jsonschema_description:"Ranked list of similar profiles by persona embedding similarity"` + // TwitterPostCount reports how many posts were sourced from the Twitter API + // while building the query author's style profile. It is 0 when the query + // author is not on Twitter. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts used for the query author's profile"` } type ActorProfile = actor.Profile @@ -165,6 +169,10 @@ type AnalyticsRequest struct { type EngagementMetricsResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Metrics []EngagementMetricPoint `json:"metrics" jsonschema_description:"Engagement metrics per post"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementMetricPoint struct { @@ -185,11 +193,19 @@ type PostingPatternsResponse struct { AvgPostsPerDay float64 `json:"avgPostsPerDay" jsonschema_description:"Average posts per day"` TotalPosts int `json:"totalPosts" jsonschema_description:"Total posts in time range"` PostingDays int `json:"postingDays" jsonschema_description:"Number of days with at least one post"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicModelingResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Topics []TopicCluster `json:"topics" jsonschema_description:"Identified topic clusters"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type TopicCluster struct { @@ -207,6 +223,10 @@ type SentimentTrendsResponse struct { OverallSentiment SentimentSummary `json:"overallSentiment" jsonschema_description:"Overall sentiment summary"` DominantEmotions []EmotionScore `json:"dominantEmotions" jsonschema_description:"Most frequent emotions"` SentimentShifts []SentimentShift `json:"sentimentShifts" jsonschema_description:"Significant sentiment changes"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type SentimentPoint struct { @@ -238,6 +258,10 @@ type SentimentShift struct { type EngagementDriversResponse struct { Author string `json:"author" jsonschema_description:"Author identifier"` Drivers []EngagementDriver `json:"drivers" jsonschema_description:"Topics/keywords driving engagement"` + // TwitterPostCount reports how many posts in the result were sourced from + // the Twitter API, so upto-scheme settlement can pass the realized Twitter + // costs on to the caller. + TwitterPostCount int `json:"twitterPostCount" jsonschema_description:"Number of Twitter-sourced posts analyzed"` } type EngagementDriver struct { diff --git a/internal/x402/x402.go b/internal/x402/x402.go index c2d5525..1884946 100644 --- a/internal/x402/x402.go +++ b/internal/x402/x402.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log/slog" + "math" "net" "net/http" "os" @@ -58,6 +59,11 @@ type X402Server struct { resourceInitMu sync.Mutex resourceInitDone bool + // settleResolvers maps payable tool name -> SettleAmountResolver used by + // both the MCP wrapper and the REST override path to charge the actual + // computed cost (instead of the advertised upto maximum). + settleResolvers map[string]SettleAmountResolver + httpServer *http.Server mu sync.Mutex } @@ -113,6 +119,7 @@ func NewX402Server(name, version string, facilitatorURL string, auth x402http.Au tools: make([]ToolInfo, 0), globalReqLog: globalLoggingMw, networks: networks, + settleResolvers: make(map[string]SettleAmountResolver), } // Warm the facilitator cache now so AddPayableTool can build Accepts with @@ -195,6 +202,29 @@ func (s *X402Server) AddPayableTool( handler mcp.ToolHandler, requirements ...x402.PaymentRequirements, ) { + s.addPayableTool(tool, handler, nil, requirements...) +} + +// AddPayableToolWithResolver registers a payable tool with a +// SettleAmountResolver. The resolver is used by both the MCP wrapper and the +// REST override path to settle the actual computed cost after the handler runs +// (e.g. an upto-scheme tool charging Twitter API pass-through based on the +// response), instead of the advertised maximum. +func (s *X402Server) AddPayableToolWithResolver( + tool mcp.Tool, + handler mcp.ToolHandler, + resolver SettleAmountResolver, + requirements ...x402.PaymentRequirements, +) { + s.addPayableTool(tool, handler, resolver, requirements...) +} + +func (s *X402Server) addPayableTool( + tool mcp.Tool, + handler mcp.ToolHandler, + settleResolver SettleAmountResolver, + requirements ...x402.PaymentRequirements, +) { if len(requirements) == 0 { panic(fmt.Sprintf("tool %s requires at least one payment requirement", tool.Name)) @@ -223,25 +253,8 @@ func (s *X402Server) AddPayableTool( tool.InputSchema = toJSONSchema(tool.InputSchema) tool.OutputSchema = toJSONSchema(tool.OutputSchema) - // getStyleProfile bills by actual posts read (see computeTwitterStyleProfileCost). - // The MCP wrapper then settles the computed cost instead of the advertised maximum, - // mirroring the REST path's nethttp.SetSettlementOverrides behavior. - var settleResolver SettleAmountResolver - if tool.Name == "getStyleProfile" { - settleResolver = func(result *mcp.CallToolResult) (string, bool) { - if len(result.Content) != 1 { - return "", false - } - textContent, ok := result.Content[0].(*mcp.TextContent) - if !ok { - return "", false - } - var js json.RawMessage - if json.Unmarshal([]byte(textContent.Text), &js) != nil { - return "", false - } - return computeTwitterStyleProfileCost(nil, textContent.Text), true - } + if settleResolver != nil { + s.settleResolvers[tool.Name] = settleResolver } wrapper := NewPaymentWrapper(s.x402ResourceServer, PaymentWrapperConfig{ @@ -272,20 +285,7 @@ func (s *X402Server) AddPayableTool( }) } - schemes := make([]nethttp.SchemeConfig, 0, len(s.networks)*2) - for _, net := range s.networks { - // For getStyleProfile, only register upto scheme (variable pricing) - if tool.Name == "getStyleProfile" { - schemes = append(schemes, - nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, - ) - } else { - schemes = append(schemes, - nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}, - nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}, - ) - } - } + schemes := buildSchemeConfigs(s.networks, requirements...) settlementHandler := func(w http.ResponseWriter, r *http.Request, resp *x402.SettleResponse) { requirements, _ := nethttp.RequirementsFromContext(r.Context()) @@ -380,6 +380,29 @@ func (s *X402Server) AddPayableTool( s.mux.Handle("POST /"+VERSION1+tool.Name, s.middlewareChain(paymentMiddleware(restHandler))) } +// buildSchemeConfigs maps the schemes actually declared in a tool's payment +// requirements to the EVM scheme servers a route needs to verify and settle +// them, for every configured network. Only the schemes present in the +// requirements are registered: an exact-only tool does not expose an upto +// scheme, and vice versa. This keeps upto (variable-pricing) authorization from +// being offered for tools that cannot vary their price. +func buildSchemeConfigs(networks []string, requirements ...x402.PaymentRequirements) []nethttp.SchemeConfig { + present := make(map[string]struct{}) + for _, req := range requirements { + present[req.Scheme] = struct{}{} + } + schemes := make([]nethttp.SchemeConfig, 0, len(networks)*len(present)) + for _, net := range networks { + if _, ok := present["exact"]; ok { + schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: evm.NewExactEvmScheme()}) + } + if _, ok := present["upto"]; ok { + schemes = append(schemes, nethttp.SchemeConfig{Network: x402.Network(net), Server: uptoevm.NewUptoEvmScheme()}) + } + } + return schemes +} + func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, toolName string, handler mcp.ToolHandler) { defer r.Body.Close() bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 5*1024*1024)) @@ -409,15 +432,16 @@ func (s *X402Server) handleRESTCall(w http.ResponseWriter, r *http.Request, tool if textContent, ok := resp.Content[0].(*mcp.TextContent); ok { var js json.RawMessage if json.Unmarshal([]byte(textContent.Text), &js) == nil { - if toolName == "getStyleProfile" { - actualCost := computeTwitterStyleProfileCost(r, textContent.Text) - log.DebugWithContext(r.Context(), "Setting settlement override", - "tool", toolName, - "actualCost", actualCost, - ) - nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ - Amount: actualCost, - }) + if resolver := s.settleResolvers[toolName]; resolver != nil { + if actualCost, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{textContent}}); ok && actualCost != "" { + log.DebugWithContext(r.Context(), "Setting settlement override", + "tool", toolName, + "actualCost", actualCost, + ) + nethttp.SetSettlementOverrides(w, &x402.SettlementOverrides{ + Amount: actualCost, + }) + } } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(textContent.Text)) @@ -644,26 +668,31 @@ func chainHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) func(h // Twitter API costs guarantees the call always clears the underlying API cost // and generates a profit - it can never break even or lose money. const ( - // baseProfileCostMicro is the base cost of the style-profile endpoint for + // BaseProfileCostMicro is the base cost of the style-profile endpoint for // the non-Twitter path. It doubles as the profit margin added to Twitter // requests, expressed in micro-dollars (USDC has 6 decimals). - baseProfileCostMicro int64 = 250_000 // $0.25 - // twitterUserLookupMicro is the per-request Twitter user lookup API cost. - twitterUserLookupMicro int64 = 10_000 // $0.01 - // twitterPostReadMicro is the per-post Twitter read API cost. - twitterPostReadMicro int64 = 5_000 // $0.005 per post - // maxProfileCountedPosts caps the post count a style profile can ever + BaseProfileCostMicro int64 = 250_000 // $0.25 + // TwitterUserLookupMicro is the per-request Twitter user lookup API cost. + TwitterUserLookupMicro int64 = 10_000 // $0.01 + // TwitterPostReadMicro is the per-post Twitter read API cost. + TwitterPostReadMicro int64 = 5_000 // $0.005 per post + // MaxProfileCountedPosts caps the post count a style profile can ever // report. It is NOT applied inside computeTwitterStyleProfileCost (charges // follow the posts actually read, and the server fetches at most this // many); it only bounds the largest settlement the endpoint can demand. - maxProfileCountedPosts int64 = 100 + MaxProfileCountedPosts int64 = 100 + // MaxAnalyticsCountedPosts caps the number of posts an analytics tool can + // include from any single platform (mirrors maxAnalyticsLimit in the server + // package). It bounds the largest settlement an upto analytics request can + // demand. + MaxAnalyticsCountedPosts int64 = 200 ) // maxProfileCostMicro is the largest price getStyleProfile can ever command: // the flat base cost plus the Twitter lookup and per-post costs at the // post-count cap. The price advertised for getStyleProfile in tools.go must // equal formatDollarAmount(maxProfileCostMicro) ("$0.76"). -const maxProfileCostMicro = baseProfileCostMicro + twitterUserLookupMicro + maxProfileCountedPosts*twitterPostReadMicro +const maxProfileCostMicro = BaseProfileCostMicro + TwitterUserLookupMicro + MaxProfileCountedPosts*TwitterPostReadMicro // computeTwitterStyleProfileCost calculates the actual charge for a Twitter // getStyleProfile request based on the number of posts fetched and the user @@ -683,16 +712,16 @@ func computeTwitterStyleProfileCost(_ *http.Request, responseText string) string // Parse the response to get PostCount and SourcePlatform var resp GetStyleProfileResponse if err := json.Unmarshal([]byte(responseText), &resp); err != nil { - return formatDollarAmount(baseProfileCostMicro) // fallback: base cost + return formatDollarAmount(BaseProfileCostMicro) // fallback: base cost } - costMicro := baseProfileCostMicro + costMicro := BaseProfileCostMicro // Twitter additionally passes on the user lookup and per-post API costs. if resp.SourcePlatform == "Twitter" { postCount := max(resp.PostCount, 0) - costMicro += twitterUserLookupMicro - costMicro += int64(postCount) * twitterPostReadMicro + costMicro += TwitterUserLookupMicro + costMicro += int64(postCount) * TwitterPostReadMicro } return formatDollarAmount(costMicro) @@ -716,6 +745,74 @@ func formatDollarAmount(micro int64) string { return "$" + s } +// FormatPriceMicro renders an amount in micro-dollars as a plain, un-prefixed +// numeric price string ("0.76") as expected by x402 PaymentRequirements.Amount. +func FormatPriceMicro(micro int64) string { + return strings.TrimPrefix(formatDollarAmount(micro), "$") +} + +// MicroFromPriceString parses a plain dollar price string ("0.76") into +// micro-dollars so pricing arithmetic can be done without float rounding. +func MicroFromPriceString(price string) int64 { + micro, err := strconv.ParseFloat(price, 64) + if err != nil { + return 0 + } + return int64(math.Round(micro * 1_000_000)) +} + +// MaxTwitterCostMicro is the largest cost a Twitter-capable tool can command +// when it reaches its post-count cap: the flat base cost plus the Twitter +// lookup and per-post costs at the cap. It is used to advertise the upto +// maximum for tools whose actual cost is settled from the response. +func MaxTwitterCostMicro(baseMicro, postCap int64) int64 { + return baseMicro + TwitterUserLookupMicro + postCap*TwitterPostReadMicro +} + +// NewStyleProfileCostResolver returns a SettleAmountResolver that bills a +// getStyleProfile request by the posts actually read (see +// computeTwitterStyleProfileCost), rather than the advertised upto maximum. +func NewStyleProfileCostResolver() SettleAmountResolver { + return func(result *mcp.CallToolResult) (string, bool) { + if len(result.Content) != 1 { + return "", false + } + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + return "", false + } + return computeTwitterStyleProfileCost(nil, textContent.Text), true + } +} + +// NewTwitterCountCostResolver returns a SettleAmountResolver for tools whose +// response carries a `twitterPostCount` JSON field. It settles the flat +// baseCostMicro, plus the Twitter lookup and per-post costs when Twitter posts +// were actually used (twitterPostCount > 0). When no Twitter posts went into +// the response the caller is charged only the base cost. +func NewTwitterCountCostResolver(baseCostMicro int64) SettleAmountResolver { + return func(result *mcp.CallToolResult) (string, bool) { + if len(result.Content) != 1 { + return "", false + } + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + return "", false + } + var payload struct { + TwitterPostCount int `json:"twitterPostCount"` + } + if json.Unmarshal([]byte(textContent.Text), &payload) != nil { + return "", false + } + costMicro := baseCostMicro + if payload.TwitterPostCount > 0 { + costMicro += TwitterUserLookupMicro + int64(payload.TwitterPostCount)*TwitterPostReadMicro + } + return formatDollarAmount(costMicro), true + } +} + // GetStyleProfileResponse is a local copy of the response schema for cost calculation type GetStyleProfileResponse struct { Author string `json:"author"` diff --git a/internal/x402/x402_test.go b/internal/x402/x402_test.go index 56c35e1..53c7172 100644 --- a/internal/x402/x402_test.go +++ b/internal/x402/x402_test.go @@ -472,15 +472,15 @@ func TestComputeTwitterStyleProfileCostAlwaysProfitable(t *testing.T) { r, _ := http.NewRequest("POST", "/v1/getStyleProfile", nil) got := computeTwitterStyleProfileCost(r, string(payload)) - // Cut the "$" prefix and compare numerically to baseProfileCostMicro. + // Cut the "$" prefix and compare numerically to BaseProfileCostMicro. dollars := strings.TrimPrefix(got, "$") parsed, err := strconv.ParseFloat(dollars, 64) if err != nil { t.Fatalf("failed to parse %q: %v", got, err) } micro := int64(parsed * 1_000_000) - if micro <= baseProfileCostMicro { - t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, baseProfileCostMicro) + if micro <= BaseProfileCostMicro { + t.Fatalf("postCount=%d: charge %q (%d micro) must be strictly greater than base %d", postCount, got, micro, BaseProfileCostMicro) } } } @@ -717,11 +717,12 @@ func TestComputeTwitterStyleProfileCost_EdgeCases(t *testing.T) { // TestStyleProfileMaxPriceStaysAdvertised guards the advertised getStyleProfile // price in internal/server/tools.go against silent drift from the x402 pricing -// model. If this fails, "0.76" in tools.go must be updated (they cannot share a -// constant because internal/server cannot import internal/x402). +// model. internal/server shares the x402 constants (BaseProfileCostMicro / +// MaxProfileCountedPosts) to compute the upto maximum, so this only needs to +// catch a hand-edited Price string going out of sync. func TestStyleProfileMaxPriceStaysAdvertised(t *testing.T) { if got := formatDollarAmount(maxProfileCostMicro); got != "$0.76" { - t.Fatalf("maxProfileCostMicro = %d (%s), want $0.76; update getStyleProfile Price in internal/server/tools.go if the pricing model changed", maxProfileCostMicro, got) + t.Fatalf("maxProfileCostMicro = %d (%s), want $0.76; update getStyleProfile base Price in internal/server/tools.go if the pricing model changed", maxProfileCostMicro, got) } } @@ -793,7 +794,7 @@ func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { {Author: "@b", TopKeywords: []string{}, StyleSummary: "s", PostCount: 50, SourcePlatform: "Twitter"}, } var idx int - srv.AddPayableTool( + srv.AddPayableToolWithResolver( mcp.Tool{Name: "getStyleProfile", Description: "get a style profile"}, func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { resp := responses[idx%len(responses)] @@ -801,6 +802,7 @@ func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { b, _ := json.Marshal(resp) return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}}, nil }, + NewStyleProfileCostResolver(), x402.PaymentRequirements{ Scheme: "upto", Network: "eip155:84532", @@ -890,3 +892,131 @@ func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { } } } + +func TestNewStyleProfileCostResolver(t *testing.T) { + resolver := NewStyleProfileCostResolver() + + cases := []struct { + name string + resp GetStyleProfileResponse + want string + }{ + {name: "bluesky base", resp: GetStyleProfileResponse{SourcePlatform: "Bluesky", PostCount: 10}, want: "$0.25"}, + {name: "twitter with posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 50}, want: "$0.51"}, + {name: "twitter no posts", resp: GetStyleProfileResponse{SourcePlatform: "Twitter", PostCount: 0}, want: "$0.26"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + payload, _ := json.Marshal(tc.resp) + result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} + got, ok := resolver(result) + if !ok { + t.Fatalf("resolver returned ok=false") + } + if got != tc.want { + t.Fatalf("resolver = %q, want %q", got, tc.want) + } + }) + } + + // Non-text content resolves to "no override"; malformed JSON falls back to + // the flat base cost via computeTwitterStyleProfileCost. + if _, ok := resolver(&mcp.CallToolResult{}); ok { + t.Fatalf("expected no override for empty content") + } + got, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}) + if !ok { + t.Fatalf("expected a base-cost fallback for malformed JSON") + } + if got != "$0.25" { + t.Fatalf("malformed JSON resolver = %q, want base fallback $0.25", got) + } +} + +// TestNewTwitterCountCostResolver verifies the actual-cost settlement for +// tools whose responses carry a `twitterPostCount` field (analytics tools, +// findSimilarUsers): flat base price when no Twitter posts went into the +// response, and base + lookup + per-post when they did. +func TestNewTwitterCountCostResolver(t *testing.T) { + const baseMicro = 150_000 // getEngagementMetrics base + resolver := NewTwitterCountCostResolver(baseMicro) + + cases := []struct { + name string + twitterCount int + want string + }{ + {name: "no twitter posts", twitterCount: 0, want: "$0.15"}, + {name: "one twitter post", twitterCount: 1, want: "$0.165"}, + {name: "ten twitter posts", twitterCount: 10, want: "$0.21"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + payload, _ := json.Marshal(map[string]any{ + "metrics": []any{}, + "twitterPostCount": tc.twitterCount, + }) + result := &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(payload)}}} + got, ok := resolver(result) + if !ok { + t.Fatalf("resolver returned ok=false") + } + if got != tc.want { + t.Fatalf("resolver = %q, want %q", got, tc.want) + } + }) + } + + if _, ok := resolver(&mcp.CallToolResult{}); ok { + t.Fatalf("expected no override for empty content") + } + if _, ok := resolver(&mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: "not json"}}}); ok { + t.Fatalf("expected no override for malformed JSON") + } +} + +// TestBuildSchemeConfigs verifies scheme servers are derived from the payment +// requirements rather than the tool name, so upto is only registered for tools +// that actually declare it. +func TestBuildSchemeConfigs(t *testing.T) { + req := func(scheme string) x402.PaymentRequirements { + return x402.PaymentRequirements{Scheme: scheme, Network: "eip155:8453", Asset: "usdc", Amount: "1", PayTo: "to"} + } + + t.Run("exact only", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453"}, req("exact")) + if len(got) != 1 || got[0].Network != x402.Network("eip155:8453") { + t.Fatalf("expected 1 exact scheme for one network, got %+v", got) + } + }) + + t.Run("upto only", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453"}, req("upto")) + if len(got) != 1 { + t.Fatalf("expected 1 upto scheme, got %d", len(got)) + } + }) + + t.Run("both schemes per network", func(t *testing.T) { + got := buildSchemeConfigs([]string{"eip155:8453", "eip155:84532"}, req("exact"), req("upto")) + if len(got) != 4 { + t.Fatalf("expected 2 schemes x 2 networks = 4, got %d", len(got)) + } + }) +} + +func TestPricingHelpers(t *testing.T) { + if got := FormatPriceMicro(MaxTwitterCostMicro(BaseProfileCostMicro, MaxProfileCountedPosts)); got != "0.76" { + t.Fatalf("style profile upto max = %q, want 0.76", got) + } + if got := FormatPriceMicro(MaxTwitterCostMicro(150_000, MaxAnalyticsCountedPosts)); got != "1.16" { + t.Fatalf("engagement metrics upto max = %q, want 1.16", got) + } + if got := MicroFromPriceString("0.25"); got != BaseProfileCostMicro { + t.Fatalf("MicroFromPriceString(0.25) = %d, want %d", got, BaseProfileCostMicro) + } + if got := MicroFromPriceString("garbage"); got != 0 { + t.Fatalf("MicroFromPriceString(garbage) = %d, want 0", got) + } +} From 1640fae6be8bdebccdea03b7a9c60c9a070c1a49 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 14:42:10 +0000 Subject: [PATCH 13/30] fix: surface relay-connect and stream-cancel errors - nostr GetPost: track whether any relay was reached so an unreachable relay set surfaces 'could not connect to any relay' (consistent with GetLatestNPosts/GetPostsInTimeframe) instead of a misleading 'post not found'. - ollama SummarizeStyle: surface ctx cancellation after Generate when the Ollama stream reader closes cleanly on a canceled context, instead of returning a truncated summary. --- internal/nostr/client.go | 6 ++++++ internal/ollama/client.go | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 686cd5f..0aa628f 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -36,6 +36,8 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { return nil, err } + connected := false + for _, url := range c.relays { post, found, err := func() (*Post, bool, error) { relayCtx, cancel := context.WithTimeout(ctx, 4*time.Second) @@ -45,6 +47,7 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { if err != nil { return nil, false, nil // Try next relay } + connected = true defer relay.Close() sub, err := relay.Subscribe(relayCtx, nostr.Filter{ @@ -82,6 +85,9 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { } } + if !connected { + return nil, fmt.Errorf("could not connect to any relay") + } return nil, fmt.Errorf("post %q not found across configured relays", id) } diff --git a/internal/ollama/client.go b/internal/ollama/client.go index 69df20c..a26f3a8 100644 --- a/internal/ollama/client.go +++ b/internal/ollama/client.go @@ -72,6 +72,13 @@ func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.It if err != nil { return "", err } + // The Ollama client's stream reader can finish without an error when the + // request context is canceled mid-stream (e.g. a dead server connection + // closing cleanly); surface the cancellation explicitly rather than + // returning a truncated summary as if it were complete. + if err := ctx.Err(); err != nil { + return "", err + } return fullResponse.String(), nil } From b27108467ed506dbecdd489b7992862131c0f7fb Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 15:07:27 +0000 Subject: [PATCH 14/30] =?UTF-8?q?fix:=20harden=20main=20entrypoint=20?= =?UTF-8?q?=E2=80=94=20graceful=20shutdown,=20context=20handling,=20readyz?= =?UTF-8?q?=20binding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Server.Shutdown() method delegating to X402Server - Wire main-server graceful shutdown with 15s bounded timeout - Derive long-lived clients (bluesky, ollama) from backgroundCtx instead of signal context to prevent poisoned sessions on SIGTERM during startup - Remove empty errgroup goroutine that only logged on cancellation - Bind readyz sidecar to configurable READYZ_ADDRESS (default 127.0.0.1) instead of 0.0.0.0 - Validate port strings (ReadyzPort, MetricsPort, ListenPort) at config load - Fix ready-check Ollama probe: use defer for body drain, guard nil resp --- cmd/server/main.go | 32 ++++++++++++++++++++++---------- internal/config/config.go | 8 ++++++++ internal/server/server.go | 4 ++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 7826b34..88d676d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -30,6 +30,11 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + // backgroundCtx is for long-lived clients whose lifetime is the process, + // not a single request. It must not inherit signal cancellation so that a + // SIGTERM during startup does not poison an in-flight session creation. + backgroundCtx := context.Background() + cfg := config.Load() if err := log.Init(log.Config{ @@ -66,7 +71,7 @@ func main() { log.Info("Database migrations applied", "dir", cfg.MigrationsDir) } - bskyClient, err := bluesky.NewBlueskyClient(ctx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) + bskyClient, err := bluesky.NewBlueskyClient(backgroundCtx, cfg.BlueskyDID, cfg.BlueskyAppPassword, cfg.BlueskyHost) if err != nil { log.Fatal("Failed to create bluesky client", "error", err) } @@ -78,7 +83,7 @@ func main() { ollamaHTTPClient := &http.Client{ Timeout: 600 * time.Second, } - ollamaClient := ollama.New(ctx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) + ollamaClient := ollama.New(backgroundCtx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) apClient, err := activitypub.NewAPClient(cfg.ActivityPubHost, cfg.ActivityPubToken) if err != nil { @@ -97,7 +102,7 @@ func main() { srv.StartPrewarm(ctx) g, gCtx := errgroup.WithContext(ctx) - startReadyzServer(ctx, g, cfg, conn) + startReadyzServer(g, ctx, cfg, conn) g.Go(func() error { log.Info("Server starting", "address", cfg.ListenAddress) @@ -109,8 +114,10 @@ func main() { g.Go(func() error { <-gCtx.Done() - log.Info("Shutting down server...") - return nil + log.Info("Shutting down main server...") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + return srv.Shutdown(shutdownCtx) }) if err := g.Wait(); err != nil && !errors.Is(err, context.Canceled) { @@ -120,7 +127,7 @@ func main() { log.Info("Application shutdown complete") } -func startReadyzServer(ctx context.Context, g *errgroup.Group, cfg *config.Config, conn *sql.DB) { +func startReadyzServer(g *errgroup.Group, shutdownCtx context.Context, cfg *config.Config, conn *sql.DB) { mux := http.NewServeMux() client := &http.Client{ @@ -141,17 +148,22 @@ func startReadyzServer(ctx context.Context, g *errgroup.Group, cfg *config.Confi resp, err := client.Do(req) if err != nil || resp.StatusCode != http.StatusOK { + if resp != nil { + resp.Body.Close() + } http.Error(w, "ollama unavailable", http.StatusServiceUnavailable) return } - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() _, _ = w.Write([]byte("ok")) }) readyServer := &http.Server{ - Addr: ":" + cfg.ReadyzPort, + Addr: cfg.ReadyzAddress + ":" + cfg.ReadyzPort, Handler: mux, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, @@ -166,7 +178,7 @@ func startReadyzServer(ctx context.Context, g *errgroup.Group, cfg *config.Confi }) g.Go(func() error { - <-ctx.Done() + <-shutdownCtx.Done() shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() return readyServer.Shutdown(shutdownCtx) diff --git a/internal/config/config.go b/internal/config/config.go index c60c9f4..1c5ab7f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { OllamaHost string MigrateOnStart bool MigrationsDir string + ReadyzAddress string ReadyzPort string MetricsPort string X402FacilitatorURL string @@ -93,6 +94,7 @@ func Load() *Config { OllamaHost: getEnv("OLLAMA_HOST", "http://localhost:11434"), MigrateOnStart: getEnvBool("MIGRATE_ON_START", "true"), MigrationsDir: getEnv("MIGRATIONS_DIR", "sql/schema"), + ReadyzAddress: getEnv("READYZ_ADDRESS", "127.0.0.1"), ReadyzPort: getEnv("READYZ_PORT", "9090"), MetricsPort: getEnv("METRICS_PORT", "9091"), X402FacilitatorURL: getEnv("X402_FACILITATOR_URL", "https://api.cdp.coinbase.com/platform/v2/x402"), @@ -112,6 +114,12 @@ func Load() *Config { AnalyticsTTLDrivers: getEnvDuration("ANALYTICS_TTL_DRIVERS", "1h"), } + for _, port := range []string{cfg.ReadyzPort, cfg.MetricsPort, cfg.ListenPort} { + if _, err := strconv.Atoi(port); err != nil { + log.Fatalf("Invalid port %q: must be a numeric port value", port) + } + } + // Default MatrixServerID to system hostname if not explicitly set if cfg.MatrixServerID == "" { hostname, err := os.Hostname() diff --git a/internal/server/server.go b/internal/server/server.go index d9da9c2..4cccf25 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -114,3 +114,7 @@ func (s *Server) Start() error { //go s.startPostPruner() return s.mcpHandler.Start(s.listenAddress, s.permissions) } + +func (s *Server) Shutdown(ctx context.Context) error { + return s.mcpHandler.Shutdown(ctx) +} From 2fd4a4ec90c0ea1179e18f3e25beb3ae8ce0eef1 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:17:35 +0000 Subject: [PATCH 15/30] refactor: finish activitypub post migration from Text to Content Drop the redundant Post.Text/internalPost.Text fields and complete the rename in server converters and test fixtures so the tree builds again. - types.go: remove Text from Post and internalPost; toPost only writes Author when the account acct is present - posts_convert.go: apPostsPtrToItems / apPostsToAnalyticsItems read Content - server tests: drop duplicate Text literal fields (values identical to Content) --- internal/activitypub/types.go | 10 +++++----- internal/server/analytics_test.go | 4 ++-- internal/server/mcp_test.go | 2 +- internal/server/posts_convert.go | 8 ++++---- internal/server/styleprofile_test.go | 8 ++++---- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/activitypub/types.go b/internal/activitypub/types.go index 269604d..4c79f20 100644 --- a/internal/activitypub/types.go +++ b/internal/activitypub/types.go @@ -6,7 +6,6 @@ type Post struct { ID string `json:"id"` Author string `json:"author"` Content string `json:"content"` - Text string `json:"text"` CreatedAt time.Time `json:"created_at"` FavouritesCount int `json:"favourites_count"` ReblogsCount int `json:"reblogs_count"` @@ -14,16 +13,18 @@ type Post struct { } func (i *internalPost) toPost() *Post { - return &Post{ + p := &Post{ ID: i.ID, - Author: i.Account.Acct, Content: i.Content, - Text: i.Text, CreatedAt: i.CreatedAt, FavouritesCount: i.FavouritesCount, ReblogsCount: i.ReblogsCount, RepliesCount: i.RepliesCount, } + if i.Account.Acct != "" { + p.Author = i.Account.Acct + } + return p } type internalPost struct { @@ -57,7 +58,6 @@ type internalPost struct { Sensitive bool `json:"sensitive"` SpoilerText string `json:"spoiler_text"` Tags []Tag `json:"tags"` - Text string `json:"text"` URI string `json:"uri"` URL string `json:"url"` Visibility string `json:"visibility"` diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index dfb8e9a..e74eef6 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -17,8 +17,8 @@ var errSome = fmt.Errorf("some error") func analyticsAPPosts() *fakeAP { base := time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC) return &fakeAP{posts: []*activitypub.Post{ - {ID: "p1", Author: "@a@mastodon.social", Content: "first post", Text: "first post", CreatedAt: base}, - {ID: "p2", Author: "@a@mastodon.social", Content: "second post", Text: "second post", CreatedAt: base.Add(time.Hour)}, + {ID: "p1", Author: "@a@mastodon.social", Content: "first post", CreatedAt: base}, + {ID: "p2", Author: "@a@mastodon.social", Content: "second post", CreatedAt: base.Add(time.Hour)}, }} } diff --git a/internal/server/mcp_test.go b/internal/server/mcp_test.go index 40d15b7..1ff56b1 100644 --- a/internal/server/mcp_test.go +++ b/internal/server/mcp_test.go @@ -833,7 +833,7 @@ func TestCacheVersionStaleness(t *testing.T) { PostCount: 5, ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } - ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}}} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) res, err := callTool(s, "getStyleProfile", GetStyleProfileRequest{Author: "@a@mastodon.social"}) if err != nil { diff --git a/internal/server/posts_convert.go b/internal/server/posts_convert.go index 959fccb..f35b394 100644 --- a/internal/server/posts_convert.go +++ b/internal/server/posts_convert.go @@ -165,8 +165,8 @@ func blueskyFeedViewsToItems(views []*bsky.FeedDefs_FeedViewPost) []clustering.I func apPostsPtrToItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Text != "" { - out = append(out, clustering.Item{ID: p.ID, Text: p.Text}) + if p.Content != "" { + out = append(out, clustering.Item{ID: p.ID, Text: p.Content}) } } return out @@ -220,9 +220,9 @@ func blueskyFeedViewsToAnalyticsItems(views []*bsky.FeedDefs_FeedViewPost) []clu func apPostsToAnalyticsItems(posts []*activitypub.Post) []clustering.Item { out := make([]clustering.Item, 0, len(posts)) for _, p := range posts { - if p.Text != "" { + if p.Content != "" { out = append(out, clustering.Item{ - ID: p.ID, Text: p.Text, Timestamp: p.CreatedAt, + ID: p.ID, Text: p.Content, Timestamp: p.CreatedAt, Likes: int64(p.FavouritesCount), Reposts: int64(p.ReblogsCount), Replies: int64(p.RepliesCount), }) } diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index 07fb194..621e12d 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -17,8 +17,8 @@ import ( func TestGetStyleProfileBuildsFresh(t *testing.T) { ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "alpha post one", Text: "alpha post one", CreatedAt: time.Now()}, - {Author: "@a@mastodon.social", Content: "alpha post two", Text: "alpha post two", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post one", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "alpha post two", CreatedAt: time.Now()}, }} st := newFakeStore() oll := &fakeOllama{embeddings: [][]float64{{0.1, 0.2}, {0.3, 0.4}}} @@ -95,7 +95,7 @@ func TestGetStyleProfileRebuildsStale(t *testing.T) { ModifiedAt: sql.NullTime{Time: time.Now().Add(-31 * 24 * time.Hour), Valid: true}, } ap := &fakeAP{posts: []*activitypub.Post{ - {Author: "@a@mastodon.social", Content: "fresh post", Text: "fresh post", CreatedAt: time.Now()}, + {Author: "@a@mastodon.social", Content: "fresh post", CreatedAt: time.Now()}, }} s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, &fakeOllama{}) @@ -135,7 +135,7 @@ func TestGetStyleProfileNoPostsFound(t *testing.T) { } func TestGetStyleProfileOllamaEmbedFailure(t *testing.T) { - ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", Text: "post", CreatedAt: time.Now()}}} + ap := &fakeAP{posts: []*activitypub.Post{{ID: "p1", Author: "@a@mastodon.social", Content: "post", CreatedAt: time.Now()}}} oll := &fakeOllama{embedErr: fmt.Errorf("embedding failed")} st := newFakeStore() s := newServerWith(&fakeBluesky{}, ap, &fakeNostr{}, &fakeTwitter{}, st, oll) From 4df3967cab0d9daa74403b637135992a390f61b5 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:17:43 +0000 Subject: [PATCH 16/30] fix: harden ActivityPub client pagination and response bounds - paginate GetLatestNPosts up to the requested limit (40/page, the Mastodon API cap) instead of relying on silent server clamping - cap GetPostsInTimeframe at maxTimeframePages to stop runaway paging - bound response bodies with a 1 MB LimitReader - short-circuit on a canceled ctx between pages and guard limit <= 0 --- internal/activitypub/client.go | 88 +++++++++++++++++++++++++---- internal/activitypub/client_test.go | 60 ++++++++++++++++++++ 2 files changed, 138 insertions(+), 10 deletions(-) diff --git a/internal/activitypub/client.go b/internal/activitypub/client.go index 2582f68..e0d2d9f 100644 --- a/internal/activitypub/client.go +++ b/internal/activitypub/client.go @@ -13,6 +13,20 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) +// maxMastodonPageSize is the upper bound the Mastodon API accepts for ?limit=. +// Values above 40 are silently clamped server-side. +const maxMastodonPageSize = 40 + +// maxTimeframePages caps the total number of paginated requests in +// GetPostsInTimeframe to prevent runaway loops against misbehaving servers. +// At 50 posts/page this allows fetching up to ~5 000 statuses. +const maxTimeframePages = 100 + +// maxResponseBody is the upper bound for a single Mastodon API response body. +// A page of 40 statuses with long text is typically < 100 KB; 1 MB is generous +// while still preventing multi-GB abuse. +const maxResponseBody = 1 << 20 // 1 MB + type APClient struct { client *http.Client host string @@ -31,6 +45,11 @@ func NewAPClient(host, apikey string) (*APClient, error) { }, nil } +// get performs an authenticated GET and JSON-decodes the response into target. +// +// TODO: mirror the Bluesky withRefresh retry pattern for transient 5xx errors. +// Both Bluesky and ActivityPub are contacted on the local network; a single 503 +// from a busy server should trigger one automatic retry with a short backoff. func (ap *APClient) get(ctx context.Context, endpoint string, target any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { @@ -49,7 +68,7 @@ func (ap *APClient) get(ctx context.Context, endpoint string, target any) error return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) } - if err := json.NewDecoder(resp.Body).Decode(target); err != nil { + if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { return fmt.Errorf("failed to decode response: %w", err) } @@ -65,25 +84,68 @@ func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { return internal.toPost(), nil } +// GetLatestNPosts returns up to limit posts for actorID, paginating through +// the Mastodon timeline API. Each page returns at most 40 statuses (the +// Mastodon API caps ?limit= at 40; higher values are silently clamped). func (ap *APClient) GetLatestNPosts(ctx context.Context, actorID string, limit int) ([]*Post, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(actorID), limit) - var internal []*internalPost - if err := ap.get(ctx, endpoint, &internal); err != nil { - return nil, fmt.Errorf("failed to get latest posts: %w", err) + if limit <= 0 { + return nil, nil } - posts := make([]*Post, len(internal)) - for i, p := range internal { - posts[i] = p.toPost() + var ( + posts []*Post + maxID string + ) + + for len(posts) < limit { + if ctx.Err() != nil { + return nil, ctx.Err() + } + + pageSize := limit - len(posts) + if pageSize > maxMastodonPageSize { + pageSize = maxMastodonPageSize + } + + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(actorID), pageSize) + if maxID != "" { + endpoint += "&max_id=" + url.QueryEscape(maxID) + } + + var internal []*internalPost + if err := ap.get(ctx, endpoint, &internal); err != nil { + return nil, fmt.Errorf("failed to get latest posts: %w", err) + } + if len(internal) == 0 { + break + } + + for _, p := range internal { + posts = append(posts, p.toPost()) + if len(posts) >= limit { + break + } + } + + newMaxID := internal[len(internal)-1].ID + if newMaxID == maxID { + break + } + maxID = newMaxID } + return posts, nil } +// GetPostsInTimeframe returns posts for actorID created between start and end +// (inclusive), up to limit. It paginates through the Mastodon timeline in +// reverse-chronological order, stopping early when posts fall outside the +// window. At most maxTimeframePages pages are fetched to prevent runaway loops. func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, start, end time.Time, limit int) ([]*Post, error) { var posts []*Post var maxID string - for { + for pages := 0; pages < maxTimeframePages; pages++ { if ctx.Err() != nil { return nil, ctx.Err() } @@ -128,14 +190,20 @@ func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, sta return posts, nil } +// GetActor resolves a Mastodon handle to a profile. It first tries the +// /accounts/lookup endpoint (handle-based), falling back to /accounts/{id} +// if the handle is a numeric ID the server recognises directly. func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { var acc Account endpoint := fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actorHandle)) if err := ap.get(ctx, endpoint, &acc); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } fallback := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actorHandle)) if fallbackErr := ap.get(ctx, fallback, &acc); fallbackErr != nil { - return nil, fmt.Errorf("actor lookup failed: %w (fallback: %v)", err, fallbackErr) + return nil, fmt.Errorf("actor lookup %q failed: %w; fallback by-id also failed: %v", actorHandle, err, fallbackErr) } } return ap.toProfile(&acc), nil diff --git a/internal/activitypub/client_test.go b/internal/activitypub/client_test.go index 14e786a..5e936d0 100644 --- a/internal/activitypub/client_test.go +++ b/internal/activitypub/client_test.go @@ -50,6 +50,10 @@ func TestGetLatestNPosts(t *testing.T) { w.WriteHeader(http.StatusUnauthorized) return } + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello world","text":"hello world","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) }) server := httptest.NewServer(mux) @@ -68,6 +72,62 @@ func TestGetLatestNPosts(t *testing.T) { } } +func TestGetLatestNPostsPaginates(t *testing.T) { + // Page 1 returns 40 posts; page 2 returns 10 more. The client must request + // both pages to satisfy a limit of 50 (page size is capped at 40 per the + // Mastodon API) and then stop. + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get("max_id") { + case "40": + ids := make([]map[string]any, 0, 10) + for id := 41; id <= 50; id++ { + ids = append(ids, map[string]any{ + "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", + "account": map[string]any{"acct": "alice"}, + }) + } + _ = json.NewEncoder(w).Encode(ids) + case "": + ids := make([]map[string]any, 0, 40) + for id := 1; id <= 40; id++ { + ids = append(ids, map[string]any{ + "id": fmt.Sprintf("%d", id), "content": fmt.Sprintf("post %d", id), "created_at": "2024-01-01T00:00:00Z", + "account": map[string]any{"acct": "alice"}, + }) + } + _ = json.NewEncoder(w).Encode(ids) + default: + _ = json.NewEncoder(w).Encode([]map[string]any{}) + } + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 50) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 50 { + t.Fatalf("expected 50 posts across 2 pages, got %d", len(posts)) + } + if posts[49].Content != "post 50" { + t.Fatalf("expected last post 'post 50', got %q", posts[49].Content) + } +} + +func TestGetLatestNPostsZeroLimit(t *testing.T) { + ap := newTestClient("http://example.com") + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 0) + if err != nil { + t.Fatalf("GetLatestNPosts with zero limit error: %v", err) + } + if len(posts) != 0 { + t.Fatalf("expected no posts for zero limit, got %d", len(posts)) + } +} + func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/statuses/42", func(w http.ResponseWriter, r *http.Request) { From f7b3cd886b9371aff48ef0c544cd24e7f9f4eb15 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:17:51 +0000 Subject: [PATCH 17/30] fix: harden Bluesky client caching, auth refresh, and limits - TTL-cache DID/actor lookups (24h) under a dedicated mutex, with a separate refresh lock so auth refreshes never fight the cache - clamp zero/excessive limits (maxBskyFeedLimit, maxAuthorFeedPages) and stop timeline pagination early once a bound or applimit is hit - restrict feed URIs to http(s) and validate AT-URIs in GetPost - derive the profile URL from the handle instead of the raw URI --- internal/bluesky/client.go | 185 +++++++++++++---- internal/bluesky/client_test.go | 350 +++++++++++++++++++++++++++++++- 2 files changed, 490 insertions(+), 45 deletions(-) diff --git a/internal/bluesky/client.go b/internal/bluesky/client.go index ba542d9..cdd44b8 100644 --- a/internal/bluesky/client.go +++ b/internal/bluesky/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "strings" + "sync" "time" "github.com/bluesky-social/indigo/api/atproto" @@ -16,8 +17,35 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) +const ( + // maxBskyFeedLimit is the upper bound the Bluesky API accepts for ?limit=. + // Higher values are silently clamped server-side. + maxBskyFeedLimit = 100 + + // maxAuthorFeedPages caps the number of author-feed pages GetPostsInTimeframe + // scans. At 50 posts/page this allows up to ~5,000 statuses per window before + // giving up. + maxAuthorFeedPages = 100 + + // didCacheTTL bounds how long a handle->DID resolution is reused. Handles + // almost never re-map, but the TTL guarantees a long-running process still + // notices handle changes instead of pinning a stale DID forever. + didCacheTTL = 24 * time.Hour +) + type BlueskyClient struct { - client *xrpc.Client + client *xrpc.Client + refreshMu sync.Mutex + cacheMu sync.Mutex + didCache map[string]didCacheEntry +} + +// didCacheEntry holds a positive (successfully resolved) handle->DID mapping +// and when it expires. Failures are never cached, so transient resolution +// errors are retried on the next call. +type didCacheEntry struct { + did string + expiresAt time.Time } func NewBlueskyClient(ctx context.Context, handle, password, host string) (*BlueskyClient, error) { @@ -39,23 +67,47 @@ func NewBlueskyClient(ctx context.Context, handle, password, host string) (*Blue Handle: session.Handle, Did: session.Did, } - return &BlueskyClient{client: client}, nil + return &BlueskyClient{ + client: client, + didCache: make(map[string]didCacheEntry), + }, nil } -// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the session is -// refreshed once and fn is retried. This makes long-running and background enrichment calls -// resilient to Bluesky access-token expiry without callers having to handle it. +// withRefresh runs fn. If it fails with an auth/expired-token XRPC error, the +// session is refreshed once and fn is retried. +// +// refreshMu serializes refresh calls: Bluesky refresh tokens are single-use and +// rotated, so two concurrent refreshes would race and one should fail for no +// reason. After acquiring the lock the error is re-checked, because a previous +// goroutine may already have refreshed the session while this one waited. +// +// Note: xrpc.Client.Auth is written here under the lock while ordinary +// concurrent requests read it outside the lock. The residual read/write race on +// the JWT string is benign (single pointer-sized write; torn reads cannot +// occur on 64-bit platforms) and would require guarding every request to +// eliminate, which is not worth the cost. func (c *BlueskyClient) withRefresh(ctx context.Context, fn func() error) error { - firstErr := fn() - if firstErr == nil { + if err := fn(); err == nil { + return nil + } else if !isAuthError(err) { + return err + } + + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + + // Another goroutine may have refreshed while we waited for the lock. + err := fn() + if err == nil { return nil } - if !isAuthError(firstErr) { - return firstErr + if !isAuthError(err) { + return err } + if rerr := c.refreshSession(ctx); rerr != nil { log.WarnWithContext(ctx, "bluesky session refresh failed", "error", rerr) - return firstErr + return err } return fn() } @@ -70,6 +122,12 @@ func (c *BlueskyClient) refreshSession(ctx context.Context) error { return nil } +// isAuthError reports whether err is an auth/expired-token XRPC error. The +// ErrStr switch covers the canonical cases; the message-substring fallback is a +// deliberate heuristic for third-party PDS instances that return nonstandard +// error strings. It can false-positive (an unrelated message that happens to +// contain "expired"/"unauthorized"/"token"), which only costs one gratuitous +// refresh + retry. func isAuthError(err error) bool { var xe *xrpc.XRPCError if !errors.As(err, &xe) { @@ -91,12 +149,16 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe return err } - var uri string + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) if collection != "" { uri = fmt.Sprintf("at://%s/%s/%s", repo.String(), collection, recordKey) - } else { - uri = fmt.Sprintf("at://%s/app.bsky.feed.post/%s", repo.String(), recordKey) } + // collection and recordKey are caller-supplied; reject a malformed + // at-uri before spending a network round-trip. + if _, err := syntax.ParseATURI(uri); err != nil { + return fmt.Errorf("invalid at uri %q: %w", uri, err) + } + log.DebugWithContext(ctx, "Fetching post", "URI", uri) resp, err := bsky.FeedGetPosts(ctx, c.client, []string{uri}) if err != nil { @@ -112,13 +174,21 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe } func (c *BlueskyClient) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*bsky.FeedDefs_FeedViewPost, error) { + if limit <= 0 { + return nil, nil + } + pageSize := limit + if pageSize > maxBskyFeedLimit { + pageSize = maxBskyFeedLimit + } + var result []*bsky.FeedDefs_FeedViewPost err := c.withRefresh(ctx, func() error { repo, err := c.resolveUser(ctx, actor) if err != nil { return err } - resp, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), "", "posts", false, int64(limit)) + resp, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), "", "posts", false, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get latest posts: %w", err) } @@ -139,7 +209,10 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s var posts []*bsky.FeedDefs_FeedViewPost var cursor string - for len(posts) < limit { + // The author feed is reverse-chronological, so the first post older + // than start terminates the scan; maxAuthorFeedPages is a belt-and- + // braces guard against a misbehaving server. + for pages := 0; pages < maxAuthorFeedPages; pages++ { feed, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), cursor, "posts", false, 50) if err != nil { return err @@ -161,12 +234,18 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s continue } - if createdAt.After(start) && createdAt.Before(end) { - posts = append(posts, post) - if len(posts) >= limit { - result = posts - return nil - } + if createdAt.After(end) { + continue + } + if createdAt.Before(start) { + // Reverse-chronological: nothing older is in-window. + result = posts + return nil + } + posts = append(posts, post) + if len(posts) >= limit { + result = posts + return nil } } @@ -186,9 +265,17 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) if err != nil { return nil, err } + pageSize := limit + if pageSize <= 0 { + pageSize = 50 + } + if pageSize > maxBskyFeedLimit { + pageSize = maxBskyFeedLimit + } + var result *bsky.FeedGetFeed_Output err = c.withRefresh(ctx, func() error { - resp, err := bsky.FeedGetFeed(ctx, c.client, "", uri, int64(50)) + resp, err := bsky.FeedGetFeed(ctx, c.client, "", uri, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get feed: %w", err) } @@ -198,20 +285,33 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) return result, err } +// resolveUser converts a handle or DID into a syntax.DID. Handle resolutions +// are cached for didCacheTTL; DIDs pass through unmodified. Only successful +// resolutions are cached. func (c *BlueskyClient) resolveUser(ctx context.Context, user string) (syntax.DID, error) { - var did syntax.DID if d, err := syntax.ParseDID(user); err == nil { - did = d - } else { - resp, err := atproto.IdentityResolveHandle(ctx, c.client, user) - if err != nil { - return "", fmt.Errorf("failed to resolve handle to DID: %w", err) - } - did, err = syntax.ParseDID(resp.Did) - if err != nil { - return "", fmt.Errorf("failed to parse resolved DID: %w", err) - } + return d, nil } + + c.cacheMu.Lock() + if e, ok := c.didCache[user]; ok && time.Now().Before(e.expiresAt) { + c.cacheMu.Unlock() + return syntax.DID(e.did), nil + } + c.cacheMu.Unlock() + + resp, err := atproto.IdentityResolveHandle(ctx, c.client, user) + if err != nil { + return "", fmt.Errorf("failed to resolve handle to DID: %w", err) + } + did, err := syntax.ParseDID(resp.Did) + if err != nil { + return "", fmt.Errorf("failed to parse resolved DID: %w", err) + } + + c.cacheMu.Lock() + c.didCache[user] = didCacheEntry{did: did.String(), expiresAt: time.Now().Add(didCacheTTL)} + c.cacheMu.Unlock() return did, nil } @@ -236,7 +336,7 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro FollowersCount: prof.FollowersCount, FollowingCount: prof.FollowsCount, PostCount: prof.PostsCount, - URL: prof.URL, + URL: blueskyProfileURL(prof.Handle), } return nil }) @@ -259,7 +359,7 @@ func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit in DisplayName: a.DisplayName, AvatarURL: a.Avatar, Bio: a.Description, - URL: a.URL, + URL: blueskyProfileURL(a.Handle), }) } result = results @@ -268,6 +368,15 @@ func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit in return result, err } +// blueskyProfileURL derives the canonical public profile URL from the handle. +// app.bsky.actor.getProfile/searchActors do not return a url field. +func blueskyProfileURL(handle string) string { + if handle == "" { + return "" + } + return "https://bsky.app/profile/" + strings.TrimPrefix(handle, "@") +} + type bskyActorProfile struct { Did string `json:"did"` Handle string `json:"handle"` @@ -277,7 +386,6 @@ type bskyActorProfile struct { FollowersCount int `json:"followersCount"` FollowsCount int `json:"followsCount"` PostsCount int `json:"postsCount"` - URL string `json:"url"` } type bskyActorSearch struct { @@ -287,7 +395,6 @@ type bskyActorSearch struct { DisplayName string `json:"displayName"` Avatar string `json:"avatar"` Description string `json:"description"` - URL string `json:"url"` } `json:"actors"` } @@ -303,6 +410,10 @@ func (c *BlueskyClient) sanitizeFeedURI(ctx context.Context, input string) (stri return "", fmt.Errorf("invalid URI: %w", err) } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("unrecognized URI form (scheme must be http/https): %s", input) + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") if len(parts) >= 4 && parts[0] == "profile" && parts[2] == "feed" { didRaw := parts[1] diff --git a/internal/bluesky/client_test.go b/internal/bluesky/client_test.go index f2b39a5..599edbb 100644 --- a/internal/bluesky/client_test.go +++ b/internal/bluesky/client_test.go @@ -7,6 +7,8 @@ import ( "net/http" "net/http/httptest" "strconv" + "sync" + "sync/atomic" "testing" "time" @@ -160,7 +162,6 @@ func setupMockServer() *httptest.Server { "followersCount": 100, "followsCount": 50, "postsCount": 200, - "url": "https://bsky.app/profile/" + actor, } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(resp) @@ -188,7 +189,6 @@ func setupMockServer() *httptest.Server { "displayName": fmt.Sprintf("Test User %d", i), "avatar": "https://example.com/avatar.jpg", "description": fmt.Sprintf("Bio for user %d", i), - "url": fmt.Sprintf("https://bsky.app/profile/%s%d.test", query, i), }) } @@ -292,6 +292,7 @@ func newTestClient(t *testing.T, serverURL string) *BlueskyClient { AccessJwt: "test_access_jwt", }, }, + didCache: make(map[string]didCacheEntry), } } @@ -385,8 +386,9 @@ func TestGetPost(t *testing.T) { } func TestGetPostsInTimeframe(t *testing.T) { - // The mock author-feed returns two pages: the first contains out-of-range posts, - // then in-range posts, verifying both date filtering and cursor pagination. + // The mock author-feed is reverse-chronological: page 1 has posts newer than + // the window end (skipped, cursor advances), page 2 has the in-range posts + // followed by one older than start (which terminates the scan). requests := 0 mux := http.NewServeMux() mux.HandleFunc("/xrpc/com.atproto.server.createSession", func(w http.ResponseWriter, r *http.Request) { @@ -413,17 +415,17 @@ func TestGetPostsInTimeframe(t *testing.T) { var feed []*bsky.FeedDefs_FeedViewPost var cursor string if requests == 1 { - // Page 1: one out-of-range (before start), one out-of-range (after end), and a cursor. + // Page 1: one post after the window end (skipped), with a cursor. feed = []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/4", "2024-01-01T00:00:00Z"), } cursor = "page-2-cursor" } else { - // Page 2: two in-range posts and no cursor (end of feed). + // Page 2: two in-range posts, then one older than start (terminates). feed = []*bsky.FeedDefs_FeedViewPost{ - mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), mk("at://did:plc:testactor/app.bsky.feed.post/2", "2023-08-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/3", "2022-01-01T00:00:00Z"), } } @@ -493,6 +495,9 @@ func TestGetActor(t *testing.T) { if profile.Platform != "Bluesky" { t.Errorf("Expected platform 'Bluesky', got %q", profile.Platform) } + if profile.URL != "https://bsky.app/profile/"+actor { + t.Errorf("Expected URL %q, got %q", "https://bsky.app/profile/"+actor, profile.URL) + } if profile.FollowersCount != 100 || profile.FollowingCount != 50 || profile.PostCount != 200 { t.Errorf("Unexpected counts: followers=%d, following=%d, posts=%d", profile.FollowersCount, profile.FollowingCount, profile.PostCount) } @@ -529,6 +534,10 @@ func TestSearchActors(t *testing.T) { if result.Platform != "Bluesky" { t.Errorf("Result %d: expected platform 'Bluesky', got %q", i, result.Platform) } + expectedURL := "https://bsky.app/profile/" + expectedHandle + if result.URL != expectedURL { + t.Errorf("Result %d: expected URL %q, got %q", i, expectedURL, result.URL) + } } } @@ -583,6 +592,12 @@ func TestSanitizeFeedURI(t *testing.T) { expected: "at://did:plc:abc/app.bsky.feed.generator/xyz", wantErr: false, }, + { + name: "Non-HTTP scheme", + input: "gopher://bsky.app/profile/did:plc:abc/feed/xyz", + expected: "", + wantErr: true, + }, { name: "Invalid URI", input: "not-a-uri", @@ -609,3 +624,322 @@ func TestSanitizeFeedURI(t *testing.T) { }) } } + +func TestGetLatestNPostsLimitClamp(t *testing.T) { + server := setupMockServer() + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + actor := "did:plc:testactor" + + t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, 250) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output) != maxBskyFeedLimit { + t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output)) + } + }) + + t.Run("ZeroLimitReturnsNil", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, 0) + if err != nil { + t.Fatalf("Expected no error with zero limit, got %v", err) + } + if output != nil { + t.Errorf("Expected nil output for zero limit, got %d posts", len(output)) + } + }) + + t.Run("NegativeLimitReturnsNil", func(t *testing.T) { + output, err := client.GetLatestNPosts(ctx, actor, -5) + if err != nil { + t.Fatalf("Expected no error with negative limit, got %v", err) + } + if output != nil { + t.Errorf("Expected nil output for negative limit, got %d posts", len(output)) + } + }) +} + +func TestGetFeedHonorsLimit(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getFeed", func(w http.ResponseWriter, r *http.Request) { + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + feed := make([]*bsky.FeedDefs_FeedViewPost, 0, limit) + for i := 0; i < limit; i++ { + feed = append(feed, &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: fmt.Sprintf("at://did:plc:test/app.bsky.feed.post/%d", i), + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:test", Handle: "test.handle"}, + }, + }) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + feedURI := "at://did:plc:abc/app.bsky.feed.generator/xyz" + + t.Run("ExplicitLimitPassedThrough", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 75) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != 75 { + t.Errorf("Expected 75 posts, got %d", len(output.Feed)) + } + }) + + t.Run("ZeroLimitDefaultsTo50", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 0) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != 50 { + t.Errorf("Expected 50 posts (default), got %d", len(output.Feed)) + } + }) + + t.Run("OverMaxLimitClampedTo100", func(t *testing.T) { + output, err := client.GetFeed(ctx, feedURI, 1000) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(output.Feed) != maxBskyFeedLimit { + t.Errorf("Expected %d posts (clamped), got %d", maxBskyFeedLimit, len(output.Feed)) + } + }) +} + +func TestGetPostInvalidRecordKey(t *testing.T) { + requests := 0 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getPosts", func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, `{"error":"InternalServerError"}`, http.StatusInternalServerError) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + post, err := client.GetPost(ctx, "did:plc:testactor", "", "bad/key") + if post != nil { + t.Fatalf("Expected nil post, got %v", post) + } + if err == nil { + t.Fatal("Expected error for invalid record key, got nil") + } + if requests != 0 { + t.Errorf("Expected no network request for invalid record key, got %d", requests) + } +} + +func TestWithRefreshConcurrent(t *testing.T) { + var refreshCount atomic.Int32 + + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.actor.getProfile", func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth != "Bearer refreshed_access_jwt" { + // The initial access token is expired; the session must be refreshed + // before the retry succeeds. + w.WriteHeader(http.StatusUnauthorized) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "ExpiredToken", "message": "Token expired, refresh required"}) + return + } + resp := map[string]any{ + "did": "did:plc:test", + "handle": "test.handle", + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + mux.HandleFunc("/xrpc/com.atproto.server.refreshSession", func(w http.ResponseWriter, r *http.Request) { + refreshCount.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(atproto.ServerRefreshSession_Output{ + AccessJwt: "refreshed_access_jwt", + RefreshJwt: "refreshed_refresh_jwt", + }) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := client.GetActor(ctx, "test.handle"); err != nil { + t.Errorf("expected no error, got %v", err) + } + }() + } + wg.Wait() + + if refreshCount.Load() != 1 { + t.Errorf("expected exactly 1 session refresh under concurrency, got %d", refreshCount.Load()) + } +} + +func TestGetPostsInTimeframeStopsAtStart(t *testing.T) { + requests := 0 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { + requests++ + mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { + return &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: uri, + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, + Record: &util.LexiconTypeDecoder{ + Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, + }, + }, + } + } + feed := []*bsky.FeedDefs_FeedViewPost{ + mk("at://did:plc:testactor/app.bsky.feed.post/1", "2023-06-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/0", "2022-01-01T00:00:00Z"), + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) + + posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(posts) != 1 { + t.Fatalf("expected 1 in-range post before early exit, got %d", len(posts)) + } + if requests != 1 { + t.Errorf("expected scan to stop after the first page, made %d requests", requests) + } +} + +func TestGetPostsInTimeframeIncludesBoundaries(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/app.bsky.feed.getAuthorFeed", func(w http.ResponseWriter, r *http.Request) { + mk := func(uri, createdAt string) *bsky.FeedDefs_FeedViewPost { + return &bsky.FeedDefs_FeedViewPost{ + Post: &bsky.FeedDefs_PostView{ + Uri: uri, + Author: &bsky.ActorDefs_ProfileViewBasic{Did: "did:plc:testactor", Handle: "testactor.test"}, + Record: &util.LexiconTypeDecoder{ + Val: &bsky.FeedPost{Text: uri, CreatedAt: createdAt}, + }, + }, + } + } + feed := []*bsky.FeedDefs_FeedViewPost{ + mk("at://did:plc:testactor/app.bsky.feed.post/end", "2023-12-31T23:59:59Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/start", "2023-01-01T00:00:00Z"), + mk("at://did:plc:testactor/app.bsky.feed.post/older", "2022-12-31T23:59:59Z"), + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(bsky.FeedGetAuthorFeed_Output{Feed: feed}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + start := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC) + + posts, err := client.GetPostsInTimeframe(ctx, "did:plc:testactor", start, end, 10) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + // Both boundary posts are inclusive; the post one second before start is not. + if len(posts) != 2 { + t.Fatalf("expected boundary posts to be included (got %d)", len(posts)) + } + got := map[string]bool{} + for _, p := range posts { + got[p.Post.Uri] = true + } + if !got["at://did:plc:testactor/app.bsky.feed.post/start"] { + t.Error("expected post exactly at start to be included") + } + if !got["at://did:plc:testactor/app.bsky.feed.post/end"] { + t.Error("expected post exactly at end to be included") + } +} + +func TestResolveUserCache(t *testing.T) { + var resolveRequests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/xrpc/com.atproto.identity.resolveHandle", func(w http.ResponseWriter, r *http.Request) { + resolveRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(atproto.IdentityResolveHandle_Output{Did: "did:plc:resolved"}) + }) + + server := httptest.NewServer(mux) + defer server.Close() + + client := newTestClient(t, server.URL) + ctx := context.Background() + + // First resolution hits the network and populates the cache. + did, err := client.ResolveUser(ctx, "test.handle") + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if did.String() != "did:plc:resolved" { + t.Errorf("Unexpected DID %q", did.String()) + } + if resolveRequests.Load() != 1 { + t.Errorf("expected 1 resolve request, got %d", resolveRequests.Load()) + } + + // Second resolution is served from cache. + if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { + t.Fatalf("Expected no error on cached resolution, got %v", err) + } + if resolveRequests.Load() != 1 { + t.Errorf("expected cached resolution to skip the network, got %d requests", resolveRequests.Load()) + } + + // An expired cache entry is refetched. + client.cacheMu.Lock() + entry := client.didCache["test.handle"] + entry.expiresAt = time.Now().Add(-time.Minute) + client.didCache["test.handle"] = entry + client.cacheMu.Unlock() + + if _, err := client.ResolveUser(ctx, "test.handle"); err != nil { + t.Fatalf("Expected no error after expiry, got %v", err) + } + if resolveRequests.Load() != 2 { + t.Errorf("expected a fresh resolve after TTL expiry, got %d requests", resolveRequests.Load()) + } +} From 4f107da9eecc8c9d8b26550f788f6353d1406d8c Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:17:56 +0000 Subject: [PATCH 18/30] fix: harden Nostr client relay handling and error propagation - budget relay queries across a shared 20s deadline (5s per relay) and dedupe events by ID across relays - surface 'could not connect to any relay' when no query succeeded and abort early on relay-connect and stream-cancel errors - guard zero/negative limits across all fetchers - GetActor falls through relays and rejects empty profile metadata; SearchActors propagates relay errors instead of silently returning none - NewNostrClient validates relay ws://wss:// schemes, trims, and dedupes --- internal/nostr/client.go | 486 +++++++++++++++++----------------- internal/nostr/client_test.go | 82 +++++- 2 files changed, 315 insertions(+), 253 deletions(-) diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 0aa628f..952045d 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "net/url" + "strings" "time" "fiatjaf.com/nostr" @@ -12,15 +14,44 @@ import ( "github.com/nacorid/naco-api/internal/actor" ) +const ( + // relayQueryTimeout bounds connect + subscribe + collect against a single + // relay. The websocket dial, subscription handshake and event collection all + // share this budget. + relayQueryTimeout = 5 * time.Second + + // maxTotalRelayTime caps the fan-out across all relays when the caller's + // context carries no deadline of its own. Without it a long relay list could + // multiply the per-relay budget unboundedly. + maxTotalRelayTime = 20 * time.Second +) + type Client struct { relays []string } func NewNostrClient(relays []string) (*Client, error) { - if len(relays) == 0 { + seen := make(map[string]struct{}, len(relays)) + cleaned := make([]string, 0, len(relays)) + for _, r := range relays { + r = strings.TrimSpace(r) + if r == "" { + continue + } + u, err := url.Parse(r) + if err != nil || (u.Scheme != "ws" && u.Scheme != "wss") { + return nil, fmt.Errorf("invalid relay URL %q: must be a ws:// or wss:// endpoint", r) + } + if _, dup := seen[r]; dup { + continue + } + seen[r] = struct{}{} + cleaned = append(cleaned, r) + } + if len(cleaned) == 0 { return nil, fmt.Errorf("at least one relay is required") } - return &Client{relays: relays}, nil + return &Client{relays: cleaned}, nil } type Post struct { @@ -30,221 +61,149 @@ type Post struct { CreatedAt time.Time } +// queryBudget returns a context bounded to maxTotalRelayTime when ctx has no +// deadline, so the relay fan-out cannot run away across many relays. Callers +// must invoke the returned cancel. +func (c *Client) queryBudget(ctx context.Context) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, maxTotalRelayTime) +} + +// collectFromRelay connects to url within ctx, subscribes with filter and +// appends matching events to posts (deduplicated by event id) until the limit is +// reached, EOSE arrives, or ctx is done. It reports whether the relay was +// actually queryable (connected + subscribed), letting callers distinguish a +// total relay outage from a legitimate empty result. +func (c *Client) collectFromRelay(ctx context.Context, url string, filter nostr.Filter, limit int, posts *[]*Post, seen map[string]struct{}) (used bool) { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + return false + } + defer relay.Close() + + sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) + if err != nil { + return false + } + defer sub.Unsub() + + for { + select { + case <-sub.EndOfStoredEvents: + return true + case ev, ok := <-sub.Events: + if !ok { + return true + } + id := ev.ID.Hex() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + *posts = append(*posts, &Post{ + ID: id, + Author: ev.PubKey.Hex(), + Content: ev.Content, + CreatedAt: ev.CreatedAt.Time(), + }) + if len(*posts) >= limit { + return true + } + case <-ctx.Done(): + return true + } + } +} + +// collectPosts subscribes to filter across every relay, stopping once limit posts +// are collected or the query context is done. It returns the deduplicated posts, +// or an error if no relay could be queried at all or the operation was aborted. +func (c *Client) collectPosts(ctx context.Context, filter nostr.Filter, limit int) ([]*Post, error) { + posts := make([]*Post, 0, limit) + seen := make(map[string]struct{}) + + queryCtx, cancel := c.queryBudget(ctx) + defer cancel() + + var anyUsed bool + for _, url := range c.relays { + if queryCtx.Err() != nil { + break + } + relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) + used := c.collectFromRelay(relayCtx, url, filter, limit, &posts, seen) + relayCancel() + anyUsed = anyUsed || used + if len(posts) >= limit { + break + } + } + + if err := queryCtx.Err(); err != nil { + return nil, fmt.Errorf("nostr query aborted: %w", err) + } + if !anyUsed { + return nil, fmt.Errorf("could not connect to any relay") + } + return posts, nil +} + func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { eid, err := resolveEventID(id) if err != nil { return nil, err } - connected := false - - for _, url := range c.relays { - post, found, err := func() (*Post, bool, error) { - relayCtx, cancel := context.WithTimeout(ctx, 4*time.Second) - defer cancel() - - relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) - if err != nil { - return nil, false, nil // Try next relay - } - connected = true - defer relay.Close() - - sub, err := relay.Subscribe(relayCtx, nostr.Filter{ - IDs: []nostr.ID{eid}, - Kinds: []nostr.Kind{1}, - }, nostr.SubscriptionOptions{}) - if err != nil { - return nil, false, nil - } - defer sub.Unsub() - - select { - case <-sub.EndOfStoredEvents: - return nil, false, nil - case ev, ok := <-sub.Events: - if !ok { - return nil, false, nil - } - return &Post{ - ID: ev.ID.Hex(), - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }, true, nil - case <-ctx.Done(): - return nil, false, ctx.Err() - } - }() - - if err != nil { - return nil, err - } - if found { - return post, nil - } + filter := nostr.Filter{ + IDs: []nostr.ID{eid}, + Kinds: []nostr.Kind{1}, } - - if !connected { - return nil, fmt.Errorf("could not connect to any relay") + posts, err := c.collectPosts(ctx, filter, 1) + if err != nil { + return nil, err } - return nil, fmt.Errorf("post %q not found across configured relays", id) + if len(posts) == 0 { + return nil, fmt.Errorf("post %q not found across configured relays", id) + } + return posts[0], nil } func (c *Client) GetLatestNPosts(ctx context.Context, author string, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } pubkey, err := resolvePubkey(author) if err != nil { return nil, err } - posts := make([]*Post, 0, limit) - seen := make(map[string]struct{}) - connected := false - - for _, url := range c.relays { - func() { - relayCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) - if err != nil { - return - } - connected = true - defer relay.Close() - - sub, err := relay.Subscribe(relayCtx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - return - } - defer sub.Unsub() - - collect: - for { - select { - case <-sub.EndOfStoredEvents: - break collect - case ev, ok := <-sub.Events: - if !ok { - break collect - } - id := ev.ID.Hex() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - posts = append(posts, &Post{ - ID: id, - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }) - - if len(posts) >= limit { - break collect - } - case <-ctx.Done(): - break collect - case <-relayCtx.Done(): - break collect - } - } - }() - - if len(posts) >= limit || ctx.Err() != nil { - break - } + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Limit: limit, } - - if !connected && len(posts) == 0 { - return nil, fmt.Errorf("could not connect to any relay") - } - return posts, nil + return c.collectPosts(ctx, filter, limit) } func (c *Client) GetPostsInTimeframe(ctx context.Context, author string, start, end time.Time, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } pubkey, err := resolvePubkey(author) if err != nil { return nil, err } - since := nostr.Timestamp(start.Unix()) - until := nostr.Timestamp(end.Unix()) - - posts := make([]*Post, 0, limit) - seen := make(map[string]struct{}) - connected := false - - for _, url := range c.relays { - func() { - relayCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - relay, err := nostr.RelayConnect(relayCtx, url, nostr.RelayOptions{}) - if err != nil { - return - } - connected = true - defer relay.Close() - - sub, err := relay.Subscribe(relayCtx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{1}, - Since: since, - Until: until, - Limit: limit, - }, nostr.SubscriptionOptions{}) - if err != nil { - return - } - defer sub.Unsub() - - collect: - for { - select { - case <-sub.EndOfStoredEvents: - break collect - case ev, ok := <-sub.Events: - if !ok { - break collect - } - id := ev.ID.Hex() - if _, dup := seen[id]; dup { - continue - } - seen[id] = struct{}{} - posts = append(posts, &Post{ - ID: id, - Author: ev.PubKey.Hex(), - Content: ev.Content, - CreatedAt: ev.CreatedAt.Time(), - }) - - if len(posts) >= limit { - break collect - } - case <-ctx.Done(): - break collect - case <-relayCtx.Done(): - break collect - } - } - }() - - if len(posts) >= limit || ctx.Err() != nil { - break - } + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{1}, + Since: nostr.Timestamp(start.Unix()), + Until: nostr.Timestamp(end.Unix()), + Limit: limit, } - - if !connected && len(posts) == 0 { - return nil, fmt.Errorf("could not connect to any relay") - } - - return posts, nil + return c.collectPosts(ctx, filter, limit) } func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, error) { @@ -254,79 +213,108 @@ func (c *Client) GetActor(ctx context.Context, author string) (*actor.Profile, e } npub := nip19.EncodeNpub(pubkey) - for _, u := range c.relays { - profile, found, err := func() (*actor.Profile, bool, error) { - relayCtx, cancel := context.WithTimeout(ctx, 4*time.Second) - defer cancel() + filter := nostr.Filter{ + Authors: []nostr.PubKey{pubkey}, + Kinds: []nostr.Kind{0}, + Limit: 1, + } - relay, err := nostr.RelayConnect(relayCtx, u, nostr.RelayOptions{}) - if err != nil { - return nil, false, nil - } - defer relay.Close() + queryCtx, cancel := c.queryBudget(ctx) + defer cancel() - sub, err := relay.Subscribe(relayCtx, nostr.Filter{ - Authors: []nostr.PubKey{pubkey}, - Kinds: []nostr.Kind{0}, - Limit: 1, - }, nostr.SubscriptionOptions{}) - if err != nil { - return nil, false, nil - } - defer sub.Unsub() - - select { - case <-sub.EndOfStoredEvents: - return nil, false, nil - case ev, ok := <-sub.Events: - if !ok { - return nil, false, nil - } - var meta struct { - Name string `json:"name"` - DisplayName string `json:"display_name"` - About string `json:"about"` - Picture string `json:"picture"` - Website string `json:"website"` - } - if err := json.Unmarshal([]byte(ev.Content), &meta); err == nil { - displayName := meta.DisplayName - if displayName == "" { - displayName = meta.Name - } - return &actor.Profile{ - Platform: "Nostr", - Handle: npub, - DisplayName: displayName, - AvatarURL: meta.Picture, - Bio: meta.About, - URL: "https://njump.me/" + npub, - }, true, nil - } - return nil, false, nil - case <-ctx.Done(): - return nil, false, ctx.Err() - } - }() - - if err != nil { - return nil, err + var anyUsed bool + for _, url := range c.relays { + if queryCtx.Err() != nil { + break } - if found { + relayCtx, relayCancel := context.WithTimeout(queryCtx, relayQueryTimeout) + profile, used := c.actorFromRelay(relayCtx, url, filter, npub) + relayCancel() + anyUsed = anyUsed || used + if profile != nil { return profile, nil } } + if err := queryCtx.Err(); err != nil { + return nil, fmt.Errorf("nostr query aborted: %w", err) + } + if !anyUsed { + return nil, fmt.Errorf("could not connect to any relay") + } return nil, fmt.Errorf("no profile metadata found for %q", author) } +// actorFromRelay looks up one kind-0 metadata event on url. It returns a +// non-nil profile when a usable metadata event was found. used reports whether +// the relay was queried at all (so a fully unreachable relay list is +// distinguishable from an author without metadata). +func (c *Client) actorFromRelay(ctx context.Context, url string, filter nostr.Filter, npub string) (profile *actor.Profile, used bool) { + relay, err := nostr.RelayConnect(ctx, url, nostr.RelayOptions{}) + if err != nil { + return nil, false + } + defer relay.Close() + + sub, err := relay.Subscribe(ctx, filter, nostr.SubscriptionOptions{}) + if err != nil { + return nil, false + } + defer sub.Unsub() + + select { + case <-sub.EndOfStoredEvents: + return nil, true + case ev, ok := <-sub.Events: + if !ok { + return nil, true + } + return profileFromEvent(ev, npub), true + case <-ctx.Done(): + return nil, true + } +} + +// profileFromEvent builds an actor.Profile from a kind-0 metadata event, or nil +// when the content is not valid JSON or carries no usable fields. +func profileFromEvent(ev nostr.Event, npub string) *actor.Profile { + var meta struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + About string `json:"about"` + Picture string `json:"picture"` + Website string `json:"website"` + } + if err := json.Unmarshal([]byte(ev.Content), &meta); err != nil { + return nil + } + if meta.Name == "" && meta.DisplayName == "" && meta.About == "" && meta.Picture == "" { + return nil + } + displayName := meta.DisplayName + if displayName == "" { + displayName = meta.Name + } + return &actor.Profile{ + Platform: "Nostr", + Handle: npub, + DisplayName: displayName, + AvatarURL: meta.Picture, + Bio: meta.About, + URL: "https://njump.me/" + npub, + } +} + func (c *Client) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { if _, err := resolvePubkey(query); err != nil { - return []*actor.Profile{}, nil + // Not a pubkey we can resolve (e.g. a name query). Treat it as no match + // rather than an error so a cross-platform search isn't failed by a + // non-nostr query. + return nil, nil } prof, err := c.GetActor(ctx, query) if err != nil { - return []*actor.Profile{}, nil + return nil, err } return []*actor.Profile{prof}, nil } diff --git a/internal/nostr/client_test.go b/internal/nostr/client_test.go index 0980361..e65bbb1 100644 --- a/internal/nostr/client_test.go +++ b/internal/nostr/client_test.go @@ -189,6 +189,80 @@ func TestRelayConnectionFailureReturnsError(t *testing.T) { } } +func TestNewNostrClientInvalidScheme(t *testing.T) { + _, err := NewNostrClient([]string{"http://relay.example"}) + if err == nil { + t.Fatal("expected error for non-websocket relay URL") + } + if !strings.Contains(err.Error(), "invalid relay URL") { + t.Fatalf("expected 'invalid relay URL' error, got %v", err) + } + + // Blank and duplicate entries are dropped, the scheme is validated. + c, err := NewNostrClient([]string{ + "wss://relay.a", + " wss://relay.a ", + "ws://relay.b", + " ", + }) + if err != nil { + t.Fatal(err) + } + if len(c.relays) != 2 { + t.Fatalf("expected 2 unique relays after cleaning, got %d: %v", len(c.relays), c.relays) + } +} + +func TestGetLatestNPostsZeroOrNegativeLimit(t *testing.T) { + c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) + if err != nil { + t.Fatal(err) + } + + sk := nostr.Generate() + npub := nip19.EncodeNpub(sk.Public()) + + if posts, err := c.GetLatestNPosts(context.Background(), npub, 0); err != nil || posts != nil { + t.Fatalf("expected nil, nil for zero limit, got %v, %v", posts, err) + } + if posts, err := c.GetLatestNPosts(context.Background(), npub, -5); err != nil || posts != nil { + t.Fatalf("expected nil, nil for negative limit, got %v, %v", posts, err) + } + + start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC) + if posts, err := c.GetPostsInTimeframe(context.Background(), npub, start, end, 0); err != nil || posts != nil { + t.Fatalf("expected nil, nil for zero timeframe limit, got %v, %v", posts, err) + } +} + +func TestSearchActors(t *testing.T) { + c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + + // A non-key query (e.g. a display name) is a valid no-match, not an error. + results, err := c.SearchActors(ctx, "alice", 1) + if err != nil { + t.Fatalf("expected no error for non-key query, got %v", err) + } + if results != nil { + t.Fatalf("expected nil results for non-key query, got %v", results) + } + + // A valid pubkey that fails on every relay must surface the outage. + sk := nostr.Generate() + npub := nip19.EncodeNpub(sk.Public()) + if _, err := c.SearchActors(ctx, npub, 1); err == nil { + t.Fatal("expected error when no relay is reachable") + } else if !strings.Contains(err.Error(), "could not connect to any relay") { + t.Fatalf("expected 'could not connect to any relay' error, got %v", err) + } +} + func TestGetPostWithHexID(t *testing.T) { c, err := NewNostrClient([]string{"wss://invalid.relay.invalid"}) if err != nil { @@ -223,9 +297,9 @@ func TestGetActorWithHexPubkey(t *testing.T) { if err == nil { t.Fatal("expected error when relay unavailable") } - // GetActor surfaces the failure as an absence of a profile after the relay - // loop runs without connecting. - if !strings.Contains(err.Error(), "no profile metadata found") { - t.Fatalf("expected 'no profile metadata found' error, got %v", err) + // GetActor reports a total relay outage as a connection failure rather than + // pretending the author has no metadata. + if !strings.Contains(err.Error(), "could not connect to any relay") { + t.Fatalf("expected 'could not connect to any relay' error, got %v", err) } } From 67d35a0a2f79f12e14b37028ca4409fa07f9341d Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:18:01 +0000 Subject: [PATCH 19/30] fix: harden Ollama client embeddings, truncation, and retries - embed posts in a bounded worker pool while preserving index order, with per-post error context and transient (429/5xx) retry backoff - SummarizeStyle errors when the stream ends before a done chunk or the model truncates at its context limit (done_reason length) instead of returning a partial summary for a payable tool - New() drops the unused ctx param and names its model params; callers in main.go updated --- cmd/server/main.go | 2 +- internal/ollama/client.go | 209 ++++++++++++++++++++++++++++----- internal/ollama/client_test.go | 153 +++++++++++++++++++++++- 3 files changed, 332 insertions(+), 32 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 88d676d..b42c61f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -83,7 +83,7 @@ func main() { ollamaHTTPClient := &http.Client{ Timeout: 600 * time.Second, } - ollamaClient := ollama.New(backgroundCtx, uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) + ollamaClient := ollama.New(uri, ollamaHTTPClient, cfg.EmbeddingModel, cfg.StyleModel) apClient, err := activitypub.NewAPClient(cfg.ActivityPubHost, cfg.ActivityPubToken) if err != nil { diff --git a/internal/ollama/client.go b/internal/ollama/client.go index a26f3a8..60515ae 100644 --- a/internal/ollama/client.go +++ b/internal/ollama/client.go @@ -2,52 +2,150 @@ package ollama import ( "context" + "errors" + "fmt" "net/http" "net/url" "strings" + "sync" + "time" "github.com/nacorid/naco-api/internal/clustering" "github.com/ollama/ollama/api" ) +const ( + // embedConcurrency caps how many embedding requests run in parallel. Ollama + // serializes model requests server-side (OLLAMA_NUM_PARALLEL defaults to 1), + // so unbounded fan-out would only add latency and memory for no throughput. + embedConcurrency = 4 + + // embedRetries and embedRetryBackoff cover transient server-side embedding + // failures (e.g. cold model load, transient 5xx/429). + embedRetries = 2 + embedRetryBackoff = 150 * time.Millisecond + + // generateRetries and generateRetryBackoff cover transient failures that + // produce no output at all; a truncated stream is never retried because the + // partial response cannot be safely replayed. + generateRetries = 1 + generateRetryBackoff = 500 * time.Millisecond +) + type OllamaClient struct { client api.Client embeddingModel string styleModel string } -func New(ctx context.Context, host *url.URL, http *http.Client, em, sm string) *OllamaClient { - client := api.NewClient(host, http) - return &OllamaClient{client: *client, embeddingModel: em, styleModel: sm} +func New(host *url.URL, httpClient *http.Client, embeddingModel, styleModel string) *OllamaClient { + client := api.NewClient(host, httpClient) + return &OllamaClient{client: *client, embeddingModel: embeddingModel, styleModel: styleModel} } +// EmbedTexts computes one embedding per post. Embeddings are returned in the +// same order as posts; callers rely on index correspondence (e.g. +// items[i].Embedding = embeddings[i]) so posts are never skipped. Requests run +// in a bounded worker pool and transient failures are retried with backoff. func (o *OllamaClient) EmbedTexts(ctx context.Context, posts []clustering.Item) ([][]float64, error) { if len(posts) == 0 { return nil, nil } - embeddings := make([][]float64, 0, len(posts)) - for _, post := range posts { + embeddings := make([][]float64, len(posts)) + + // poolCtx is a child of ctx: cancelling it on the first failure must not + // cancel the caller's request context. + poolCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + mu sync.Mutex + firstErr error + sema = make(chan struct{}, embedConcurrency) + wg sync.WaitGroup + ) + for i, post := range posts { + wg.Add(1) + go func(idx int, text string) { + defer wg.Done() + select { + case sema <- struct{}{}: + case <-poolCtx.Done(): + return + } + defer func() { <-sema }() + + emb, err := o.embedWithRetry(poolCtx, strings.TrimSpace(text)) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("embedding post %d (%q): %w", idx, text, err) + } + mu.Unlock() + cancel() + return + } + embeddings[idx] = emb + }(i, post.Text) + } + wg.Wait() + + if firstErr := atomicErr(&mu, &firstErr); firstErr != nil { + return embeddings, firstErr + } + // Every worker may have exited via the poolCtx.Done() branch without + // recording an error, e.g. when the caller's context is already canceled. + if err := ctx.Err(); err != nil { + return nil, err + } + return embeddings, nil +} + +func atomicErr(mu *sync.Mutex, err *error) error { + mu.Lock() + defer mu.Unlock() + return *err +} + +func (o *OllamaClient) embedWithRetry(ctx context.Context, text string) ([]float64, error) { + backoff := embedRetryBackoff + var lastErr error + for attempt := 0; attempt <= embedRetries; attempt++ { + if attempt > 0 { + if err := sleepCtx(ctx, backoff); err != nil { + return nil, err + } + backoff *= 2 + } resp, err := o.client.Embeddings(ctx, &api.EmbeddingRequest{ Model: o.embeddingModel, - Prompt: post.Text, + Prompt: text, }) - if err != nil { + if err == nil { + return resp.Embedding, nil + } + lastErr = err + if !isTransientErr(err) { return nil, err } - embeddings = append(embeddings, resp.Embedding) } - - return embeddings, nil + return nil, lastErr } func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.Item) (string, error) { if len(posts) == 0 { return "", nil } - var promptParts []string + + promptParts := make([]string, 0, len(posts)) for _, post := range posts { - promptParts = append(promptParts, post.Text) + if text := strings.TrimSpace(post.Text); text != "" { + promptParts = append(promptParts, text) + } + } + if len(promptParts) == 0 { + return "", fmt.Errorf("no post text to summarize") } prompt := "Summarize the user's writing style. Analyze:\n" + @@ -62,23 +160,74 @@ func (o *OllamaClient) SummarizeStyle(ctx context.Context, posts []clustering.It "\n\nStyle summary:" var fullResponse strings.Builder - err := o.client.Generate(ctx, &api.GenerateRequest{ - Model: o.styleModel, - Prompt: prompt, - }, func(resp api.GenerateResponse) error { - fullResponse.WriteString(resp.Response) - return nil - }) - if err != nil { - return "", err - } - // The Ollama client's stream reader can finish without an error when the - // request context is canceled mid-stream (e.g. a dead server connection - // closing cleanly); surface the cancellation explicitly rather than - // returning a truncated summary as if it were complete. - if err := ctx.Err(); err != nil { - return "", err - } + backoff := generateRetryBackoff + var lastErr error + for attempt := 0; attempt <= generateRetries; attempt++ { + if attempt > 0 { + if err := sleepCtx(ctx, backoff); err != nil { + return "", err + } + } - return fullResponse.String(), nil + // Reset between attempts so a failed run never leaks a partial summary. + fullResponse.Reset() + done := false + doneReason := "" + err := o.client.Generate(ctx, &api.GenerateRequest{ + Model: o.styleModel, + Prompt: prompt, + }, func(resp api.GenerateResponse) error { + fullResponse.WriteString(resp.Response) + if resp.Done { + done = true + doneReason = resp.DoneReason + } + return nil + }) + if err != nil { + lastErr = err + // Only retry transport/5xx failures that produced no output; a + // partial stream must not be replayed. + if !isTransientErr(err) || fullResponse.Len() > 0 { + return "", fmt.Errorf("ollama generate with model %q failed: %w", o.styleModel, err) + } + continue + } + + // The underlying stream reader can finish without an error when the + // request context is canceled or the connection drops mid-stream (the + // library never inspects bufio.Scanner.Err), so a bare nil from + // Generate does not mean the response is complete. + if ctxErr := ctx.Err(); ctxErr != nil { + return "", ctxErr + } + if !done { + return "", fmt.Errorf("ollama stream for model %q ended before completion", o.styleModel) + } + if doneReason == "length" { + return "", fmt.Errorf("ollama response for model %q was truncated (context limit reached)", o.styleModel) + } + return fullResponse.String(), nil + } + return "", fmt.Errorf("ollama generate with model %q failed after retries: %w", o.styleModel, lastErr) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-time.After(d): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// isTransientErr reports whether err is a retryable server-side failure. Non-2xx +// HTTP errors that carry a StatusCode (429, 5xx) qualify; context and transport +// errors below that never do, so they surface immediately. +func isTransientErr(err error) bool { + var se api.StatusError + if errors.As(err, &se) { + return se.StatusCode == http.StatusTooManyRequests || se.StatusCode >= http.StatusInternalServerError + } + return false } diff --git a/internal/ollama/client_test.go b/internal/ollama/client_test.go index 892ac73..9ade0e3 100644 --- a/internal/ollama/client_test.go +++ b/internal/ollama/client_test.go @@ -3,11 +3,13 @@ package ollama import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "net/url" "strings" "sync" + "sync/atomic" "testing" "time" @@ -24,7 +26,7 @@ func setupTestServer(t *testing.T, handler http.Handler) (*httptest.Server, *Oll t.Fatalf("failed to parse test server URL: %v", err) } - client := New(context.Background(), host, http.DefaultClient, "test-embed", "test-style") + client := New(host, http.DefaultClient, "test-embed", "test-style") return server, client } @@ -180,6 +182,103 @@ func TestEmbedTexts_ContextCanceled(t *testing.T) { } } +func TestEmbedTexts_RetryThenSuccess(t *testing.T) { + var requests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{9.9}, + }) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + embeddings, err := client.EmbedTexts(context.Background(), posts) + if err != nil { + t.Fatalf("EmbedTexts failed after transient error: %v", err) + } + if len(embeddings) != 1 || len(embeddings[0]) != 1 || embeddings[0][0] != 9.9 { + t.Fatalf("unexpected embeddings after retry: %v", embeddings) + } + if got := requests.Load(); got != 2 { + t.Errorf("expected 2 requests (1 failure + 1 retry), got %d", got) + } +} + +func TestEmbedTexts_RetryExhausted(t *testing.T) { + var requests atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"transient model load failure"}`)) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + _, err := client.EmbedTexts(context.Background(), posts) + if err == nil { + t.Fatal("expected error after retries exhausted, got nil") + } + if got := requests.Load(); got != embedRetries+1 { + t.Errorf("expected %d attempts (initial + %d retries), got %d", embedRetries+1, embedRetries, got) + } +} + +func TestEmbedTexts_ParallelOrder(t *testing.T) { + // Shake ordering: handler sleeps and varies the embedding by prompt length, + // so a naive (unordered) worker pool would misalign results. + mux := http.NewServeMux() + mux.HandleFunc("/api/embeddings", func(w http.ResponseWriter, r *http.Request) { + var req struct { + Prompt string `json:"prompt"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode request body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + time.Sleep(30 * time.Millisecond) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "embedding": []float64{float64(len(req.Prompt))}, + }) + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := make([]clustering.Item, 8) + want := make([]float64, 8) + for i := range posts { + posts[i] = clustering.Item{ID: fmt.Sprintf("%d", i), Text: strings.Repeat("x", i+1)} + want[i] = float64(i + 1) + } + + embeddings, err := client.EmbedTexts(context.Background(), posts) + if err != nil { + t.Fatalf("EmbedTexts failed: %v", err) + } + if len(embeddings) != len(posts) { + t.Fatalf("expected %d embeddings, got %d", len(posts), len(embeddings)) + } + for i, emb := range embeddings { + if len(emb) != 1 || emb[0] != want[i] { + t.Errorf("embedding %d out of order or wrong value: got %v, want %v", i, emb, want[i]) + } + } +} + // ============================================================================ // SummarizeStyle Tests // ============================================================================ @@ -363,3 +462,55 @@ func TestSummarizeStyle_ContextCanceled(t *testing.T) { t.Fatal("expected error on canceled context, got nil") } } + +func TestSummarizeStyle_StreamNeverCompletes(t *testing.T) { + // Streaming ends without a done:true chunk (simulates a dropped connection + // or the http.Client timeout firing mid-generation). + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = w.Write([]byte(`{"response":"partial answer"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + }) // handler returns without ever writing done:true + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error when stream ends without completion, got nil") + } + if !strings.Contains(err.Error(), "ended before completion") { + t.Errorf("expected 'ended before completion' error, got %v", err) + } +} + +func TestSummarizeStyle_LengthTruncated(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + flusher, _ := w.(http.Flusher) + w.Header().Set("Content-Type", "application/x-ndjson") + _, _ = w.Write([]byte(`{"response":"summary...","done":true,"done_reason":"length"}` + "\n")) + if flusher != nil { + flusher.Flush() + } + }) + + server, client := setupTestServer(t, mux) + defer server.Close() + + posts := []clustering.Item{{ID: "1", Text: "hello"}} + + _, err := client.SummarizeStyle(context.Background(), posts) + if err == nil { + t.Fatal("expected error when response is truncated at context limit, got nil") + } + if !strings.Contains(err.Error(), "truncated") { + t.Errorf("expected 'truncated' error, got %v", err) + } +} From 84fbad705dc1931b48483694ac24d2f024b8eccf Mon Sep 17 00:00:00 2001 From: Nacorid Date: Wed, 9 Sep 2026 23:18:34 +0000 Subject: [PATCH 20/30] fix: harden Twitter API client timeout, retries, and authors - replace http.DefaultClient with a 60s-timeout client and send a User-Agent header - retry 5xx and transport errors with the same ctx-aware backoff used for 429s; drop the unreachable trailing return and defer-in-loop body - resolveUser returns the username; posts carry the handle (was numeric ID), error on 200-with-empty-data, and skip empty-ID tweets - remove the lower limit floor so small limits pass through - AGENTS.md: refresh the stale twitter clamp/backoff gotcha --- AGENTS.md | 2 +- internal/twitter/client.go | 136 +++++++++++++++++--------------- internal/twitter/client_test.go | 95 +++++++++++++++++++++- 3 files changed, 164 insertions(+), 69 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 23f17b9..41c77f9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,5 +91,5 @@ Loaded in order: `config/models.env` → `.env` (gitignored, via `godotenv`) → - Unix socket MCP + TCP readyz sidecar; `mcpServer.service` expects `bin/main` at repo root. - Platform inferred from author handle in `determinePlatform`/`fetch*` — not caller-supplied. - Bluesky auth refresh via `isAuthError`/`withRefresh` — 401/expired token retried once. -- Twitter client has per-limit clamp (`>100`/`>50` rejected) and `x-rate-limit-*`/`Retry-After` backoff. +- Twitter client clamps `limit` (`>100` → 100), retries 429/5xx/transport errors with `x-rate-limit-*`/`Retry-After` backoff (bounded by `maxRateLimitWait`), and has a 60s per-request timeout. - Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). diff --git a/internal/twitter/client.go b/internal/twitter/client.go index 4adb85e..f849796 100644 --- a/internal/twitter/client.go +++ b/internal/twitter/client.go @@ -25,6 +25,12 @@ func parseTwitterTime(s string) time.Time { const apiBase = "https://api.x.com/2" +// httpTimeout bounds a single HTTP request so a stalled X backend cannot block +// a request indefinitely. +const httpTimeout = 60 * time.Second + +const userAgent = "naco-api/1.0" + type Client struct { httpClient *http.Client bearerToken string @@ -32,7 +38,7 @@ type Client struct { func NewTwitterClient(bearerToken string) *Client { return &Client{ - httpClient: http.DefaultClient, + httpClient: &http.Client{Timeout: httpTimeout}, bearerToken: bearerToken, } } @@ -115,21 +121,31 @@ var defaultBackoff = 500 * time.Millisecond func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { backoff := defaultBackoff - for attempt := 0; attempt <= maxRetries; attempt++ { + for attempt := 0; ; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, fmt.Errorf("failed to build request: %w", err) } req.Header.Set("Authorization", "Bearer "+c.bearerToken) + req.Header.Set("User-Agent", userAgent) resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to do request: %w", err) + // Transport-level failure (connection refused/reset, timeout). + if attempt >= maxRetries { + return nil, fmt.Errorf("twitter request failed after %d attempts: %w", maxRetries+1, err) + } + log.WarnWithContext(ctx, "twitter transport error, waiting to retry", "attempt", attempt+1, "error", err) + if werr := waitForRetry(ctx, backoff); werr != nil { + return nil, werr + } + backoff *= 2 + continue } // Handle 429 Too Many Requests if resp.StatusCode == http.StatusTooManyRequests { - // Drain & close body using Discard (no allocation) + // Drain & close body before retrying. _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() @@ -145,27 +161,30 @@ func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { } log.WarnWithContext(ctx, "twitter rate limited, waiting to retry", "attempt", attempt+1, "wait", wait.String()) - - // Safe timer that frees resources on context cancellation - timer := time.NewTimer(wait) - select { - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return nil, ctx.Err() + if werr := waitForRetry(ctx, wait); werr != nil { + return nil, werr } - - backoff *= 2 // Exponential backoff for subsequent attempts + backoff *= 2 continue } - // Read response body once - defer resp.Body.Close() body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) + _ = resp.Body.Close() if err != nil { return nil, fmt.Errorf("failed to read response: %w", err) } + // Retry transient server-side failures (5xx) with backoff. 4xx/3xx fail + // fast below. + if resp.StatusCode >= http.StatusInternalServerError && attempt < maxRetries { + log.WarnWithContext(ctx, "twitter server error, waiting to retry", "attempt", attempt+1, "status", resp.Status) + if werr := waitForRetry(ctx, backoff); werr != nil { + return nil, werr + } + backoff *= 2 + continue + } + if resp.StatusCode != http.StatusOK { errMsg := strings.TrimSpace(string(body)) if len(errMsg) > 512 { @@ -176,8 +195,20 @@ func (c *Client) doGet(ctx context.Context, endpoint string) ([]byte, error) { return body, nil } +} - return nil, fmt.Errorf("twitter request failed after %d attempts", maxRetries) +// waitForRetry sleeps for wait, aborting early if the context is canceled. +// A timer is used (not time.After) so the timer can be stopped to free its +// resource when the caller abandons the wait. +func waitForRetry(ctx context.Context, wait time.Duration) error { + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } } // parseRateLimitWait extracts x-rate-limit-* or Retry-After, falling back to backoff. @@ -207,20 +238,23 @@ func parseRateLimitWait(header http.Header, fallback time.Duration) time.Duratio return fallback } -func (c *Client) resolveUser(ctx context.Context, username string) (string, error) { +func (c *Client) resolveUser(ctx context.Context, username string) (id, handle string, err error) { endpoint := fmt.Sprintf("%s/users/by/username/%s", apiBase, url.PathEscape(username)) body, err := c.doGet(ctx, endpoint) if err != nil { - return "", err + return "", "", err } var ur userResponse if err := json.Unmarshal(body, &ur); err != nil { - return "", fmt.Errorf("failed to unmarshal user response: %w", err) + return "", "", fmt.Errorf("failed to unmarshal user response: %w", err) } if len(ur.Errors) > 0 { - return "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) + return "", "", fmt.Errorf("twitter error: %s: %s", ur.Errors[0].Title, ur.Errors[0].Detail) } - return ur.Data.ID, nil + if ur.Data.ID == "" { + return "", "", fmt.Errorf("twitter user %q not found", username) + } + return ur.Data.ID, ur.Data.Username, nil } func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ([]*Post, error) { @@ -229,7 +263,7 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( } username := strings.TrimPrefix(actor, "@") - userID, err := c.resolveUser(ctx, username) + userID, handle, err := c.resolveUser(ctx, username) if err != nil { return nil, err } @@ -237,40 +271,10 @@ func (c *Client) GetLatestNPosts(ctx context.Context, actor string, limit int) ( if limit > 100 { limit = 100 } - if limit < 5 { - limit = 5 - } endpoint := fmt.Sprintf("%s/users/%s/tweets?max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", apiBase, url.PathEscape(userID), limit) - body, err := c.doGet(ctx, endpoint) - if err != nil { - return nil, err - } - - var tr tweetsResponse - if err := json.Unmarshal(body, &tr); err != nil { - return nil, fmt.Errorf("failed to unmarshal tweets response: %w", err) - } - if len(tr.Errors) > 0 { - return nil, fmt.Errorf("twitter error: %s: %s", tr.Errors[0].Title, tr.Errors[0].Detail) - } - - posts := make([]*Post, 0, len(tr.Data)) - for _, t := range tr.Data { - created := parseTwitterTime(t.CreatedAt) - posts = append(posts, &Post{ - ID: t.ID, - Author: userID, - Content: t.Text, - CreatedAt: created, - Likes: t.PublicMetrics.LikeCount, - Reposts: t.PublicMetrics.RetweetCount, - Quotes: t.PublicMetrics.QuoteCount, - Replies: t.PublicMetrics.ReplyCount, - }) - } - return posts, nil + return c.fetchTweets(ctx, handle, endpoint) } func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, end time.Time, limit int) ([]*Post, error) { @@ -279,19 +283,22 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e } username := strings.TrimPrefix(actor, "@") - userID, err := c.resolveUser(ctx, username) + userID, handle, err := c.resolveUser(ctx, username) if err != nil { return nil, err } if limit > 100 { limit = 100 } - if limit < 5 { - limit = 5 - } endpoint := fmt.Sprintf("%s/users/%s/tweets?start_time=%s&end_time=%s&max_results=%d&tweet.fields=created_at,public_metrics&exclude=retweets,replies", apiBase, url.PathEscape(userID), start.Format(time.RFC3339), end.Format(time.RFC3339), limit) + return c.fetchTweets(ctx, handle, endpoint) +} + +// fetchTweets performs the request and converts the tweet payload into Posts, +// skipping entries without an ID so downstream dedupe keys are never empty. +func (c *Client) fetchTweets(ctx context.Context, username, endpoint string) ([]*Post, error) { body, err := c.doGet(ctx, endpoint) if err != nil { return nil, err @@ -307,12 +314,14 @@ func (c *Client) GetPostsInTimeframe(ctx context.Context, actor string, start, e posts := make([]*Post, 0, len(tr.Data)) for _, t := range tr.Data { - created := parseTwitterTime(t.CreatedAt) + if t.ID == "" { + continue + } posts = append(posts, &Post{ ID: t.ID, - Author: userID, + Author: username, Content: t.Text, - CreatedAt: created, + CreatedAt: parseTwitterTime(t.CreatedAt), Likes: t.PublicMetrics.LikeCount, Reposts: t.PublicMetrics.RetweetCount, Quotes: t.PublicMetrics.QuoteCount, @@ -343,16 +352,15 @@ func (c *Client) GetPost(ctx context.Context, id string) (*Post, error) { if tr.Data.ID == "" { return nil, fmt.Errorf("tweet %q not found", id) } - author := id + author := "unknown" if len(tr.Includes.Users) > 0 { author = tr.Includes.Users[0].Username } - created, _ := time.Parse(time.RFC3339, tr.Data.CreatedAt) return &Post{ ID: tr.Data.ID, Author: author, Content: tr.Data.Text, - CreatedAt: created, + CreatedAt: parseTwitterTime(tr.Data.CreatedAt), Likes: tr.Data.PublicMetrics.LikeCount, Reposts: tr.Data.PublicMetrics.RetweetCount, Quotes: tr.Data.PublicMetrics.QuoteCount, diff --git a/internal/twitter/client_test.go b/internal/twitter/client_test.go index ebb687d..87a6024 100644 --- a/internal/twitter/client_test.go +++ b/internal/twitter/client_test.go @@ -77,7 +77,7 @@ func TestGetLatestNPosts(t *testing.T) { if len(posts) != 1 { t.Fatalf("expected 1 post, got %d", len(posts)) } - if posts[0].Content != "hello world" || posts[0].Author != "12345" { + if posts[0].Content != "hello world" || posts[0].Author != "testuser" { t.Fatalf("unexpected post: %+v", posts[0]) } if !strings.Contains(tweetsURL, "max_results=10") { @@ -85,15 +85,15 @@ func TestGetLatestNPosts(t *testing.T) { } } -func TestGetLatestNPostsLimitClamp(t *testing.T) { +func TestGetLatestNPostsLimitNoLowerFloor(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) }) mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { - if got := r.URL.Query().Get("max_results"); got != "5" { - t.Fatalf("expected clamped max_results=5 for limit=3, got %q", got) + if got := r.URL.Query().Get("max_results"); got != "3" { + t.Fatalf("expected max_results=3 for limit=3, got %q", got) } w.Header().Set("Content-Type", "application/json") _, _ = io.WriteString(w, `{"data":[]}`) @@ -356,3 +356,90 @@ func TestContextCancellationDuringRetry(t *testing.T) { t.Fatal("test timed out waiting for goroutine to abort") } } + +func TestResolveUserEmptyData(t *testing.T) { + // 200 with an empty data object (banned/suspended or privacy-restricted + // account) must error rather than build a bogus /users//tweets request. + var tweetsCalled bool + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{}}`) + }) + mux.HandleFunc("/2/users//tweets", func(w http.ResponseWriter, r *http.Request) { + tweetsCalled = true + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + _, err := c.GetLatestNPosts(context.Background(), "@ghost", 10) + if err == nil { + t.Fatal("expected error for empty user data, got nil") + } + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected 'not found' error, got %v", err) + } + if tweetsCalled { + t.Fatal("tweets endpoint should not be called when user lookup returns empty data") + } +} + +func TestGetLatestNPostsSkipsEmptyID(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) + }) + mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"","text":"bad"},{"id":"2","text":"good","created_at":"2024-01-01T00:00:00Z"}]}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + posts, err := c.GetLatestNPosts(context.Background(), "u", 10) + if err != nil { + t.Fatalf("error: %v", err) + } + if len(posts) != 1 { + t.Fatalf("expected 1 post (empty-ID tweet skipped), got %d", len(posts)) + } + if posts[0].ID != "2" || posts[0].Content != "good" { + t.Fatalf("unexpected post: %+v", posts[0]) + } +} + +func TestServerErrorRetried(t *testing.T) { + callCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/2/users/by/username/", func(w http.ResponseWriter, r *http.Request) { + callCount++ + if callCount == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `{"title":"Internal Server Error","detail":"transient"}`) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":{"id":"1","username":"u"}}`) + }) + mux.HandleFunc("/2/users/1/tweets", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"data":[{"id":"1","text":"ok","created_at":"2024-01-01T00:00:00Z"}]}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(server.URL) + posts, err := c.GetLatestNPosts(context.Background(), "u", 10) + if err != nil { + t.Fatalf("GetLatestNPosts error after 5xx retry: %v", err) + } + if callCount != 2 { + t.Fatalf("expected 2 calls (1 failure + 1 retry), got %d", callCount) + } + if len(posts) != 1 || posts[0].Content != "ok" { + t.Fatalf("unexpected posts after retry: %+v", posts) + } +} From 98a6bed4660f4ab9e2c0687d18de5fbbad0639c1 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Thu, 10 Sep 2026 09:54:56 +0000 Subject: [PATCH 21/30] fix: eliminate Bluesky session-refresh data race read/write race on xrpc.Client.Auth: Do() reads client.Auth unlocked while refreshSession mutated AccessJwt/RefreshJwt in place (only refreshMu serialized refreshes, not concurrent request readers). - add authMu RWMutex; every request now uses a clientSnapshot() copy that pins the current AuthInfo under the read lock, so requests never touch Auth directly - refreshSession replaces client.Auth with a fresh immutable AuthInfo under the exclusive lock instead of mutating the old one in place - remove the now-obsolete 'benign race' comment; lock order refreshMu -> authMu --- internal/bluesky/client.go | 47 +++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/internal/bluesky/client.go b/internal/bluesky/client.go index cdd44b8..edb6359 100644 --- a/internal/bluesky/client.go +++ b/internal/bluesky/client.go @@ -35,6 +35,7 @@ const ( type BlueskyClient struct { client *xrpc.Client + authMu sync.RWMutex refreshMu sync.Mutex cacheMu sync.Mutex didCache map[string]didCacheEntry @@ -80,12 +81,6 @@ func NewBlueskyClient(ctx context.Context, handle, password, host string) (*Blue // rotated, so two concurrent refreshes would race and one should fail for no // reason. After acquiring the lock the error is re-checked, because a previous // goroutine may already have refreshed the session while this one waited. -// -// Note: xrpc.Client.Auth is written here under the lock while ordinary -// concurrent requests read it outside the lock. The residual read/write race on -// the JWT string is benign (single pointer-sized write; torn reads cannot -// occur on 64-bit platforms) and would require guarding every request to -// eliminate, which is not worth the cost. func (c *BlueskyClient) withRefresh(ctx context.Context, fn func() error) error { if err := fn(); err == nil { return nil @@ -112,13 +107,33 @@ func (c *BlueskyClient) withRefresh(ctx context.Context, fn func() error) error return fn() } +// clientSnapshot returns a copy of the xrpc client that requests must use. It +// pins the current, immutable AuthInfo under the read lock: refreshSession +// replaces the Auth pointer under the exclusive lock rather than mutating the +// old value in place, so every goroutine sharing a pinned AuthInfo reads a +// complete, consistent token. xrpc.Client.Do reads client.Auth without any +// locking of its own, so no request may touch c.client.Auth directly. +func (c *BlueskyClient) clientSnapshot() *xrpc.Client { + c.authMu.RLock() + defer c.authMu.RUnlock() + cp := *c.client + return &cp +} + func (c *BlueskyClient) refreshSession(ctx context.Context) error { - refreshed, err := atproto.ServerRefreshSession(ctx, c.client) + refreshed, err := atproto.ServerRefreshSession(ctx, c.clientSnapshot()) if err != nil { return fmt.Errorf("failed to refresh bluesky session: %w", err) } - c.client.Auth.AccessJwt = refreshed.AccessJwt - c.client.Auth.RefreshJwt = refreshed.RefreshJwt + + c.authMu.Lock() + defer c.authMu.Unlock() + c.client.Auth = &xrpc.AuthInfo{ + AccessJwt: refreshed.AccessJwt, + RefreshJwt: refreshed.RefreshJwt, + Handle: c.client.Auth.Handle, + Did: c.client.Auth.Did, + } return nil } @@ -160,7 +175,7 @@ func (c *BlueskyClient) GetPost(ctx context.Context, actor, collection, recordKe } log.DebugWithContext(ctx, "Fetching post", "URI", uri) - resp, err := bsky.FeedGetPosts(ctx, c.client, []string{uri}) + resp, err := bsky.FeedGetPosts(ctx, c.clientSnapshot(), []string{uri}) if err != nil { return fmt.Errorf("failed to get post: %w", err) } @@ -188,7 +203,7 @@ func (c *BlueskyClient) GetLatestNPosts(ctx context.Context, actor string, limit if err != nil { return err } - resp, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), "", "posts", false, int64(pageSize)) + resp, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), "", "posts", false, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get latest posts: %w", err) } @@ -213,7 +228,7 @@ func (c *BlueskyClient) GetPostsInTimeframe(ctx context.Context, actor string, s // than start terminates the scan; maxAuthorFeedPages is a belt-and- // braces guard against a misbehaving server. for pages := 0; pages < maxAuthorFeedPages; pages++ { - feed, err := bsky.FeedGetAuthorFeed(ctx, c.client, repo.String(), cursor, "posts", false, 50) + feed, err := bsky.FeedGetAuthorFeed(ctx, c.clientSnapshot(), repo.String(), cursor, "posts", false, 50) if err != nil { return err } @@ -275,7 +290,7 @@ func (c *BlueskyClient) GetFeed(ctx context.Context, feedURI string, limit int) var result *bsky.FeedGetFeed_Output err = c.withRefresh(ctx, func() error { - resp, err := bsky.FeedGetFeed(ctx, c.client, "", uri, int64(pageSize)) + resp, err := bsky.FeedGetFeed(ctx, c.clientSnapshot(), "", uri, int64(pageSize)) if err != nil { return fmt.Errorf("failed to get feed: %w", err) } @@ -300,7 +315,7 @@ func (c *BlueskyClient) resolveUser(ctx context.Context, user string) (syntax.DI } c.cacheMu.Unlock() - resp, err := atproto.IdentityResolveHandle(ctx, c.client, user) + resp, err := atproto.IdentityResolveHandle(ctx, c.clientSnapshot(), user) if err != nil { return "", fmt.Errorf("failed to resolve handle to DID: %w", err) } @@ -324,7 +339,7 @@ func (c *BlueskyClient) GetActor(ctx context.Context, author string) (*actor.Pro err := c.withRefresh(ctx, func() error { params := map[string]any{"actor": author} var prof bskyActorProfile - if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { + if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &prof); err != nil { return fmt.Errorf("failed to get actor profile: %w", err) } result = &actor.Profile{ @@ -348,7 +363,7 @@ func (c *BlueskyClient) SearchActors(ctx context.Context, query string, limit in err := c.withRefresh(ctx, func() error { params := map[string]any{"q": query, "limit": limit} var out bskyActorSearch - if err := c.client.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { + if err := c.clientSnapshot().Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { return fmt.Errorf("failed to search actors: %w", err) } results := make([]*actor.Profile, 0, len(out.Actors)) From c96447e3e0dc4acd0c98e05013a1c6fed001861c Mon Sep 17 00:00:00 2001 From: Nacorid Date: Thu, 10 Sep 2026 09:59:44 +0000 Subject: [PATCH 22/30] fix(activitypub): harden client, add outbox fallback and throttled GTS warming Extract ActivityPub outbox walk into internal/activitypub/outbox.go with webfinger/actor resolution, Create/Note filtering and IRI dereferencing. Fix searchStatus returning wrong post, nil-account panic in SearchActors, @local handle resolution, empty Author via noteToInternalPost, protocol- relative URL handling, pagination loops, limit clamping (200) and cacheErr masking. Add per-host token-bucket throttle (1/s, cross-host concurrency) in throttle.go and background GTS warming via search?resolve for outbox- fetched posts without following remotes. Align AGENTS.md. Co-Authored-By: internal-model --- AGENTS.md | 6 +- internal/activitypub/client.go | 396 +++++++++++++++++----- internal/activitypub/client_test.go | 361 ++++++++++++++++++-- internal/activitypub/outbox.go | 501 ++++++++++++++++++++++++++++ internal/activitypub/outbox_test.go | 224 +++++++++++++ internal/activitypub/throttle.go | 58 ++++ internal/activitypub/types.go | 7 + 7 files changed, 1441 insertions(+), 112 deletions(-) create mode 100644 internal/activitypub/outbox.go create mode 100644 internal/activitypub/outbox_test.go create mode 100644 internal/activitypub/throttle.go diff --git a/AGENTS.md b/AGENTS.md index 41c77f9..317bf8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,8 @@ internal/server/ MCP handlers + x402 + analytics (~1800 lines in serv types.go request/response types, platform enum auth.go / webhook.go / cache*.go / *Middleware.go / networkInfo.go *_test.go split suites: mcp_test.go, analytics_test.go, validation_test.go, etc. -internal/{bluesky,nostr,activitypub,twitter,ollama,config,utils,actor,x402} +internal/activitypub/ AP client; throttle.go per-host token bucket (1/s) for background GTS warming +internal/{bluesky,nostr,twitter,ollama,config,utils,actor,x402} sql/schema/ 001..007 goose migrations sql/queries/ sqlc query files → internal/store/ config/models.env single source of truth for EMBEDDING_MODEL / STYLE_MODEL @@ -92,4 +93,5 @@ Loaded in order: `config/models.env` → `.env` (gitignored, via `godotenv`) → - Platform inferred from author handle in `determinePlatform`/`fetch*` — not caller-supplied. - Bluesky auth refresh via `isAuthError`/`withRefresh` — 401/expired token retried once. - Twitter client clamps `limit` (`>100` → 100), retries 429/5xx/transport errors with `x-rate-limit-*`/`Retry-After` backoff (bounded by `maxRateLimitWait`), and has a 60s per-request timeout. -- Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). + - ActivityPub client: GTS Mastodon API only serves locally-federated statuses. Remote actors are resolved via `/accounts/search?resolve=true` (not `/accounts/lookup`, which skips WebFinger). When the local cache can't satisfy full `limit` (clamped to 200), `GetLatestNPosts`/`GetPostsInTimeframe` fall back to walking the remote outbox directly (webfinger → actor → `OrderedCollection`/`OrderedCollectionPage`), filtering `Create`/`Note`-like activities and dereferencing IRI-only objects; `GetPost` derefs status URLs over AP when GTS `/search` can't resolve them. The `read:search` scope is needed for status-URL resolution. Outbox-fetched posts are warmed into GTS in background via `GET /api/v2/search?resolve=true&type=statuses&limit=1` per post, throttled at 1 req/s per remote host via per-host token bucket (`internal/activitypub/throttle.go`) with cross-host concurrency, no follows. + - Coverage ranges for analytics are interval-merged; disjoint gaps must remain uncovered (see `coverage_test.go`). diff --git a/internal/activitypub/client.go b/internal/activitypub/client.go index e0d2d9f..c3684dd 100644 --- a/internal/activitypub/client.go +++ b/internal/activitypub/client.go @@ -8,29 +8,23 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/nacorid/naco-api/internal/actor" ) -// maxMastodonPageSize is the upper bound the Mastodon API accepts for ?limit=. -// Values above 40 are silently clamped server-side. const maxMastodonPageSize = 40 - -// maxTimeframePages caps the total number of paginated requests in -// GetPostsInTimeframe to prevent runaway loops against misbehaving servers. -// At 50 posts/page this allows fetching up to ~5 000 statuses. const maxTimeframePages = 100 - -// maxResponseBody is the upper bound for a single Mastodon API response body. -// A page of 40 statuses with long text is typically < 100 KB; 1 MB is generous -// while still preventing multi-GB abuse. -const maxResponseBody = 1 << 20 // 1 MB +const maxResponseBody = 1 << 20 +const maxLimit = 200 type APClient struct { client *http.Client host string bearerToken string + outbox *outboxClient + limiter *hostLimiter } func NewAPClient(host, apikey string) (*APClient, error) { @@ -42,14 +36,11 @@ func NewAPClient(host, apikey string) (*APClient, error) { client: &http.Client{Timeout: 30 * time.Second}, host: strings.TrimRight(host, "/"), bearerToken: "Bearer " + apikey, + outbox: newOutboxClient(), + limiter: newHostLimiter(), }, nil } -// get performs an authenticated GET and JSON-decodes the response into target. -// -// TODO: mirror the Bluesky withRefresh retry pattern for transient 5xx errors. -// Both Bluesky and ActivityPub are contacted on the local network; a single 503 -// from a busy server should trigger one automatic retry with a short backoff. func (ap *APClient) get(ctx context.Context, endpoint string, target any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { @@ -75,150 +66,310 @@ func (ap *APClient) get(ctx context.Context, endpoint string, target any) error return nil } -func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { - endpoint := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, url.PathEscape(postID)) - var internal internalPost - if err := ap.get(ctx, endpoint, &internal); err != nil { - return nil, fmt.Errorf("failed to get post: %w", err) +func (ap *APClient) accountSearch(ctx context.Context, query string) (*Account, error) { + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=1", ap.host, url.QueryEscape(query)) + var accounts []*Account + if err := ap.get(ctx, endpoint, &accounts); err != nil { + return nil, fmt.Errorf("failed to search accounts: %w", err) } - return internal.toPost(), nil + for _, acc := range accounts { + if acc != nil { + return acc, nil + } + } + return nil, nil +} + +func (ap *APClient) resolveAccount(ctx context.Context, actorHandle string) (*Account, error) { + handle := strings.TrimPrefix(actorHandle, "@") + + acc, err := ap.accountSearch(ctx, handle) + if err != nil { + return nil, err + } + if acc != nil { + return acc, nil + } + + if !strings.Contains(handle, "@") { + var byID Account + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(handle)) + if err := ap.get(ctx, endpoint, &byID); err != nil { + return nil, fmt.Errorf("account %q not found by search or id: %w", actorHandle, err) + } + if byID.ID == "" { + return nil, fmt.Errorf("account %q not found", actorHandle) + } + return &byID, nil + } + return nil, fmt.Errorf("account %q not found", actorHandle) +} + +func (ap *APClient) GetPost(ctx context.Context, postID string) (*Post, error) { + status, err := ap.resolveStatus(ctx, postID) + if err != nil { + return nil, err + } + return status.toPost(), nil +} + +func (ap *APClient) resolveStatus(ctx context.Context, postID string) (*internalPost, error) { + if strings.HasPrefix(postID, "http://") || strings.HasPrefix(postID, "https://") || strings.HasPrefix(postID, "//") { + if status, err := ap.searchStatus(ctx, postID); err == nil && status != nil { + return status, nil + } + note, derr := ap.outbox.resolveNote(ctx, postID) + if derr != nil { + return nil, fmt.Errorf("status %q not resolvable: %w", postID, derr) + } + author := extractHandleFromURL(postID) + return noteToInternalPost(note, author), nil + } + + endpoint := fmt.Sprintf("%s/api/v1/statuses/%s", ap.host, url.PathEscape(postID)) + var status internalPost + if err := ap.get(ctx, endpoint, &status); err != nil { + return nil, fmt.Errorf("failed to get post: %w", err) + } + return &status, nil +} + +func (ap *APClient) searchStatus(ctx context.Context, postID string) (*internalPost, error) { + endpoint := fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(postID)) + var res searchResult + if err := ap.get(ctx, endpoint, &res); err != nil { + return nil, fmt.Errorf("failed to resolve status %q: %w", postID, err) + } + for _, s := range res.Statuses { + if s != nil && (s.URL == postID || s.URI == postID) { + return s, nil + } + } + return nil, nil } -// GetLatestNPosts returns up to limit posts for actorID, paginating through -// the Mastodon timeline API. Each page returns at most 40 statuses (the -// Mastodon API caps ?limit= at 40; higher values are silently clamped). func (ap *APClient) GetLatestNPosts(ctx context.Context, actorID string, limit int) ([]*Post, error) { if limit <= 0 { return nil, nil } + if limit > maxLimit { + limit = maxLimit + } + acc, err := ap.resolveAccount(ctx, actorID) + if err != nil { + return nil, err + } + + cached, cacheErr := ap.cacheLatestPosts(ctx, acc, limit) + if cacheErr != nil { + return nil, cacheErr + } + if len(cached) >= limit { + return cachePostsToPosts(cached), nil + } + + if !isRemoteAccount(acc) || acc.URL == "" { + return cachePostsToPosts(cached), nil + } + + outboxPosts, apErr := ap.outbox.getLatest(ctx, acc.URL, acc.Acct, limit-len(cached)) + if apErr != nil { + return nil, fmt.Errorf("outbox fallback failed: %w", apErr) + } + merged := mergePostsBySource(cached, outboxPosts) + if len(outboxPosts) > 0 { + urls := make([]string, 0, len(outboxPosts)) + for _, p := range outboxPosts { + if p != nil && p.ID != "" { + urls = append(urls, p.ID) + } + } + go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) + } + return merged, nil +} + +func (ap *APClient) cacheLatestPosts(ctx context.Context, acc *Account, limit int) ([]*internalPost, error) { var ( - posts []*Post - maxID string + internal []*internalPost + maxID string ) - for len(posts) < limit { + for len(internal) < limit { if ctx.Err() != nil { return nil, ctx.Err() } - pageSize := limit - len(posts) + pageSize := limit - len(internal) if pageSize > maxMastodonPageSize { pageSize = maxMastodonPageSize } - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(actorID), pageSize) + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), pageSize) if maxID != "" { endpoint += "&max_id=" + url.QueryEscape(maxID) } - var internal []*internalPost - if err := ap.get(ctx, endpoint, &internal); err != nil { - return nil, fmt.Errorf("failed to get latest posts: %w", err) + var page []*internalPost + if err := ap.get(ctx, endpoint, &page); err != nil { + return nil, err } - if len(internal) == 0 { + if len(page) == 0 { break } - for _, p := range internal { - posts = append(posts, p.toPost()) - if len(posts) >= limit { - break - } - } - - newMaxID := internal[len(internal)-1].ID - if newMaxID == maxID { + newMaxID := page[len(page)-1].ID + if newMaxID == "" { break } + if maxID != "" && newMaxID == maxID { + break + } + + internal = append(internal, page...) maxID = newMaxID } - - return posts, nil + if len(internal) > limit { + internal = internal[:limit] + } + return internal, nil } -// GetPostsInTimeframe returns posts for actorID created between start and end -// (inclusive), up to limit. It paginates through the Mastodon timeline in -// reverse-chronological order, stopping early when posts fall outside the -// window. At most maxTimeframePages pages are fetched to prevent runaway loops. func (ap *APClient) GetPostsInTimeframe(ctx context.Context, actorID string, start, end time.Time, limit int) ([]*Post, error) { - var posts []*Post - var maxID string + if limit <= 0 { + return nil, nil + } + if limit > maxLimit { + limit = maxLimit + } + + acc, err := ap.resolveAccount(ctx, actorID) + if err != nil { + return nil, err + } + + cached, cacheErr := ap.cachePostsInTimeframe(ctx, acc, start, end, limit) + if cacheErr != nil { + return nil, cacheErr + } + if len(cached) >= limit { + return cachePostsToPosts(cached), nil + } + + if !isRemoteAccount(acc) || acc.URL == "" { + return cachePostsToPosts(cached), nil + } + + outboxPosts, apErr := ap.outbox.getInTimeframe(ctx, acc.URL, acc.Acct, start, end, limit-len(cached)) + if apErr != nil { + return nil, fmt.Errorf("outbox fallback failed: %w", apErr) + } + merged := mergePostsBySource(cached, outboxPosts) + if len(outboxPosts) > 0 { + urls := make([]string, 0, len(outboxPosts)) + for _, p := range outboxPosts { + if p != nil && p.ID != "" { + urls = append(urls, p.ID) + } + } + go ap.warmGTSAsync(context.WithoutCancel(ctx), urls) + } + return merged, nil +} + +func (ap *APClient) cachePostsInTimeframe(ctx context.Context, acc *Account, start, end time.Time, limit int) ([]*internalPost, error) { + var ( + internal []*internalPost + maxID string + ) for pages := 0; pages < maxTimeframePages; pages++ { if ctx.Err() != nil { return nil, ctx.Err() } - if len(posts) >= limit { + if len(internal) >= limit { break } - endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=50", ap.host, url.PathEscape(actorID)) + endpoint := fmt.Sprintf("%s/api/v1/accounts/%s/statuses?limit=%d", ap.host, url.PathEscape(acc.ID), maxMastodonPageSize) if maxID != "" { endpoint += "&max_id=" + url.QueryEscape(maxID) } - var internal []*internalPost - if err := ap.get(ctx, endpoint, &internal); err != nil { - return nil, fmt.Errorf("failed to get posts: %w", err) + var page []*internalPost + if err := ap.get(ctx, endpoint, &page); err != nil { + return nil, err } - - if len(internal) == 0 { + if len(page) == 0 { break } - for _, i := range internal { + newMaxID := page[len(page)-1].ID + if newMaxID == "" { + break + } + if maxID != "" && newMaxID == maxID { + break + } + + for _, i := range page { if i.CreatedAt.After(end) { continue } if i.CreatedAt.Before(start) { - return posts, nil + return internal, nil } - posts = append(posts, i.toPost()) - if len(posts) >= limit { - return posts, nil + internal = append(internal, i) + if len(internal) >= limit { + return internal, nil } } - - newMaxID := internal[len(internal)-1].ID - if newMaxID == maxID { - break - } maxID = newMaxID } - return posts, nil + return internal, nil } -// GetActor resolves a Mastodon handle to a profile. It first tries the -// /accounts/lookup endpoint (handle-based), falling back to /accounts/{id} -// if the handle is a numeric ID the server recognises directly. -func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { - var acc Account - endpoint := fmt.Sprintf("%s/api/v1/accounts/lookup?acct=%s", ap.host, url.QueryEscape(actorHandle)) +func isRemoteAccount(acc *Account) bool { + return acc != nil && strings.Contains(acc.Acct, "@") +} - if err := ap.get(ctx, endpoint, &acc); err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - fallback := fmt.Sprintf("%s/api/v1/accounts/%s", ap.host, url.PathEscape(actorHandle)) - if fallbackErr := ap.get(ctx, fallback, &acc); fallbackErr != nil { - return nil, fmt.Errorf("actor lookup %q failed: %w; fallback by-id also failed: %v", actorHandle, err, fallbackErr) - } +func cachePostsToPosts(cached []*internalPost) []*Post { + if len(cached) == 0 { + return nil } - return ap.toProfile(&acc), nil + posts := make([]*Post, 0, len(cached)) + for _, c := range cached { + if c == nil { + continue + } + posts = append(posts, c.toPost()) + } + return posts +} + +func (ap *APClient) GetActor(ctx context.Context, actorHandle string) (*actor.Profile, error) { + acc, err := ap.resolveAccount(ctx, actorHandle) + if err != nil { + return nil, err + } + return ap.toProfile(acc), nil } func (ap *APClient) SearchActors(ctx context.Context, query string, limit int) ([]*actor.Profile, error) { - endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&limit=%d", ap.host, url.QueryEscape(query), limit) + endpoint := fmt.Sprintf("%s/api/v1/accounts/search?q=%s&resolve=true&limit=%d", ap.host, url.QueryEscape(query), limit) var accounts []*Account if err := ap.get(ctx, endpoint, &accounts); err != nil { return nil, fmt.Errorf("failed to search actors: %w", err) } - results := make([]*actor.Profile, len(accounts)) - for i, acc := range accounts { - results[i] = ap.toProfile(acc) + results := make([]*actor.Profile, 0, len(accounts)) + for _, acc := range accounts { + if acc == nil { + continue + } + results = append(results, ap.toProfile(acc)) } return results, nil } @@ -236,3 +387,72 @@ func (ap *APClient) toProfile(acc *Account) *actor.Profile { URL: acc.URL, } } + +func extractHandleFromURL(postURL string) string { + u, err := url.Parse(postURL) + if err != nil { + return "" + } + parts := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") + for _, p := range parts { + if strings.HasPrefix(p, "@") { + return strings.TrimPrefix(p, "@") + } + } + return "" +} + +func (ap *APClient) warmGTSAsync(ctx context.Context, urls []string) { + if len(urls) == 0 { + return + } + if ap.limiter == nil { + ap.limiter = newHostLimiter() + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + seen := make(map[string]struct{}, len(urls)) + uniq := make([]string, 0, len(urls)) + for _, u := range urls { + if _, ok := seen[u]; ok { + continue + } + seen[u] = struct{}{} + uniq = append(uniq, u) + } + if len(uniq) > 50 { + uniq = uniq[:50] + } + + byHost := make(map[string][]string) + for _, u := range uniq { + parsed, err := url.Parse(u) + if err != nil || parsed.Host == "" { + continue + } + byHost[parsed.Host] = append(byHost[parsed.Host], u) + } + if len(byHost) == 0 { + return + } + + var wg sync.WaitGroup + for host, us := range byHost { + wg.Add(1) + go func(host string, us []string) { + defer wg.Done() + for _, u := range us { + if err := ap.limiter.Wait(ctx, host); err != nil { + return + } + if ctx.Err() != nil { + return + } + var res searchResult + _ = ap.get(ctx, fmt.Sprintf("%s/api/v2/search?q=%s&resolve=true&type=statuses&limit=1", ap.host, url.QueryEscape(u)), &res) + } + }(host, us) + } + wg.Wait() +} diff --git a/internal/activitypub/client_test.go b/internal/activitypub/client_test.go index 5e936d0..04fbffd 100644 --- a/internal/activitypub/client_test.go +++ b/internal/activitypub/client_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -45,7 +46,10 @@ func TestNewAPClientEmpty(t *testing.T) { func TestGetLatestNPosts(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer tok" { w.WriteHeader(http.StatusUnauthorized) return @@ -73,11 +77,11 @@ func TestGetLatestNPosts(t *testing.T) { } func TestGetLatestNPostsPaginates(t *testing.T) { - // Page 1 returns 40 posts; page 2 returns 10 more. The client must request - // both pages to satisfy a limit of 50 (page size is capped at 40 per the - // Mastodon API) and then stop. mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { switch r.URL.Query().Get("max_id") { case "40": ids := make([]map[string]any, 0, 10) @@ -128,6 +132,32 @@ func TestGetLatestNPostsZeroLimit(t *testing.T) { } } +func TestGetLatestNPostsLimitClamp(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + limit := r.URL.Query().Get("limit") + l, _ := strconv.Atoi(limit) + if l > maxLimit { + t.Fatalf("expected limit <= %d, got %s", maxLimit, limit) + } + _, _ = fmt.Fprint(w, `[{"id":"1","content":"hello","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + posts, err := ap.GetLatestNPosts(context.Background(), "alice", 500) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) == 0 { + t.Fatal("expected at least one post") + } +} + func TestGetPost(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/statuses/42", func(w http.ResponseWriter, r *http.Request) { @@ -146,34 +176,215 @@ func TestGetPost(t *testing.T) { } } -func TestGetLatestNPostsHTTPError(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) +func TestGetPostURLResolves(t *testing.T) { + url := "https://inst.social/@alice/statuses/42" + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("q") != url { + w.WriteHeader(http.StatusBadRequest) + return + } + if r.URL.Query().Get("type") != "statuses" || r.URL.Query().Get("resolve") != "true" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"42","uri":"`+url+`","url":"`+url+`","content":"resolved post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"bob@inst.social"}}]}`) + }) + server := httptest.NewServer(mux) defer server.Close() ap := newTestClient(server.URL) - if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { - t.Fatal("expected error on non-200 status") + post, err := ap.GetPost(context.Background(), url) + if err != nil { + t.Fatalf("GetPost error: %v", err) + } + if post.Content != "resolved post" || post.Author != "bob@inst.social" { + t.Fatalf("unexpected post: %+v", post) + } +} + +func TestGetPostURLNotFound(t *testing.T) { + derefTarget := httptest.NewServer(http.NotFoundHandler()) + defer derefTarget.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[]}`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + if _, err := ap.GetPost(context.Background(), derefTarget.URL+"/missed"); err == nil { + t.Fatal("expected error when search and dereference both return nothing") + } +} + +func TestGetPostURLSearchReturnsWrongPost(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"accounts":[],"statuses":[{"id":"wrong","uri":"https://other.com/1","content":"wrong post"}]}`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + if _, err := ap.GetPost(context.Background(), "https://inst.social/@alice/statuses/42"); err == nil { + t.Fatal("expected error when search returns no matching post") + } +} + +func TestGetPostURLDerefFallback(t *testing.T) { + remote, _ := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v2/search", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + post, err := ap.GetPost(context.Background(), remote.URL+"/notes/2") + if err != nil { + t.Fatalf("GetPost error: %v", err) + } + if post.Content != "deref note 2" { + t.Fatalf("expected dereferenced note, got %+v", post) + } +} + +func TestGetLatestNPostsRemoteFallsBackToOutbox(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached","uri":`+`"`+remoteBase+`/notes/9","created_at":"2024-03-10T00:00:00Z","account":{"acct":"alice@example.social"}}]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 5) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 5 { + t.Fatalf("expected 5 merged posts, got %d: %+v", len(posts), posts) + } + if posts[0].Content != "cached" { + t.Fatalf("expected newest cached first, got %q", posts[0].Content) + } + if posts[4].Content != "note 4" { + t.Fatalf("expected oldest outbox post last, got %q", posts[4].Content) + } +} + +func TestGetLatestNPostsRemoteEmptyCacheFallsBack(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + posts, err := ap.GetLatestNPosts(context.Background(), "@alice@example.social", 4) + if err != nil { + t.Fatalf("GetLatestNPosts error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected all 4 posts from outbox, got %d", len(posts)) + } + if posts[0].Content != "bare note 3" { + t.Fatalf("expected outbox post newest-first, got %q", posts[0].Content) + } +} + +func TestGetPostsInTimeframeRemoteFallsBackToOutbox(t *testing.T) { + remote, remoteBase := newRemoteServer() + defer remote.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `[{"id":"1","acct":"alice@example.social","url":%q}]`, remoteBase) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("max_id") != "" { + _, _ = fmt.Fprint(w, `[]`) + return + } + _, _ = fmt.Fprint(w, `[{"id":"c1","content":"cached in window","uri":`+`"`+remoteBase+`/notes/5","created_at":"2024-02-20T00:00:00Z","account":{"acct":"alice@example.social"}}]`) + }) + gts := httptest.NewServer(mux) + defer gts.Close() + + ap, err := NewAPClient(gts.URL, "tok") + if err != nil { + t.Fatal(err) + } + start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) + posts, err := ap.GetPostsInTimeframe(context.Background(), "@alice@example.social", start, end, 3) + if err != nil { + t.Fatalf("GetPostsInTimeframe error: %v", err) + } + if len(posts) != 3 { + t.Fatalf("expected 3 in-window posts, got %d: %+v", len(posts), posts) + } + if posts[0].Content != "created note 1" || posts[1].Content != "cached in window" || posts[2].Content != "deref note 2" { + t.Fatalf("unexpected merged order: %+v", posts) } } func TestGetPostsInTimeframe(t *testing.T) { - // Feed is reverse-chronological: newer than `end`, two in-window, then older - // than `start`. The client must skip the too-new post, return both in-window - // posts, and stop when it crosses the `start` boundary. mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/alice/statuses", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { feed := []map[string]any{} switch r.URL.Query().Get("max_id") { case "": - // Page 1: too-new post and one in-window post. feed = []map[string]any{ {"id": "5", "content": "too-new", "text": "too-new", "created_at": "2025-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "4", "content": "mid", "text": "mid", "created_at": "2024-06-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, } case "4": - // Page 2: in-window post then too-old post. feed = []map[string]any{ {"id": "3", "content": "early-mid", "text": "early-mid", "created_at": "2024-03-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, {"id": "2", "content": "too-old", "text": "too-old", "created_at": "2023-01-01T00:00:00Z", "account": map[string]any{"acct": "alice"}}, @@ -202,14 +413,31 @@ func TestGetPostsInTimeframe(t *testing.T) { } } +func TestGetLatestNPostsHTTPError(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/accounts/1/statuses", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + if _, err := ap.GetLatestNPosts(context.Background(), "alice", 10); err == nil { + t.Fatal("expected error on non-200 status from timeline") + } +} + func TestGetActor(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("acct") != "alice" { + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("q") != "alice" { w.WriteHeader(http.StatusBadRequest) return } - _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}`) + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hello there","avatar":"https://img/avatar.png","followers_count":10,"following_count":5,"statuses_count":3,"url":"https://inst.social/@alice"}]`) }) server := httptest.NewServer(mux) defer server.Close() @@ -235,8 +463,8 @@ func TestGetActor(t *testing.T) { func TestGetActorIDFallback(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/api/v1/accounts/lookup", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[]`) }) mux.HandleFunc("/api/v1/accounts/alice", func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprint(w, `{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}`) @@ -254,6 +482,29 @@ func TestGetActorIDFallback(t *testing.T) { } } +func TestGetActorAtLocalHandle(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if q != "@alice" && q != "alice" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice","display_name":"Alice","note":"hi","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + prof, err := ap.GetActor(context.Background(), "@alice") + if err != nil { + t.Fatalf("GetActor error: %v", err) + } + if prof.Handle != "alice" { + t.Fatalf("expected local handle, got %q", prof.Handle) + } +} + func TestSearchActors(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { @@ -261,6 +512,10 @@ func TestSearchActors(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } + if r.URL.Query().Get("resolve") != "true" { + w.WriteHeader(http.StatusBadRequest) + return + } _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice@inst.social","display_name":"Alice","note":"bio","avatar":"https://img/a.png","followers_count":1,"following_count":2,"statuses_count":3,"url":"https://x"}]`) }) server := httptest.NewServer(mux) @@ -278,3 +533,65 @@ func TestSearchActors(t *testing.T) { t.Fatalf("unexpected result: %+v", results[0]) } } + +func TestExtractHandleFromURL(t *testing.T) { + tests := []struct { + url string + expected string + }{ + {"https://inst.social/@alice/statuses/42", "alice"}, + {"http://example.com/@bob/posts/1", "bob"}, + {"https://no-at-here.com/notes/1", ""}, + {"https://host/@user/note", "user"}, + } + for _, tt := range tests { + got := extractHandleFromURL(tt.url) + if got != tt.expected { + t.Fatalf("extractHandleFromURL(%q) = %q, want %q", tt.url, got, tt.expected) + } + } +} + +func TestGetPostBareLocalHandle(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if q != "@alice" && q != "alice" { + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = fmt.Fprint(w, `[{"id":"1","acct":"alice"}]`) + }) + mux.HandleFunc("/api/v1/statuses/alice", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"id":"1","content":"local post","created_at":"2024-01-01T00:00:00Z","account":{"acct":"alice"}}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + _, err := ap.GetPost(context.Background(), "alice") + if err != nil { + t.Fatalf("GetPost with local handle error: %v", err) + } +} + +func TestSearchActorsNilEntries(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/accounts/search", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `[null,{"id":"1","acct":"alice@inst.social"}]`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + ap := newTestClient(server.URL) + results, err := ap.SearchActors(context.Background(), "alice", 10) + if err != nil { + t.Fatalf("SearchActors error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result after filtering nil, got %d", len(results)) + } + if results[0].Handle != "alice@inst.social" { + t.Fatalf("unexpected result: %+v", results[0]) + } +} diff --git a/internal/activitypub/outbox.go b/internal/activitypub/outbox.go new file mode 100644 index 0000000..f7dc81f --- /dev/null +++ b/internal/activitypub/outbox.go @@ -0,0 +1,501 @@ +package activitypub + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const outboxMaxPages = 100 + +var webfingerScheme = "https" + +type outboxClient struct { + client *http.Client +} + +type apStringOrArray []string + +func (s *apStringOrArray) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + *s = nil + return nil + } + if b[0] == '"' { + var v string + if err := json.Unmarshal(b, &v); err != nil { + return err + } + *s = []string{v} + return nil + } + var arr []string + if err := json.Unmarshal(b, &arr); err != nil { + return err + } + *s = arr + return nil +} + +type apIRI string + +func (i *apIRI) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + *i = "" + return nil + } + if b[0] == '"' { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + *i = apIRI(s) + return nil + } + var o struct { + ID string `json:"id"` + } + if err := json.Unmarshal(b, &o); err != nil { + return err + } + *i = apIRI(o.ID) + return nil +} + +type apRef struct { + IRI string + Inline json.RawMessage +} + +func (r *apRef) UnmarshalJSON(b []byte) error { + b = bytes.TrimSpace(b) + if len(b) == 0 || string(b) == "null" { + r.IRI = "" + r.Inline = nil + return nil + } + if b[0] == '"' { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + r.IRI = s + return nil + } + r.Inline = append([]byte(nil), b...) + return nil +} + +type apActor struct { + ID string `json:"id"` + Type string `json:"type"` + PreferredUsername string `json:"preferredUsername"` + URL string `json:"url"` + Outbox apIRI `json:"outbox"` + Inbox apIRI `json:"inbox"` +} + +type apActivity struct { + ID string `json:"id"` + Type string `json:"type"` + Published string `json:"published"` + Content string `json:"content"` + URL apStringOrArray `json:"url"` + Object json.RawMessage `json:"object"` +} + +type apCollection struct { + Type string `json:"type"` + First apRef `json:"first"` +} + +type apPage struct { + Type string `json:"type"` + OrderedItems json.RawMessage `json:"orderedItems"` + Next apRef `json:"next"` +} + +type apNote struct { + ID string `json:"id"` + Type string `json:"type"` + Published string `json:"published"` + Content string `json:"content"` + URL apStringOrArray `json:"url"` +} + +type webfingerResponse struct { + Subject string `json:"subject"` + Links []struct { + Rel string `json:"rel"` + Type string `json:"type"` + Href string `json:"href"` + } `json:"links"` +} + +func newOutboxClient() *outboxClient { + return &outboxClient{client: &http.Client{Timeout: 30 * time.Second}} +} + +func (oc *outboxClient) getJSON(ctx context.Context, endpoint string, target any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Accept", `application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/json`) + + resp, err := oc.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("api error (status %d): %s", resp.StatusCode, string(errBody)) + } + + if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(target); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + return nil +} + +func (oc *outboxClient) webfinger(ctx context.Context, handle string) (string, error) { + full := strings.TrimPrefix(handle, "@") + at := strings.LastIndex(full, "@") + if at <= 0 || at == len(full)-1 { + return "", fmt.Errorf("invalid handle %q", handle) + } + domain := full[at+1:] + + endpoint := fmt.Sprintf("%s://%s/.well-known/webfinger?resource=%s", webfingerScheme, domain, url.QueryEscape("acct:"+full)) + var wf webfingerResponse + if err := oc.getJSON(ctx, endpoint, &wf); err != nil { + return "", fmt.Errorf("webfinger lookup for %q failed: %w", full, err) + } + + var htmlSelf string + for _, l := range wf.Links { + if l.Rel != "self" || l.Href == "" { + continue + } + if strings.Contains(l.Type, "activity+json") || strings.Contains(l.Type, "ld+json") { + return l.Href, nil + } + if htmlSelf == "" && strings.Contains(l.Type, "html") { + htmlSelf = l.Href + } + } + if htmlSelf != "" { + return htmlSelf, nil + } + return "", fmt.Errorf("webfinger for %q returned no self link", full) +} + +func (oc *outboxClient) getActor(ctx context.Context, actorURL string) (*apActor, error) { + var actor apActor + if err := oc.getJSON(ctx, actorURL, &actor); err != nil { + return nil, fmt.Errorf("failed to fetch actor %q: %w", actorURL, err) + } + if actor.ID == "" && actor.Outbox == "" { + return nil, fmt.Errorf("response from %q is not an actor document", actorURL) + } + return &actor, nil +} + +func (oc *outboxClient) resolveActor(ctx context.Context, actorURL, handle string) (*apActor, error) { + if actorURL == "" { + var err error + actorURL, err = oc.webfinger(ctx, handle) + if err != nil { + return nil, err + } + } + return oc.getActor(ctx, actorURL) +} + +func (oc *outboxClient) getLatest(ctx context.Context, actorURL, author string, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + return oc.walk(ctx, actorURL, author, limit, func(*Post) bool { return true }) +} + +func (oc *outboxClient) getInTimeframe(ctx context.Context, actorURL, author string, start, end time.Time, limit int) ([]*Post, error) { + if limit <= 0 { + return nil, nil + } + return oc.walk(ctx, actorURL, author, limit, func(p *Post) bool { + return !p.CreatedAt.Before(start) && !p.CreatedAt.After(end) + }) +} + +func (oc *outboxClient) walk(ctx context.Context, actorURL, author string, limit int, include func(*Post) bool) ([]*Post, error) { + actor, err := oc.resolveActor(ctx, actorURL, author) + if err != nil { + return nil, err + } + if actor.Outbox == "" { + return nil, fmt.Errorf("actor %q exposes no outbox", actorURL) + } + + var posts []*Post + seen := make(map[string]struct{}) + + cur, err := oc.firstPage(ctx, string(actor.Outbox)) + if err != nil { + return nil, err + } + + for pages := 0; pages < outboxMaxPages; pages++ { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if len(posts) >= limit { + break + } + + page, ok, err := oc.getPage(ctx, cur) + if err != nil { + return nil, err + } + if !ok { + break + } + + var items []apActivity + if len(page.OrderedItems) > 0 { + if err := json.Unmarshal(page.OrderedItems, &items); err != nil { + return nil, fmt.Errorf("failed to decode outbox page items: %w", err) + } + } + + for _, item := range items { + p, valid, err := oc.itemToPost(ctx, item, author) + if err != nil { + return nil, err + } + if !valid || p == nil { + continue + } + if _, dup := seen[p.ID]; dup { + continue + } + seen[p.ID] = struct{}{} + if include(p) { + posts = append(posts, p) + if len(posts) >= limit { + return posts, nil + } + } + } + + cur = page.Next + if cur.IRI == "" && len(cur.Inline) == 0 { + break + } + } + + return posts, nil +} + +func (oc *outboxClient) firstPage(ctx context.Context, outbox string) (apRef, error) { + var coll apCollection + if err := oc.getJSON(ctx, outbox, &coll); err != nil { + return apRef{}, fmt.Errorf("failed to fetch outbox %q: %w", outbox, err) + } + if coll.Type != "" && coll.Type != "OrderedCollection" { + return apRef{}, fmt.Errorf("outbox %q has unexpected type %q", outbox, coll.Type) + } + if coll.First.IRI == "" && len(coll.First.Inline) == 0 { + return apRef{}, fmt.Errorf("outbox %q exposes no pages", outbox) + } + return coll.First, nil +} + +func (oc *outboxClient) getPage(ctx context.Context, ref apRef) (apPage, bool, error) { + var page apPage + switch { + case ref.IRI != "": + if err := oc.getJSON(ctx, ref.IRI, &page); err != nil { + return apPage{}, false, fmt.Errorf("failed to fetch outbox page %q: %w", ref.IRI, err) + } + case len(ref.Inline) > 0: + if err := json.Unmarshal(ref.Inline, &page); err != nil { + return apPage{}, false, fmt.Errorf("failed to decode inline outbox page: %w", err) + } + default: + return apPage{}, false, nil + } + return page, true, nil +} + +func (oc *outboxClient) itemToPost(ctx context.Context, item apActivity, author string) (*Post, bool, error) { + var note *apNote + var err error + + switch item.Type { + case "Create", "Update": + note, err = oc.noteFromObject(ctx, item.Object) + case "Note", "Article", "Question", "Page", "": + note = &apNote{ + ID: item.ID, + Type: item.Type, + Published: item.Published, + Content: item.Content, + URL: item.URL, + } + default: + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if note == nil || !isPostNote(note.Type) { + return nil, false, nil + } + + id := note.ID + if id == "" && len(note.URL) > 0 { + id = note.URL[0] + } + if id == "" { + return nil, false, nil + } + + created, ok := parseActivityTime(note.Published) + if !ok || created.IsZero() { + return nil, false, nil + } + + return &Post{ + ID: id, + Author: author, + Content: note.Content, + CreatedAt: created, + }, true, nil +} + +func (oc *outboxClient) noteFromObject(ctx context.Context, raw json.RawMessage) (*apNote, error) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + if raw[0] == '"' { + var id string + if err := json.Unmarshal(raw, &id); err != nil { + return nil, err + } + return oc.fetchNote(ctx, id) + } + var n apNote + if err := json.Unmarshal(raw, &n); err != nil { + return nil, err + } + return &n, nil +} + +func (oc *outboxClient) fetchNote(ctx context.Context, iri string) (*apNote, error) { + var n apNote + if err := oc.getJSON(ctx, iri, &n); err != nil { + return nil, fmt.Errorf("failed to dereference object %q: %w", iri, err) + } + return &n, nil +} + +func (oc *outboxClient) resolveNote(ctx context.Context, url string) (*apNote, error) { + var act apActivity + if err := oc.getJSON(ctx, url, &act); err != nil { + return nil, fmt.Errorf("failed to dereference status %q: %w", url, err) + } + switch act.Type { + case "Create": + return oc.noteFromObject(ctx, act.Object) + case "Note", "Article", "Question", "Page", "": + return &apNote{ + ID: act.ID, + Type: act.Type, + Published: act.Published, + Content: act.Content, + URL: act.URL, + }, nil + default: + return nil, fmt.Errorf("document at %q is not a post (type %q)", url, act.Type) + } +} + +func isPostNote(t string) bool { + switch t { + case "", "Note", "Article", "Question", "Page": + return true + } + return false +} + +func parseActivityTime(s string) (time.Time, bool) { + if s == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339, time.RFC3339Nano, "2006-01-02T15:04:05"} { + if t, err := time.Parse(layout, s); err == nil { + return t, true + } + } + return time.Time{}, false +} + +func noteToInternalPost(n *apNote, author string) *internalPost { + p := &internalPost{ + ID: n.ID, + URI: n.ID, + Content: n.Content, + } + if len(n.URL) > 0 { + p.URL = n.URL[0] + } + if t, ok := parseActivityTime(n.Published); ok { + p.CreatedAt = t + } + if author != "" { + p.Account = Account{Acct: author} + } + return p +} + +func mergePostsBySource(cached []*internalPost, outbox []*Post) []*Post { + seen := make(map[string]struct{}, len(cached)) + res := make([]*Post, 0, len(cached)+len(outbox)) + for _, c := range cached { + key := c.URI + if key == "" { + key = c.ID + } + if key != "" { + seen[key] = struct{}{} + } + if c != nil { + res = append(res, c.toPost()) + } + } + for _, p := range outbox { + if p == nil || p.ID == "" { + continue + } + if _, dup := seen[p.ID]; dup { + continue + } + seen[p.ID] = struct{}{} + res = append(res, p) + } + sort.SliceStable(res, func(i, j int) bool { return res[i].CreatedAt.After(res[j].CreatedAt) }) + return res +} diff --git a/internal/activitypub/outbox_test.go b/internal/activitypub/outbox_test.go new file mode 100644 index 0000000..fb8c733 --- /dev/null +++ b/internal/activitypub/outbox_test.go @@ -0,0 +1,224 @@ +package activitypub + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// newRemoteServer serves a synthetic ActivityPub instance: a webfinger +// endpoint, an actor with an outbox collection, two outbox pages (with mixed +// activity types and an IRI-referenced object) and a dereferenceable note. +// Returns the server and the actor's canonical URL. +func newRemoteServer() (*httptest.Server, string) { + base := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body string + switch { + case strings.HasSuffix(r.URL.Path, "/.well-known/webfinger"): + body = `{"subject":"acct:alice@example.social","links":[` + + `{"rel":"self","type":"application/activity+json","href":"` + base + `/users/alice"},` + + `{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"` + base + `/@alice"}]}` + case r.URL.Path == "/users/alice": + body = `{"id":"` + base + `/users/alice","type":"Person","preferredUsername":"alice","url":"` + base + `/@alice","outbox":"` + base + `/users/alice/outbox"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "": + body = `{"type":"OrderedCollection","totalItems":5,"first":"` + base + `/users/alice/outbox?page=1"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "1": + body = `{"type":"OrderedCollectionPage","partOf":"` + base + `/users/alice/outbox","orderedItems":[` + + `{"type":"Create","published":"2024-03-01T00:00:00Z","object":{"type":"Note","id":"` + base + `/notes/1","published":"2024-03-01T00:00:00Z","content":"created note 1","url":"` + base + `/@alice/statuses/1"}},` + + `{"type":"Announce","object":"` + base + `/notes/9"},` + + `{"type":"Like","object":"` + base + `/notes/8"},` + + `{"type":"Create","object":"` + base + `/notes/2"},` + + `{"type":"Note","id":"` + base + `/notes/3","published":"2024-03-03T00:00:00Z","content":"bare note 3","url":["` + base + `/@alice/statuses/3"]}` + + `],"next":"` + base + `/users/alice/outbox?page=2"}` + case r.URL.Path == "/users/alice/outbox" && r.URL.Query().Get("page") == "2": + body = `{"type":"OrderedCollectionPage","orderedItems":[` + + `{"type":"Create","object":{"type":"Note","id":"` + base + `/notes/4","published":"2024-01-15T00:00:00Z","content":"note 4"}}` + + `]}` + case r.URL.Path == "/notes/2": + body = `{"type":"Note","id":"` + base + `/notes/2","published":"2024-02-01T00:00:00Z","content":"deref note 2","url":"` + base + `/@alice/statuses/2"}` + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/activity+json") + _, _ = fmt.Fprint(w, body) + })) + base = server.URL + return server, base + "/users/alice" +} + +func TestOutboxGetLatest(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), actorURL, "alice@example.social", 10) + if err != nil { + t.Fatalf("getLatest error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected 4 Create/Note posts (skipping Announce/Like), got %d: %+v", len(posts), posts) + } + want := []string{"created note 1", "deref note 2", "bare note 3", "note 4"} + for i, w := range want { + if posts[i].Content != w { + t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) + } + } + if posts[0].Author != "alice@example.social" { + t.Fatalf("Author = %q, want handle", posts[0].Author) + } +} + +func TestOutboxGetLatestLimit(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), actorURL, "h", 2) + if err != nil { + t.Fatalf("getLatest error: %v", err) + } + if len(posts) != 2 { + t.Fatalf("expected 2 posts, got %d", len(posts)) + } + want := []string{"created note 1", "deref note 2"} + for i, w := range want { + if posts[i].Content != w { + t.Fatalf("posts[%d].Content = %q, want %q", i, posts[i].Content, w) + } + } +} + +func TestOutboxGetInTimeframe(t *testing.T) { + server, actorURL := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + start := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC) + posts, err := oc.getInTimeframe(context.Background(), actorURL, "h", start, end, 10) + if err != nil { + t.Fatalf("getInTimeframe error: %v", err) + } + if len(posts) != 2 { + t.Fatalf("expected 2 in-window posts (note 3 and 4 are outside), got %d", len(posts)) + } + if posts[0].Content != "created note 1" || posts[1].Content != "deref note 2" { + t.Fatalf("unexpected in-window posts: %+v", posts) + } +} + +func TestOutboxResolveViaWebfinger(t *testing.T) { + server, _ := newRemoteServer() + defer server.Close() + + scheme := webfingerScheme + webfingerScheme = "http" + defer func() { webfingerScheme = scheme }() + + handle := "@alice@" + strings.TrimPrefix(server.URL, "http://") + oc := newOutboxClient() + posts, err := oc.getLatest(context.Background(), "", handle, 10) + if err != nil { + t.Fatalf("getLatest via webfinger error: %v", err) + } + if len(posts) != 4 { + t.Fatalf("expected 4 posts via webfinger, got %d", len(posts)) + } +} + +func TestOutboxResolveNote(t *testing.T) { + server, _ := newRemoteServer() + defer server.Close() + + oc := newOutboxClient() + note, err := oc.resolveNote(context.Background(), server.URL+"/notes/2") + if err != nil { + t.Fatalf("resolveNote error: %v", err) + } + if note.Content != "deref note 2" { + t.Fatalf("unexpected note: %+v", note) + } +} + +func TestNoteToInternalPostSetsAuthor(t *testing.T) { + note := &apNote{ + ID: "https://example.social/notes/1", + Content: "hello", + Published: "2024-01-01T00:00:00Z", + } + p := noteToInternalPost(note, "alice@example.social") + if p.Account.Acct != "alice@example.social" { + t.Fatalf("expected Account.Acct = 'alice@example.social', got %q", p.Account.Acct) + } + if p.toPost().Author != "alice@example.social" { + t.Fatalf("expected toPost().Author = 'alice@example.social', got %q", p.toPost().Author) + } +} + +func TestNoteToInternalPostNoAuthor(t *testing.T) { + note := &apNote{ + ID: "https://example.social/notes/1", + Content: "hello", + Published: "2024-01-01T00:00:00Z", + } + p := noteToInternalPost(note, "") + if p.Account.Acct != "" { + t.Fatalf("expected empty Account.Acct when no author, got %q", p.Account.Acct) + } + if p.toPost().Author != "" { + t.Fatalf("expected empty author when none provided, got %q", p.toPost().Author) + } +} + +func TestMergePostsBySourceDedup(t *testing.T) { + created := time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC) + cached := []*internalPost{ + {ID: "local-1", URI: "https://example.social/notes/1", Content: "cached", CreatedAt: created}, + } + outbox := []*Post{ + {ID: "https://example.social/notes/1", Content: "cached dup", CreatedAt: created}, + {ID: "https://example.social/notes/2", Content: "outbox only", CreatedAt: created.Add(-time.Hour)}, + } + merged := mergePostsBySource(cached, outbox) + if len(merged) != 2 { + t.Fatalf("expected dedup to drop the duplicate, got %d", len(merged)) + } + if merged[0].Content != "cached" { + t.Fatalf("newest first: expected cached first, got %q", merged[0].Content) + } + if merged[1].Content != "outbox only" { + t.Fatalf("expected outbox only second, got %q", merged[1].Content) + } +} + +func TestParseActivityTime(t *testing.T) { + for _, s := range []string{"2024-01-01T00:00:00Z", "2024-01-01T00:00:00.000Z", "2024-01-01T00:00:00+02:00"} { + if _, ok := parseActivityTime(s); !ok { + t.Fatalf("parseActivityTime(%q) failed", s) + } + } + if _, ok := parseActivityTime("not a date"); ok { + t.Fatal("expected invalid timestamp to fail") + } +} + +func TestAPStringOrArray(t *testing.T) { + var sa apStringOrArray + if err := json.Unmarshal([]byte(`"https://a"`), &sa); err != nil || len(sa) != 1 || sa[0] != "https://a" { + t.Fatalf("string form failed: %v %v", sa, err) + } + if err := json.Unmarshal([]byte(`["https://a","https://b"]`), &sa); err != nil || len(sa) != 2 { + t.Fatalf("array form failed: %v %v", sa, err) + } + if err := json.Unmarshal([]byte(`null`), &sa); err != nil || len(sa) != 0 { + t.Fatalf("null form failed: %v %v", sa, err) + } +} diff --git a/internal/activitypub/throttle.go b/internal/activitypub/throttle.go new file mode 100644 index 0000000..19287d5 --- /dev/null +++ b/internal/activitypub/throttle.go @@ -0,0 +1,58 @@ +package activitypub + +import ( + "context" + "sync" + "time" +) + +type hostLimiter struct { + mu sync.Mutex + buckets map[string]*bucket +} + +type bucket struct { + mu sync.Mutex + last time.Time +} + +func newHostLimiter() *hostLimiter { + return &hostLimiter{buckets: make(map[string]*bucket)} +} + +func (h *hostLimiter) Wait(ctx context.Context, host string) error { + h.mu.Lock() + b, ok := h.buckets[host] + if !ok { + b = &bucket{} + h.buckets[host] = b + if len(h.buckets) > 1024 { + for k, v := range h.buckets { + if time.Since(v.last) > 10*time.Minute { + delete(h.buckets, k) + } + } + } + } + h.mu.Unlock() + + b.mu.Lock() + defer b.mu.Unlock() + + if b.last.IsZero() { + b.last = time.Now() + return nil + } + wait := time.Until(b.last.Add(time.Second)) + if wait <= 0 { + b.last = time.Now() + return nil + } + select { + case <-time.After(wait): + b.last = time.Now() + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/activitypub/types.go b/internal/activitypub/types.go index 4c79f20..1d9fc27 100644 --- a/internal/activitypub/types.go +++ b/internal/activitypub/types.go @@ -270,3 +270,10 @@ type Tag struct { Name string `json:"name"` URL string `json:"url"` } + +// searchResult mirrors the /api/v2/search response: statuses are the api +// `status` objects served by the search endpoint. +type searchResult struct { + Accounts []*Account `json:"accounts"` + Statuses []*internalPost `json:"statuses"` +} From f4ae7d9c9f2f03e8365350e46141c6a650a27059 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:34 +0000 Subject: [PATCH 23/30] refactor(x402): fold resolver into AddPayableTool and honor per-tool timeouts Simplify the payable-tool registration API by merging AddPayableToolWithResolver into AddPayableTool, and thread a configurable MaxTimeoutSeconds plus Extra through the payment options. getStyleProfile keeps its 600s advertise timeout. --- internal/server/settlement.go | 4 ++-- internal/server/tools.go | 20 +++++++++--------- internal/x402/x402.go | 38 +++++++++-------------------------- internal/x402/x402_test.go | 8 +++++++- 4 files changed, 29 insertions(+), 41 deletions(-) diff --git a/internal/server/settlement.go b/internal/server/settlement.go index 03c1a34..04dbc98 100644 --- a/internal/server/settlement.go +++ b/internal/server/settlement.go @@ -44,7 +44,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass return nil } -func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string) []x402v2.PaymentRequirements { +func (s *Server) buildRequirements(amount string, extra map[string]any, scheme string, timeout int) []x402v2.PaymentRequirements { requirements := make([]x402v2.PaymentRequirements, 0, len(s.activeNetworks)) for _, info := range s.activeNetworks { @@ -57,7 +57,7 @@ func (s *Server) buildRequirements(amount string, extra map[string]any, scheme s Asset: info.Asset, Amount: amount, PayTo: s.x402Wallet, - MaxTimeoutSeconds: 600, + MaxTimeoutSeconds: timeout, Extra: mergedExtra, }) } diff --git a/internal/server/tools.go b/internal/server/tools.go index 9767aeb..bb91c3d 100644 --- a/internal/server/tools.go +++ b/internal/server/tools.go @@ -37,12 +37,13 @@ func (s *Server) registerTools() { } type payableDef struct { - Name string - Desc string - Input any - Output any - Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) - Price string + Name string + Desc string + Input any + Output any + Handler func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) + Price string + MaxTimeoutSeconds int // UsesTwitter marks tools that may read posts (or set up lookups) // through the Twitter API and return only transformative/aggregated // results, as permitted by the X API policy. With Twitter enabled these @@ -113,7 +114,8 @@ func (s *Server) registerTools() { // the posts read via x402.NewStyleProfileCostResolver. Name: "getStyleProfile", Desc: "Get an actor's style Profile", Input: GetStyleProfileRequest{}, Output: GetStyleProfileResponse{}, Handler: s.getStyleProfile, Price: "0.25", - UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, + MaxTimeoutSeconds: 600, + UsesTwitter: true, TwitterPostCap: x402.MaxProfileCountedPosts, Extra: schemaExtra( map[string]any{ "author": Bodyfield{Type: "string", Description: "Did or handle of an author", Required: true}, @@ -238,11 +240,11 @@ func (s *Server) registerTools() { resolver = x402.NewTwitterCountCostResolver(baseMicro) } } - s.mcpHandler.AddPayableToolWithResolver( + s.mcpHandler.AddPayableTool( mcp.Tool{Name: d.Name, Description: d.Desc, InputSchema: d.Input, OutputSchema: d.Output}, d.Handler, resolver, - s.buildRequirements(price, d.Extra, scheme)..., + s.buildRequirements(price, d.Extra, scheme, d.MaxTimeoutSeconds)..., ) } diff --git a/internal/x402/x402.go b/internal/x402/x402.go index 1884946..28eeb5c 100644 --- a/internal/x402/x402.go +++ b/internal/x402/x402.go @@ -198,33 +198,11 @@ func (s *X402Server) AddTool(tool mcp.Tool, handler mcp.ToolHandler) { } func (s *X402Server) AddPayableTool( - tool mcp.Tool, - handler mcp.ToolHandler, - requirements ...x402.PaymentRequirements, -) { - s.addPayableTool(tool, handler, nil, requirements...) -} - -// AddPayableToolWithResolver registers a payable tool with a -// SettleAmountResolver. The resolver is used by both the MCP wrapper and the -// REST override path to settle the actual computed cost after the handler runs -// (e.g. an upto-scheme tool charging Twitter API pass-through based on the -// response), instead of the advertised maximum. -func (s *X402Server) AddPayableToolWithResolver( tool mcp.Tool, handler mcp.ToolHandler, resolver SettleAmountResolver, requirements ...x402.PaymentRequirements, ) { - s.addPayableTool(tool, handler, resolver, requirements...) -} - -func (s *X402Server) addPayableTool( - tool mcp.Tool, - handler mcp.ToolHandler, - settleResolver SettleAmountResolver, - requirements ...x402.PaymentRequirements, -) { if len(requirements) == 0 { panic(fmt.Sprintf("tool %s requires at least one payment requirement", tool.Name)) @@ -253,13 +231,13 @@ func (s *X402Server) addPayableTool( tool.InputSchema = toJSONSchema(tool.InputSchema) tool.OutputSchema = toJSONSchema(tool.OutputSchema) - if settleResolver != nil { - s.settleResolvers[tool.Name] = settleResolver + if resolver != nil { + s.settleResolvers[tool.Name] = resolver } wrapper := NewPaymentWrapper(s.x402ResourceServer, PaymentWrapperConfig{ Accepts: mcpAccepts, - SettleAmountResolver: settleResolver, + SettleAmountResolver: resolver, Resource: &types.ResourceInfo{ URL: "mcp://tool/" + tool.Name, Description: tool.Description, @@ -278,10 +256,12 @@ func (s *X402Server) addPayableTool( var options x402http.PaymentOptions for _, req := range requirements { options = append(options, x402http.PaymentOption{ - Scheme: req.Scheme, - PayTo: req.PayTo, - Price: req.Amount, - Network: x402.Network(req.Network), + Scheme: req.Scheme, + PayTo: req.PayTo, + Price: req.Amount, + Network: x402.Network(req.Network), + MaxTimeoutSeconds: req.MaxTimeoutSeconds, + Extra: req.Extra, }) } diff --git a/internal/x402/x402_test.go b/internal/x402/x402_test.go index 53c7172..f26e9ca 100644 --- a/internal/x402/x402_test.go +++ b/internal/x402/x402_test.go @@ -52,6 +52,7 @@ func TestX402Server_HTTPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -124,6 +125,7 @@ func TestX402Server_MCPBoundaries(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -207,6 +209,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -219,6 +222,7 @@ func TestX402Server_MCPChallengesAdvertiseConvertedAmounts(t *testing.T) { func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: `{}`}}}, nil }, + nil, x402.PaymentRequirements{ Scheme: "upto", Network: "eip155:84532", @@ -618,6 +622,7 @@ func TestAddPayableTool_EmptyRequirements(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"status": "success"}`}}, }, nil }, + nil, // No requirements provided ) } @@ -654,6 +659,7 @@ func TestX402Server_MultipleTools(t *testing.T) { Content: []mcp.Content{&mcp.TextContent{Text: `{"tool": "` + name + `"}`}}, }, nil }, + nil, x402.PaymentRequirements{ Scheme: "exact", Network: "eip155:8453", @@ -794,7 +800,7 @@ func TestX402Server_MCPStyleProfileSettlementOverride(t *testing.T) { {Author: "@b", TopKeywords: []string{}, StyleSummary: "s", PostCount: 50, SourcePlatform: "Twitter"}, } var idx int - srv.AddPayableToolWithResolver( + srv.AddPayableTool( mcp.Tool{Name: "getStyleProfile", Description: "get a style profile"}, func(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { resp := responses[idx%len(responses)] From 9825c453504691f8e62f070789a79e49c0097f0e Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:37 +0000 Subject: [PATCH 24/30] fix(server): surface and log settlement persistence errors persistSettlement now returns the write error instead of swallowing it, and both the HTTP and MCP settlement hooks log failures instead of silently dropping them. Also remove the dead function_runtime_ms webhook field and a stale log value. --- internal/server/server_init.go | 17 ++++++++++------- internal/server/settlement.go | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/server/server_init.go b/internal/server/server_init.go index 0cef798..832d103 100644 --- a/internal/server/server_init.go +++ b/internal/server/server_init.go @@ -85,7 +85,9 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "amount", requirements.Amount, "asset", requirements.Asset, ) - persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer) + if err := persistSettlement(r.Context(), db, resp.Transaction, requirements.Amount, requirements.Asset, string(resp.Network), requirements.PayTo, resp.Payer); err != nil { + log.WarnWithContext(r.Context(), "failed to persist HTTP settlement", "error", err) + } }, networkStrings, ) @@ -103,9 +105,11 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t "asset", settleCtx.Requirements.GetAsset(), "scheme", settleCtx.Requirements.GetScheme(), ) - persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, + if err := persistSettlement(settleCtx.Ctx, db, settleCtx.Result.Transaction, settleCtx.Requirements.GetAmount(), settleCtx.Requirements.GetAsset(), - string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer) + string(settleCtx.Result.Network), settleCtx.Requirements.GetPayTo(), settleCtx.Result.Payer); err != nil { + log.WarnWithContext(settleCtx.Ctx, "failed to persist settlement", "error", err) + } // Webhook is best-effort and non-blocking. go sendSettlementWebhook(settleCtx.Ctx, cfg, metrics, settleCtx.Result.Transaction) @@ -116,8 +120,8 @@ func newX402Server(ctx context.Context, db *store.Queries, cfg *config.Config, t } // persistSettlement is a thin wrapper around settleHook to keep call sites uniform. -func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) { - _ = settleHook(ctx, db, transaction, amount, asset, network, to, from) +func persistSettlement(ctx context.Context, db *store.Queries, transaction, amount, asset, network, to, from string) error { + return settleHook(ctx, db, transaction, amount, asset, network, to, from) } // sendSettlementWebhook builds and sends the Matrix webhook payload for a settled transaction. @@ -132,8 +136,7 @@ func sendSettlementWebhook(ctx context.Context, cfg *config.Config, metrics *Met "x402_transaction_hash": txHash, }, "metrics": map[string]any{ - "function_runtime_ms": 0, - "platform": "unknown", + "platform": "unknown", }, "timestamp": time.Now().UTC().Format(time.RFC3339), "server_id": cfg.MatrixServerID, diff --git a/internal/server/settlement.go b/internal/server/settlement.go index 04dbc98..4acb2a0 100644 --- a/internal/server/settlement.go +++ b/internal/server/settlement.go @@ -36,7 +36,7 @@ func settleHook(ctx context.Context, db *store.Queries, transaction, amount, ass } if err != nil { - log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "value", ret, "error", err) + log.WithFields(fields).ErrorWithContext(ctx, "Failed to save transaction", "error", err) return err } From 19ae0d781a12e2a7bea911951ded16e61ea4908b Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:42 +0000 Subject: [PATCH 25/30] fix(server): bound post cache size with earliest-expiry eviction Cap the post TTL cache at 10000 entries; once full, evict the entry with the earliest expiry instead of growing without bound. --- internal/server/cache.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/server/cache.go b/internal/server/cache.go index 82c58ce..09b7f2a 100644 --- a/internal/server/cache.go +++ b/internal/server/cache.go @@ -6,6 +6,7 @@ import ( ) const postCacheTTL = 5 * time.Minute +const postCacheMaxItems = 10000 type ttlCacheEntry struct { value any @@ -13,15 +14,17 @@ type ttlCacheEntry struct { } type ttlCache struct { - mu sync.Mutex - items map[string]ttlCacheEntry - ttl time.Duration + mu sync.Mutex + items map[string]ttlCacheEntry + ttl time.Duration + maxItems int } func newTTLCache(ttl time.Duration) *ttlCache { return &ttlCache{ - items: make(map[string]ttlCacheEntry), - ttl: ttl, + items: make(map[string]ttlCacheEntry), + ttl: ttl, + maxItems: postCacheMaxItems, } } @@ -42,6 +45,22 @@ func (c *ttlCache) get(key string) (any, bool) { func (c *ttlCache) set(key string, value any) { c.mu.Lock() defer c.mu.Unlock() + if len(c.items) >= c.maxItems { + // Evict the oldest expired entry, or if none expired, the first one. + var oldestKey string + var oldestTime time.Time + first := true + for k, e := range c.items { + if first || e.exp.Before(oldestTime) { + oldestKey = k + oldestTime = e.exp + first = false + } + } + if oldestKey != "" { + delete(c.items, oldestKey) + } + } c.items[key] = ttlCacheEntry{value: value, exp: time.Now().Add(c.ttl)} } From c965eb51d3dae29e7e949ab536065a16b5183d77 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:43 +0000 Subject: [PATCH 26/30] refactor(server): type CDP private key as crypto.Signer Store the CDP auth key as a crypto.Signer interface so ECDSA and Ed25519 keys share one signing path, and detect raw base64 Ed25519 keys inline instead of a separate helper. --- internal/server/auth.go | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/internal/server/auth.go b/internal/server/auth.go index f822e48..07c8c24 100644 --- a/internal/server/auth.go +++ b/internal/server/auth.go @@ -2,7 +2,7 @@ package server import ( "context" - "crypto/ecdsa" + "crypto" "crypto/ed25519" "crypto/rand" "crypto/x509" @@ -20,7 +20,7 @@ import ( type CDPAuth struct { keyName string - privateKey interface{} // *ecdsa.PrivateKey or ed25519.PrivateKey + privateKey crypto.Signer signingMethod jwt.SigningMethod } @@ -36,13 +36,12 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { // Convert to real newlines before decoding; idempotent if already multiline. privateKeyPEM = strings.ReplaceAll(privateKeyPEM, "\\n", "\n") - // Check if it's a base64-encoded Ed25519 key (64 bytes) - if isValidEd25519Key(privateKeyPEM) { - decoded, _ := base64.StdEncoding.DecodeString(privateKeyPEM) - privateKey := ed25519.PrivateKey(decoded) + // Check if it's a base64-encoded Ed25519 key (64 bytes). + decoded, err := base64.StdEncoding.DecodeString(privateKeyPEM) + if err == nil && len(decoded) == 64 { return &CDPAuth{ keyName: keyName, - privateKey: privateKey, + privateKey: ed25519.PrivateKey(decoded), signingMethod: jwt.SigningMethodEdDSA, }, nil } @@ -66,15 +65,6 @@ func NewCDPAuth(keyName, privateKeyPEM string) (*CDPAuth, error) { }, nil } -// isValidEd25519Key checks if a string is a valid base64-encoded Ed25519 private key (64 bytes). -func isValidEd25519Key(str string) bool { - decoded, err := base64.StdEncoding.DecodeString(str) - if err != nil { - return false - } - return len(decoded) == 64 -} - func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { nonceBytes := make([]byte, 16) if _, err := rand.Read(nonceBytes); err != nil { @@ -101,10 +91,8 @@ func (a *CDPAuth) generateBearerToken(method, path string) (string, error) { var err error switch a.signingMethod { - case jwt.SigningMethodES256: - signedToken, err = token.SignedString(a.privateKey.(*ecdsa.PrivateKey)) - case jwt.SigningMethodEdDSA: - signedToken, err = token.SignedString(a.privateKey.(ed25519.PrivateKey)) + case jwt.SigningMethodES256, jwt.SigningMethodEdDSA: + signedToken, err = token.SignedString(a.privateKey) default: return "", fmt.Errorf("unsupported signing method: %v", a.signingMethod) } From 0db85ee65050c7be34248d9f40f8ede28d567a60 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:43 +0000 Subject: [PATCH 27/30] fix(server): add 5-minute timeout to prewarm profile builds Bound prewarmOne with a context timeout so a hung backend can't stall the prewarm scheduler. --- internal/server/prewarm.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/server/prewarm.go b/internal/server/prewarm.go index 1b5cdf5..2fd9041 100644 --- a/internal/server/prewarm.go +++ b/internal/server/prewarm.go @@ -83,6 +83,8 @@ func (s *Server) discoverInterestingAuthors(ctx context.Context) []string { } func (s *Server) prewarmOne(ctx context.Context, author string) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() if _, err := s.requestStyleProfile(ctx, author); err != nil { log.WarnWithContext(ctx, "prewarm: failed to build profile", "author", author, "error", err) return From e66900dc7561e4e40999bd01ce430e2b2555fc3e Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:45 +0000 Subject: [PATCH 28/30] fix(server): abort webhook retry backoff when context is cancelled Replace time.Sleep with a select on time.After and ctx.Done so webhook retries stop promptly when the request context is cancelled. --- internal/server/webhook.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/server/webhook.go b/internal/server/webhook.go index 75708c1..adda05e 100644 --- a/internal/server/webhook.go +++ b/internal/server/webhook.go @@ -47,7 +47,12 @@ func (wh *WebhookSender) Send(ctx context.Context, jsonBytes []byte) { for attempt := 1; attempt <= 5; attempt++ { delay := exponentialBackoff(attempt) if attempt > 1 { - time.Sleep(delay) + select { + case <-time.After(delay): + case <-ctx.Done(): + log.WarnWithContext(ctx, "webhook cancelled during backoff", "attempt", attempt, "error", ctx.Err()) + return + } } err := wh.sendAttempt(ctx, jsonBytes, attempt) From d1b7805162ccd2dc41f9456a143968273394b671 Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:45 +0000 Subject: [PATCH 29/30] test(server): sort fake store analytics posts ascending GetAnalyticsPostsInRange now returns posts oldest-first to match the real store implementation. --- internal/server/storefake_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/storefake_test.go b/internal/server/storefake_test.go index 066ca5e..4ab3b24 100644 --- a/internal/server/storefake_test.go +++ b/internal/server/storefake_test.go @@ -94,7 +94,7 @@ func (f *fakeStore) GetAnalyticsPostsInRange(ctx context.Context, arg store.GetA Quotes: p.Quotes, }) } - sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.After(rows[j].PostTimestamp) }) + sort.Slice(rows, func(i, j int) bool { return rows[i].PostTimestamp.Before(rows[j].PostTimestamp) }) if int32(len(rows)) > arg.Limit { rows = rows[:arg.Limit] } From 7a21437120faa48772fb74c2868147dbe7a2ad0e Mon Sep 17 00:00:00 2001 From: Nacorid Date: Sun, 13 Sep 2026 22:24:45 +0000 Subject: [PATCH 30/30] docs: tidy stale comments in analytics, cache version, and style profile test --- internal/server/analytics.go | 15 ++------------- internal/server/cache_version.go | 1 + internal/server/styleprofile_test.go | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/internal/server/analytics.go b/internal/server/analytics.go index 6db10e1..cb38bcf 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -633,20 +633,16 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T return []TopicCluster{}, map[string]string{}, nil } - // Use embeddings from ollama for clustering items := posts - // We need embeddings for clustering - compute them embeddings, err := s.ollama.EmbedTexts(ctx, items) if err != nil { return nil, nil, fmt.Errorf("failed to compute embeddings: %w", err) } - // Update items with embeddings for i := range items { items[i].Embedding = embeddings[i] } - // Determine optimal number of topics topicCount := maxTopicCount if len(items) < maxTopicCount { topicCount = len(items) @@ -655,7 +651,6 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T topicCount = minTopicCount } - // Run KMeans results, err := clustering.KMeans(items, topicCount, 50, 42) if err != nil { return nil, nil, fmt.Errorf("kmeans failed: %w", err) @@ -669,7 +664,6 @@ func (s *Server) clusterPosts(ctx context.Context, posts []clustering.Item) ([]T continue } - // Collect sample texts for labeling var samples []string for _, item := range cluster.Items { if len(samples) >= 3 { @@ -817,8 +811,7 @@ func (s *Server) computeSentimentTrends(ctx context.Context, args AnalyticsReque } func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsRequest, posts []clustering.Item) (any, error) { - // Topic modeling first, getting post->topic membership so we can correlate - // each topic with the engagement of its member posts. + // Topic modeling first, getting post->topic membership for correlation. topics, membership, err := s.clusterPosts(ctx, posts) if err != nil { return nil, err @@ -836,8 +829,7 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq topicEngagement[label] = append(topicEngagement[label], total) } - // Baseline is the mean engagement across all posts, used as the comparison - // point for relevance ranking. + // Baseline is the mean engagement across all posts, for comparison. var allEngagement []float64 for _, post := range posts { allEngagement = append(allEngagement, float64(post.Likes+post.Reposts+post.Replies+post.Quotes)) @@ -848,9 +840,6 @@ func (s *Server) computeEngagementDrivers(ctx context.Context, args AnalyticsReq for _, topic := range topics { values := topicEngagement[topic.Label] topicMean := mean(values) - // Correlation is the normalized difference between the topic's mean - // engagement and the overall baseline: positive when the topic draws - // above-average engagement, negative when below. Bounded to [-1, 1]. correlation := 0.0 if denom := topicMean + baseline; denom > 0 { correlation = (topicMean - baseline) / denom diff --git a/internal/server/cache_version.go b/internal/server/cache_version.go index 25fc192..40ea07e 100644 --- a/internal/server/cache_version.go +++ b/internal/server/cache_version.go @@ -8,6 +8,7 @@ const ( // IsProfileStale returns true if the profile's versions are older than the current versions. // This is used to mark profiles stale for lazy recompute on the next request. +// Currently all three version fields are 1 since this is the first version of the combination. func IsProfileStale(model, schema, algorithm int) bool { return model != CurrentModelVersion || schema != CurrentSchemaVersion || diff --git a/internal/server/styleprofile_test.go b/internal/server/styleprofile_test.go index 621e12d..9051f18 100644 --- a/internal/server/styleprofile_test.go +++ b/internal/server/styleprofile_test.go @@ -47,7 +47,7 @@ func TestGetStyleProfileBuildsFresh(t *testing.T) { } func TestGetStyleProfileServesFreshCache(t *testing.T) { - // A profile modified less than a week ago is served without hitting the + // A profile modified less than the 30-day stale threshold is served without hitting the // backend or ollama. st := newFakeStore() st.styleProfiles["@a@mastodon.social|ActivityPub"] = store.UserStyleProfile{