Add OpenAI article summaries

CLAUDE.md

@@ -27,9 +27,9 @@ Optional Cloudflare Browser Rendering flags (when both are set, uses `Cloudflare
 
 ```
 main.go        → CLI flags (jessevdk/go-flags), wiring, startup
-datastore/     → MongoDB access (RulesDAO, Rule struct)
+datastore/     → MongoDB access (RulesDAO/Rule, SummariesDAO/Summary)
 extractor/     → URL fetching, content extraction, charset conversion
-  mocks/       → moq-generated mock for Rules interface
+  mocks/       → moq-generated mocks for Rules, Summaries, OpenAIClient
 rest/          → HTTP server, routing (go-pkgz/routegroup), handlers, basicAuth
 web/           → Go HTML templates (HTMX v2), static assets
 ```
@@ -39,6 +39,9 @@ web/           → Go HTML templates (HTMX v2), static assets
 **Key interfaces:**
 - `extractor.Rules` (defined consumer-side in `extractor/readability.go`), implemented by `datastore.RulesDAO`. Mock generated with `//go:generate moq` in extractor package.
 - `extractor.Retriever` (defined in `extractor/retriever.go`) — abstracts URL content fetching. Two implementations: `HTTPRetriever` (default, standard HTTP GET with Safari user-agent) and `CloudflareRetriever` (Cloudflare Browser Rendering API for JS-rendered pages). When `UReadability.Retriever` is nil, defaults to `HTTPRetriever`.
+- `extractor.OpenAIClient` and `extractor.Summaries` (defined in `extractor/summary.go`): the go-openai client and the summary cache (`datastore.SummariesDAO`, TTL index on `expires_at`). `UReadability.OpenAI` is set only when `--openai.api-key` is given; nil means summaries are off. Mocks in `extractor/mocks/`; moq cannot read Go 1.27 export data, so regenerate them with a newer moq or edit by hand.
+
+Optional OpenAI flags, all in the `--openai.*` group: `api-key` / `OPENAI_API_KEY` (enables summaries), `model` / `OPENAI_MODEL` (default `gpt-6-luna`), `summary-prompt`, `max-content-length` (characters, default 10000).
 
 ## Content Extraction Flow
 

README.md

@@ -21,6 +21,17 @@
 | cf-route-all | CF_ROUTE_ALL    | `false`        | route every request through Cloudflare Browser Rendering |
 | dbg          | DEBUG           | `false`        | debug mode                                            |
 
+#### OpenAI summaries (optional)
+
+Setting `--openai.api-key` enables article summaries; without it no OpenAI calls are made.
+
+| Command line              | Environment               | Default      | Description                                                        |
+|---------------------------|---------------------------|--------------|--------------------------------------------------------------------|
+| openai.api-key            | OPENAI_API_KEY            | none         | OpenAI API key, enables summaries                                  |
+| openai.model              | OPENAI_MODEL              | `gpt-6-luna` | OpenAI model                                                       |
+| openai.summary-prompt     | OPENAI_SUMMARY_PROMPT     | built-in     | custom system prompt for summaries                                 |
+| openai.max-content-length | OPENAI_MAX_CONTENT_LENGTH | `10000`      | max characters of article text sent for summarizing (0 for no limit) |
+
 ### Cloudflare Browser Rendering (optional)
 
 Cloudflare Browser Rendering is useful for JavaScript-heavy pages and sites behind a "please enable JS" wall, but it's slower than direct HTTP and the free tier throttles at 1 request per 10 seconds. To keep the service cost-effective, Cloudflare routing is **opt-in**.
@@ -33,8 +44,11 @@ When Cloudflare credentials are not set, the service uses a standard HTTP client
 ### API
 
     GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
+    GET /api/content/v1/parser?token=secret&url=http://aa.com/blah&summary=true - extract content with AI-generated summary
     POST /api/extract {url: http://aa.com/blah}  - extract content
 
+Summary generation requires the server token to be set and an OpenAI API key. Summaries are cached in MongoDB for 30 days, keyed by model, prompt and article text; a TTL index removes expired ones. When the key is set, rule previews in the web UI also show a summary.
+
 ## Development
 
 ### Running tests
@@ -45,4 +59,4 @@ Command to run full test suite would be:
 
 ```shell
 ENABLE_MONGO_TESTS=true go test ./...
-```
+```

datastore/mongo.go

@@ -39,7 +39,8 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error
 
 // Stores contains all DAO instances
 type Stores struct {
-	Rules RulesDAO
+	Rules     RulesDAO
+	Summaries SummariesDAO
 }
 
 // GetStores initialize collections and make indexes
@@ -50,8 +51,13 @@ func (m *MongoServer) GetStores() Stores {
 		{Keys: bson.D{{Key: fieldDomain, Value: 1}, {Key: "match_urls", Value: 1}}},
 	}
 
+	sIndexes := []mongo.IndexModel{
+		{Keys: bson.D{{Key: fieldExpiresAt, Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)},
+	}
+
 	return Stores{
-		Rules: RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Rules:     RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Summaries: SummariesDAO{Collection: m.collection("summaries", sIndexes)},
 	}
 }
 

datastore/rules.go

@@ -13,10 +13,11 @@ import (
 
 // mongo field names and operators reused across queries
 const (
-	fieldDomain  = "domain"
-	fieldEnabled = "enabled"
-	fieldID      = "_id"
-	opSet        = "$set"
+	fieldDomain    = "domain"
+	fieldEnabled   = "enabled"
+	fieldID        = "_id"
+	fieldExpiresAt = "expires_at"
+	opSet          = "$set"
 )
 
 // RulesDAO data-access obj for custom parsing rules, implements Rules

datastore/summaries.go

@@ -0,0 +1,49 @@
+package datastore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+// Summary is a cached article summary, keyed by a hash of the summarized input
+type Summary struct {
+	ID        string    `bson:"_id"`
+	Summary   string    `bson:"summary"`
+	Model     string    `bson:"model"`
+	CreatedAt time.Time `bson:"created_at"`
+	ExpiresAt time.Time `bson:"expires_at"` // mongo TTL index on this field deletes the document
+}
+
+// SummariesDAO data-access obj for cached summaries, implements Summaries
+type SummariesDAO struct {
+	*mongo.Collection
+}
+
+// Get returns an unexpired summary by id
+func (s SummariesDAO) Get(ctx context.Context, id string) (Summary, bool) {
+	var res Summary
+	// the TTL monitor runs about once a minute, so expired documents can still be present
+	err := s.FindOne(ctx, bson.M{fieldID: id, fieldExpiresAt: bson.M{"$gt": time.Now()}}).Decode(&res)
+	if err != nil {
+		if !errors.Is(err, mongo.ErrNoDocuments) {
+			log.Printf("[WARN] can't get summary %s, error=%v", id, err)
+		}
+		return Summary{}, false
+	}
+	return res, true
+}
+
+// Save upserts summary by id
+func (s SummariesDAO) Save(ctx context.Context, summary Summary) error {
+	if _, err := s.ReplaceOne(ctx, bson.M{fieldID: summary.ID}, summary, options.Replace().SetUpsert(true)); err != nil {
+		return fmt.Errorf("save summary %s: %w", summary.ID, err)
+	}
+	return nil
+}

datastore/summaries_test.go

@@ -0,0 +1,77 @@
+package datastore
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/testutils/containers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSummariesSaveAndGet(t *testing.T) {
+	summaries := setupStores(t).Summaries
+	ctx := context.Background()
+	now := time.Now().Truncate(time.Millisecond)
+
+	t.Run("save and get", func(t *testing.T) {
+		s := Summary{ID: "id-1", Summary: "short version", Model: "gpt-6-luna", CreatedAt: now, ExpiresAt: now.Add(time.Hour)}
+		require.NoError(t, summaries.Save(ctx, s))
+		got, ok := summaries.Get(ctx, "id-1")
+		require.True(t, ok)
+		assert.Equal(t, "short version", got.Summary)
+		assert.Equal(t, "gpt-6-luna", got.Model)
+		assert.True(t, got.CreatedAt.Equal(now))
+	})
+
+	t.Run("save replaces existing", func(t *testing.T) {
+		require.NoError(t, summaries.Save(ctx, Summary{ID: "id-2", Summary: "old", ExpiresAt: now.Add(time.Hour)}))
+		require.NoError(t, summaries.Save(ctx, Summary{ID: "id-2", Summary: "new", ExpiresAt: now.Add(time.Hour)}))
+		got, ok := summaries.Get(ctx, "id-2")
+		require.True(t, ok)
+		assert.Equal(t, "new", got.Summary)
+	})
+
+	t.Run("missing", func(t *testing.T) {
+		_, ok := summaries.Get(ctx, "no-such-id")
+		assert.False(t, ok)
+	})
+
+	t.Run("expired is not returned", func(t *testing.T) {
+		require.NoError(t, summaries.Save(ctx, Summary{ID: "id-3", Summary: "stale", ExpiresAt: now.Add(-time.Minute)}))
+		_, ok := summaries.Get(ctx, "id-3")
+		assert.False(t, ok)
+	})
+}
+
+func TestSummariesTTLIndex(t *testing.T) {
+	summaries := setupStores(t).Summaries
+	cursor, err := summaries.Indexes().List(context.Background())
+	require.NoError(t, err)
+	var indexes []struct {
+		Key                map[string]int `bson:"key"`
+		ExpireAfterSeconds *int           `bson:"expireAfterSeconds"`
+	}
+	require.NoError(t, cursor.All(context.Background(), &indexes))
+
+	for _, idx := range indexes {
+		if len(idx.Key) != 1 || idx.Key[fieldExpiresAt] != 1 {
+			continue
+		}
+		require.NotNil(t, idx.ExpireAfterSeconds, "index on %s is not a TTL index", fieldExpiresAt)
+		assert.Equal(t, 0, *idx.ExpireAfterSeconds)
+		return
+	}
+	t.Fatalf("no TTL index on %s, indexes=%v", fieldExpiresAt, indexes)
+}
+
+func setupStores(t *testing.T) Stores {
+	t.Helper()
+	mc := containers.NewMongoTestContainer(context.Background(), t, 5)
+	t.Cleanup(func() { mc.Close(context.Background()) }) //nolint:errcheck
+
+	server, err := New(mc.URI, "test_ureadability", 0)
+	require.NoError(t, err)
+	return server.GetStores()
+}

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - MONGO_URI=mongodb://root:Squid3kIc6Dew4ad8Ci5@mongo:27017
       - MONGO_DELAY=10s
       - CREDS=admin:jpm6AQH!kbx2tvk!fqc
-      #- OPENAI_KEY=insert_key_here
+      #- OPENAI_API_KEY=insert_key_here
 
     ports:
       - "8080:8080"