Add article summary feature with OpenAI integration

.golangci.yml

@@ -54,6 +54,8 @@ linters:
           disabled: true
         - name: nested-structs
           disabled: true
+        - name: max-public-structs
+          arguments: [6]
         - name: use-slices-sort
           disabled: true
     gocritic:

CLAUDE.md

@@ -10,7 +10,7 @@ ukeeper-readability is a Go web service that extracts article content from URLs
 
 ```bash
 go build -o ukeeper-readability .
-go test -timeout=60s -race ./...              # all tests (datastore tests use testcontainers)
+go test -timeout=60s -race ./...              # all tests (datastore rules tests use testcontainers)
 go test -timeout=60s -run TestName ./rest/     # single test
 golangci-lint run --max-issues-per-linter=0 --max-same-issues=0  # lint from repo root
 ```
@@ -21,15 +21,15 @@ Optional Cloudflare Browser Rendering flags (when both are set, uses `Cloudflare
 - `--cf-account-id` / `CF_ACCOUNT_ID` — Cloudflare account ID
 - `--cf-api-token` / `CF_API_TOKEN` — Cloudflare API token with Browser Rendering Edit permission
 
-`main_test.go` is gated behind `ENABLE_MONGO_TESTS=true` and needs MongoDB on localhost:27017. All other packages test independently — `datastore/` spins up MongoDB via testcontainers automatically.
+`main_test.go` and `datastore/summaries_test.go` are gated behind `ENABLE_MONGO_TESTS=true` and need MongoDB on localhost:27017 — without it the summaries tests skip silently. `datastore/rules_test.go` spins up MongoDB via testcontainers and needs no env var. CI sets `ENABLE_MONGO_TESTS=true` with a Mongo service, so the gap is local only.
 
 ## Architecture
 
 ```
 main.go        → CLI flags (jessevdk/go-flags), wiring, startup
-datastore/     → MongoDB access (RulesDAO, Rule struct)
+datastore/     → MongoDB access (RulesDAO/Rule, SummariesDAO/Summary)
 extractor/     → URL fetching, content extraction, charset conversion
-  mocks/       → moq-generated mock for Rules interface
+  mocks/       → moq-generated mocks for Rules, Summaries, OpenAIClient
 rest/          → HTTP server, routing (go-pkgz/routegroup), handlers, basicAuth
 web/           → Go HTML templates (HTMX v2), static assets
 ```
@@ -40,6 +40,8 @@ web/           → Go HTML templates (HTMX v2), static assets
 - `extractor.Rules` (defined consumer-side in `extractor/readability.go`), implemented by `datastore.RulesDAO`. Mock generated with `//go:generate moq` in extractor package.
 - `extractor.Retriever` (defined in `extractor/retriever.go`) — abstracts URL content fetching. Two implementations: `HTTPRetriever` (default, standard HTTP GET with Safari user-agent) and `CloudflareRetriever` (Cloudflare Browser Rendering API for JS-rendered pages). When `UReadability.Retriever` is nil, defaults to `HTTPRetriever`.
 - `extractor.AIEvaluator` (defined in `extractor/evaluator.go`) — evaluates extraction quality via OpenAI. Implementation: `OpenAIEvaluator`. Mock generated with `//go:generate moq` as test-only mock (`evaluator_mock_test.go`).
+- `extractor.Summaries` (defined in `extractor/readability.go`), implemented by `datastore.SummariesDAO` — cache for generated article summaries. Mock in `extractor/mocks/summaries.go`.
+- `extractor.OpenAIClient` (defined in `extractor/readability.go`) — the summary-generation calls, satisfied by the `go-openai` client. Mock in `extractor/mocks/openai_client.go`.
 
 ## Content Extraction Flow
 
@@ -53,10 +55,12 @@ web/           → Go HTML templates (HTMX v2), static assets
 
 `ExtractAndImprove()` is the force-mode entry point — ignores stored rules, re-extracts with general parser, then evaluates. Used by the `/api/content-parsed-wrong` protected endpoint.
 
-Optional OpenAI flags (when `--openai-api-key` is set, enables auto-evaluation):
-- `--openai-api-key` / `OPENAI_API_KEY` — OpenAI API key
+Optional OpenAI flags. One key powers both auto-evaluation and summaries; each is switched off on its own:
+- `--openai-api-key` / `OPENAI_API_KEY` — OpenAI API key, shared by both features
 - `--openai-model` / `OPENAI_MODEL` — model for evaluation (default: `gpt-5.4-mini`)
 - `--openai-max-iter` / `OPENAI_MAX_ITER` — max evaluation iterations (default: `3`)
+- `--openai-disable-eval` / `OPENAI_DISABLE_EVAL` — disable auto-evaluation
+- the `--openai.*` group (`disable-summaries`, `model-type`, `summary-prompt`, `max-content-length`, `requests-per-minute`, `cleanup-interval`) configures summaries; see README for the full table
 
 ## Key Conventions
 

README.md

@@ -19,11 +19,26 @@
 | cf-account-id| CF_ACCOUNT_ID   | none           | Cloudflare account ID for Browser Rendering API       |
 | cf-api-token | CF_API_TOKEN    | none           | Cloudflare API token with Browser Rendering Edit perm |
 | cf-route-all | CF_ROUTE_ALL    | `false`        | route every request through Cloudflare Browser Rendering |
-| openai-api-key | OPENAI_API_KEY | none          | OpenAI API key; enables auto-evaluation when set      |
+| openai-api-key | OPENAI_API_KEY | none          | OpenAI API key, shared by auto-evaluation and summaries |
 | openai-model | OPENAI_MODEL    | `gpt-5.4-mini` | OpenAI model for evaluation                           |
 | openai-max-iter | OPENAI_MAX_ITER | `3`         | max evaluation iterations per extraction               |
+| openai-disable-eval | OPENAI_DISABLE_EVAL | `false` | disable extraction auto-evaluation                    |
 | dbg          | DEBUG           | `false`        | debug mode                                            |
 
+#### OpenAI integration
+
+The API key is set once with `--openai-api-key`; the two OpenAI features share it and are switched off
+independently with `--openai-disable-eval` and `--openai.disable-summaries`.
+
+| Command line                  | Environment                | Default       | Description                                                      |
+|-------------------------------|----------------------------|---------------|------------------------------------------------------------------|
+| openai.model-type             | OPENAI_MODEL_TYPE          | `gpt-4o-mini` | OpenAI model name (e.g., gpt-4o, gpt-4o-mini)                   |
+| openai.disable-summaries      | OPENAI_DISABLE_SUMMARIES   | `false`       | disable summary generation                                       |
+| openai.summary-prompt         | OPENAI_SUMMARY_PROMPT      | built-in      | custom prompt for summary generation                             |
+| openai.max-content-length     | OPENAI_MAX_CONTENT_LENGTH  | `10000`       | maximum content length to send to OpenAI API (0 for no limit)    |
+| openai.requests-per-minute    | OPENAI_REQUESTS_PER_MINUTE | `10`          | maximum OpenAI API requests per minute (0 for no limit)          |
+| openai.cleanup-interval       | OPENAI_CLEANUP_INTERVAL    | `24h`         | interval for cleaning up expired cached summaries                |
+
 ### Cloudflare Browser Rendering (optional)
 
 Cloudflare Browser Rendering is useful for JavaScript-heavy pages and sites behind a "please enable JS" wall, but it's slower than direct HTTP and the free tier throttles at 1 request per 10 seconds. To keep the service cost-effective, Cloudflare routing is **opt-in**.
@@ -35,7 +50,7 @@ When Cloudflare credentials are not set, the service uses a standard HTTP client
 
 ### OpenAI Auto-Evaluation (optional)
 
-When `--openai-api-key` is set, the service automatically evaluates extraction quality using OpenAI. If the extracted content looks poor (missing article body, too short, mostly boilerplate), GPT suggests a CSS selector targeting the main content. The service iterates up to `--openai-max-iter` times, saving the best selector as a rule for future use.
+When `--openai-api-key` is set and `--openai-disable-eval` is not, the service automatically evaluates extraction quality using OpenAI. If the extracted content looks poor (missing article body, too short, mostly boilerplate), GPT suggests a CSS selector targeting the main content. The service iterates up to `--openai-max-iter` times, saving the best selector as a rule for future use.
 
 Evaluation only runs for domains without an existing extraction rule. For domains that already have rules, use the force-mode endpoint to re-evaluate:
 
@@ -44,13 +59,19 @@ Evaluation only runs for domains without an existing extraction rule. For domain
 This protected endpoint (requires basicAuth credentials) ignores the stored rule, re-extracts with the general parser, and runs the evaluation loop to find a better selector.
 
 When OpenAI is not configured, extraction works exactly as before — no GPT calls are made.
+Setting the key enables both auto-evaluation and summaries; disable either one on its own with
+`--openai-disable-eval` or `--openai.disable-summaries`.
 
 ### API
 
     GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
+    GET /api/content/v1/parser?token=secret&url=http://aa.com/blah&summary=true - extract content with AI-generated summary
     POST /api/extract {url: http://aa.com/blah}  - extract content
+    GET /api/metrics - summary generation metrics (cache hits, misses, response times)
     POST /api/content-parsed-wrong?url=http://aa.com/blah - force re-extraction with AI evaluation (requires basicAuth)
 
+Summary generation requires a valid token and an OpenAI API key. Summaries are cached in MongoDB with a 1-month expiration. Expired summaries are cleaned up automatically on the configured interval.
+
 ## Development
 
 ### Running tests

datastore/mongo.go

@@ -39,7 +39,8 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error
 
 // Stores contains all DAO instances
 type Stores struct {
-	Rules RulesDAO
+	Rules     RulesDAO
+	Summaries SummariesDAO
 }
 
 // GetStores initialize collections and make indexes
@@ -50,8 +51,15 @@ func (m *MongoServer) GetStores() Stores {
 		{Keys: bson.D{{Key: fieldDomain, Value: 1}, {Key: "match_urls", Value: 1}}},
 	}
 
+	sIndexes := []mongo.IndexModel{
+		{Keys: bson.D{{Key: "created_at", Value: 1}}},
+		{Keys: bson.D{{Key: "model", Value: 1}}},
+		{Keys: bson.D{{Key: fieldExpiresAt, Value: 1}}}, // index for cleaning up expired summaries
+	}
+
 	return Stores{
-		Rules: RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Rules:     RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Summaries: SummariesDAO{Collection: m.collection("summaries", sIndexes)},
 	}
 }
 

datastore/rules.go

@@ -13,10 +13,11 @@ import (
 
 // mongo field names and operators reused across queries
 const (
-	fieldDomain  = "domain"
-	fieldEnabled = "enabled"
-	fieldID      = "_id"
-	opSet        = "$set"
+	fieldDomain    = "domain"
+	fieldEnabled   = "enabled"
+	fieldID        = "_id"
+	fieldExpiresAt = "expires_at"
+	opSet          = "$set"
 )
 
 // RulesDAO data-access obj for custom parsing rules, implements Rules

datastore/summaries.go

@@ -0,0 +1,109 @@
+// Package datastore provides mongo implementation for store to keep and access summaries
+package datastore
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+// Summary contains information about a cached summary
+type Summary struct {
+	ID        string    `bson:"_id"`     // sha256 hash of the content
+	Content   string    `bson:"content"` // original content that was summarized (could be truncated for storage efficiency)
+	Summary   string    `bson:"summary"` // generated summary
+	Model     string    `bson:"model"`   // openAI model used for summarisation
+	CreatedAt time.Time `bson:"created_at"`
+	UpdatedAt time.Time `bson:"updated_at"`
+	ExpiresAt time.Time `bson:"expires_at"` // when this summary expires
+}
+
+// SummariesDAO handles database operations for article summaries
+type SummariesDAO struct {
+	Collection *mongo.Collection
+}
+
+// Get returns summary by content hash
+func (s SummariesDAO) Get(ctx context.Context, content string) (Summary, bool) {
+	contentHash := GenerateContentHash(content)
+	res := s.Collection.FindOne(ctx, bson.M{fieldID: contentHash})
+	if res.Err() != nil {
+		if res.Err() == mongo.ErrNoDocuments {
+			return Summary{}, false
+		}
+		log.Printf("[WARN] can't get summary for hash %s: %v", contentHash, res.Err())
+		return Summary{}, false
+	}
+
+	summary := Summary{}
+	if err := res.Decode(&summary); err != nil {
+		log.Printf("[WARN] can't decode summary document for hash %s: %v", contentHash, err)
+		return Summary{}, false
+	}
+
+	return summary, true
+}
+
+// Save creates or updates summary in the database
+func (s SummariesDAO) Save(ctx context.Context, summary Summary) error {
+	if summary.ID == "" {
+		summary.ID = GenerateContentHash(summary.Content)
+	}
+
+	if summary.CreatedAt.IsZero() {
+		summary.CreatedAt = time.Now()
+	}
+	summary.UpdatedAt = time.Now()
+
+	// set default expiration of 1 month if not specified
+	if summary.ExpiresAt.IsZero() {
+		summary.ExpiresAt = time.Now().AddDate(0, 1, 0)
+	}
+
+	opts := options.UpdateOne().SetUpsert(true)
+	_, err := s.Collection.UpdateOne(
+		ctx,
+		bson.M{fieldID: summary.ID},
+		bson.M{"$set": summary},
+		opts,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to save summary: %w", err)
+	}
+	return nil
+}
+
+// Delete removes summary from the database
+func (s SummariesDAO) Delete(ctx context.Context, contentHash string) error {
+	_, err := s.Collection.DeleteOne(ctx, bson.M{fieldID: contentHash})
+	if err != nil {
+		return fmt.Errorf("failed to delete summary: %w", err)
+	}
+	return nil
+}
+
+// CleanupExpired removes all summaries that have expired
+func (s SummariesDAO) CleanupExpired(ctx context.Context) (int64, error) {
+	now := time.Now()
+	result, err := s.Collection.DeleteMany(
+		ctx,
+		bson.M{fieldExpiresAt: bson.M{"$lt": now}},
+	)
+	if err != nil {
+		return 0, fmt.Errorf("failed to cleanup expired summaries: %w", err)
+	}
+	return result.DeletedCount, nil
+}
+
+// GenerateContentHash creates a hash for the content to use as an ID
+func GenerateContentHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}