Function bodies 199 total

func segmentNorm(v [128]float32, seg [2]int) float32 {
	var sum float64
	for i := seg[0]; i < seg[1]; i++ {
		sum += float64(v[i]) * float64(v[i])
	}
	return float32(math.Sqrt(sum))
}

func computeSoftScore(
	old state.StateRecord,
	proposed state.StateRecord,
	metrics update.Metrics,
	entropy float32,
	minEntropyDrop float32,
) float32 {
	var score float32

	// Entropy component: reward entropy drop (weight 0.4)
	oldNorm := vectorNorm(old.StateVector)
	if oldNorm > 0 {
		// Use entropy as proxy — lower entropy after update is better
		if entropy < 1.0 {
			score += 0.4 * (1.0 - entropy)
		}
	} else {
		score += 0.2 // neutral when no prior state
	}

	// Delta stability component: smaller deltas are more stable (weight 0.3)
	deltaNorm := metrics.DeltaNorm
	if deltaNorm == 0 {
		score += 0.3 // no change = perfectly stable
	} else if deltaNorm < 1.0 {
		score += 0.3 * (1.0 - deltaNorm)
	}

	// Segments hit component: fewer segments changed = more focused (weight 0.3)
	hitCount := len(metrics.SegmentsHit)
	switch {
	case hitCount == 0:
		score += 0.3
	case hitCount == 1:
		score += 0.2
	case hitCount == 2:
		score += 0.1
	}

	return score
}

func DefaultGateConfig() GateConfig {
	return GateConfig{
		MaxDeltaNorm:   5.0,
		MaxStateNorm:   50.0,
		MinEntropyDrop: 0.1,
		RiskSegmentCap: 10.0,
	}
}

func NewInteriorStore(db *sql.DB) (*InteriorStore, error) {
	s := &InteriorStore{db: db}
	if err := s.init(); err != nil {
		return nil, err
	}
	return s, nil
}

func (s *InteriorStore) init() error {
	_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS interior_state (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		turn_id TEXT NOT NULL,
		reflection_text TEXT NOT NULL,
		created_at TEXT NOT NULL
	)`)
	return err
}

func (s *InteriorStore) Save(turnID, reflectionText string) error {
	_, err := s.db.Exec(
		`INSERT INTO interior_state (turn_id, reflection_text, created_at) VALUES (?, ?, ?)`,
		turnID, reflectionText, time.Now().UTC().Format(time.RFC3339),
	)
	return err
}

func (s *InteriorStore) Latest() (*Reflection, error) {
	row := s.db.QueryRow(
		`SELECT turn_id, reflection_text, created_at FROM interior_state ORDER BY id DESC LIMIT 1`,
	)
	var r Reflection
	var createdAt string
	if err := row.Scan(&r.TurnID, &r.ReflectionText, &createdAt); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}
	r.CreatedAt, _ = time.Parse(time.RFC3339, createdAt)
	return &r, nil
}

func ExtractCuriosity(text string) []string {
	lower := strings.ToLower(text)
	triggers := []string{
		"i want to know",
		"i wonder",
		"i'm curious",
		"i am curious",
		"i don't know",
		"i do not know",
		"i'd like to understand",
		"i want to understand",
		"i need to understand",
	}
	var found []string
	for _, t := range triggers {
		if strings.Contains(lower, t) {
			found = append(found, t)
		}
	}
	return found
}

func LogDecision(db *sql.DB, entry ProvenanceEntry) error {
	if entry.CreatedAt.IsZero() {
		entry.CreatedAt = time.Now().UTC()
	}

	_, err := db.Exec(
		`INSERT INTO provenance_log (version_id, context_hash, trigger_type, signals_json, evidence_refs, decision, reason, created_at)
		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
		entry.VersionID,
		nullIfEmpty(entry.ContextHash),
		entry.TriggerType,
		nullIfEmpty(entry.SignalsJSON),
		nullIfEmpty(entry.EvidenceRefs),
		entry.Decision,
		nullIfEmpty(entry.Reason),
		entry.CreatedAt.Format(time.RFC3339Nano),
	)
	if err != nil {
		return fmt.Errorf("log decision: %w", err)
	}
	return nil
}

func NewPreferenceStore(db *sql.DB) (*PreferenceStore, error) {
	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS preferences (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		text TEXT NOT NULL,
		style TEXT NOT NULL DEFAULT 'general',
		source TEXT NOT NULL DEFAULT 'explicit',
		created_at DATETIME NOT NULL
	)`)
	if err != nil {
		return nil, fmt.Errorf("create preferences table: %w", err)
	}
	// Migrate: add style column if missing (pre-existing tables lack it)
	_, _ = db.Exec(`ALTER TABLE preferences ADD COLUMN style TEXT NOT NULL DEFAULT 'general'`)
	return &PreferenceStore{db: db}, nil
}

func (s *PreferenceStore) Add(text, source string) error {
	style := InferStyle(text)

	// Exact duplicate check (case-insensitive)
	var count int
	err := s.db.QueryRow("SELECT COUNT(*) FROM preferences WHERE LOWER(text) = LOWER(?)", text).Scan(&count)
	if err != nil {
		return fmt.Errorf("check duplicate preference: %w", err)
	}
	if count > 0 {
		return nil
	}

	// Contradiction handling: replace existing preference of same non-general style
	if style != StyleGeneral {
		_, err = s.db.Exec("DELETE FROM preferences WHERE style = ?", string(style))
		if err != nil {
			return fmt.Errorf("remove contradicting preference: %w", err)
		}
	}

	_, err = s.db.Exec(
		"INSERT INTO preferences (text, style, source, created_at) VALUES (?, ?, ?, ?)",
		text, string(style), source, time.Now().UTC(),
	)
	if err != nil {
		return fmt.Errorf("insert preference: %w", err)
	}
	return nil
}

func (s *PreferenceStore) List() ([]Preference, error) {
	rows, err := s.db.Query("SELECT id, text, style, source, created_at FROM preferences ORDER BY created_at")
	if err != nil {
		return nil, fmt.Errorf("list preferences: %w", err)
	}
	defer rows.Close()

	var prefs []Preference
	for rows.Next() {
		var p Preference
		var ts, style string
		if err := rows.Scan(&p.ID, &p.Text, &style, &p.Source, &ts); err != nil {
			return nil, fmt.Errorf("scan preference: %w", err)
		}
		p.Style = PreferenceStyle(style)
		p.CreatedAt, _ = time.Parse(time.RFC3339, ts)
		prefs = append(prefs, p)
	}
	return prefs, nil
}

func NewRuleStore(db *sql.DB) (*RuleStore, error) {
	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS rules (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		trigger TEXT NOT NULL,
		response TEXT NOT NULL,
		priority INTEGER NOT NULL DEFAULT 5,
		confidence REAL NOT NULL DEFAULT 1.0,
		created_at DATETIME NOT NULL
	)`)
	if err != nil {
		return nil, fmt.Errorf("create rules table: %w", err)
	}
	return &RuleStore{db: db}, nil
}

func (s *RuleStore) Add(trigger, response string, priority int, confidence float64) error {
	trigger = strings.TrimSpace(trigger)
	response = strings.TrimSpace(response)
	if trigger == "" || response == "" {
		return fmt.Errorf("rule trigger and response must be non-empty")
	}

	// Replace existing rule with same trigger (case-insensitive)
	_, err := s.db.Exec("DELETE FROM rules WHERE LOWER(trigger) = LOWER(?)", trigger)
	if err != nil {
		return fmt.Errorf("remove existing rule: %w", err)
	}

	_, err = s.db.Exec(
		"INSERT INTO rules (trigger, response, priority, confidence, created_at) VALUES (?, ?, ?, ?, ?)",
		trigger, response, priority, confidence, time.Now().UTC(),
	)
	if err != nil {
		return fmt.Errorf("insert rule: %w", err)
	}
	return nil
}

func (s *RuleStore) List() ([]Rule, error) {
	rows, err := s.db.Query("SELECT id, trigger, response, priority, confidence, created_at FROM rules ORDER BY priority DESC, created_at")
	if err != nil {
		return nil, fmt.Errorf("list rules: %w", err)
	}
	defer rows.Close()

	var rules []Rule
	for rows.Next() {
		var r Rule
		var ts string
		if err := rows.Scan(&r.ID, &r.Trigger, &r.Response, &r.Priority, &r.Confidence, &ts); err != nil {
			return nil, fmt.Errorf("scan rule: %w", err)
		}
		r.CreatedAt, _ = time.Parse(time.RFC3339, ts)
		rules = append(rules, r)
	}
	return rules, nil
}

func (s *RuleStore) Match(input string) ([]Rule, error) {
	lower := strings.ToLower(strings.TrimSpace(input))
	if lower == "" {
		return nil, nil
	}

	rules, err := s.List()
	if err != nil {
		return nil, err
	}

	var matched []Rule
	for _, r := range rules {
		if strings.ToLower(r.Trigger) == lower {
			matched = append(matched, r)
		}
	}
	return matched, nil
}

func isDesireToAction(lower, matchedPattern string) bool {
	if !desireVerbs[matchedPattern] {
		return false
	}
	idx := strings.Index(lower, matchedPattern)
	if idx < 0 {
		return false
	}
	after := strings.TrimSpace(lower[idx+len(matchedPattern):])

	// "you to [verb]" — directing AI behavior, still a preference
	if strings.HasPrefix(after, "you to ") {
		return false
	}

	// "to [verb]" — check if verb is behavior-related
	if strings.HasPrefix(after, "to ") {
		parts := strings.Fields(after[3:])
		if len(parts) > 0 && behaviorVerbs[parts[0]] {
			return false // behavior verb → still a preference
		}
		return true // action verb → filter out as request
	}

	return false
}

func DetectPreference(prompt string) (string, bool) {
	lower := strings.ToLower(strings.TrimSpace(prompt))
	if lower == "" {
		return "", false
	}

	// Question filter: prompts ending with ? are questions, not preferences
	if strings.HasSuffix(lower, "?") {
		return "", false
	}

	for _, pat := range containsPatterns {
		if strings.Contains(lower, pat) {
			if isDesireToAction(lower, pat) {
				continue
			}
			cleaned := strings.TrimSpace(prompt)
			cleaned = strings.TrimRight(cleaned, ".!?")
			return cleaned, true
		}
	}
	for _, pat := range startPatterns {
		if strings.HasPrefix(lower, pat) {
			cleaned := strings.TrimSpace(prompt)
			cleaned = strings.TrimRight(cleaned, ".!?")
			return cleaned, true
		}
	}
	return "", false
}

func DetectCorrection(prompt string) bool {
	lower := strings.ToLower(strings.TrimSpace(prompt))
	correctionPatterns := []string{
		"try again",
		"that's wrong",
		"that is wrong",
		"that's not",
		"that is not",
		"not correct",
		"incorrect",
		"wrong ",
		"nope",
		"no,",
		"no i meant",
		"not what i",
		"i said ",
		"remember i said",
		"like i said",
		"as i said",
		"i already said",
		"i told you",
	}
	for _, pat := range correctionPatterns {
		if strings.Contains(lower, pat) {
			return true
		}
	}
	return false
}

func isValidName(candidate string) bool {
	if strings.ContainsAny(candidate, ".?!") {
		return false
	}
	words := strings.Fields(candidate)
	if len(words) == 0 || len(words) > 4 {
		return false
	}
	if nameStopwords[strings.ToLower(words[0])] {
		return false
	}
	return true
}

func DetectIdentity(prompt string) (string, bool) {
	lower := strings.ToLower(strings.TrimSpace(prompt))

	identityPatterns := []struct {
		prefix string
		strip  bool // true = prefix is followed by the name
	}{
		{"my name is ", true},
		{"i'm ", true},
		{"i am ", true},
		{"call me ", true},
		{"you can call me ", true},
		{"people call me ", true},
	}

	for _, pat := range identityPatterns {
		if strings.HasPrefix(lower, pat.prefix) {
			name := strings.TrimSpace(prompt[len(pat.prefix):])
			name = strings.TrimRight(name, ".!?,;")
			if name != "" && isValidName(name) {
				return name, true
			}
		}
	}
	return "", false
}

func DetectAIDesignation(prompt string) (string, bool) {
	lower := strings.ToLower(strings.TrimSpace(prompt))

	patterns := []string{
		"your name is ",
		"your designation is ",
		"i'll call you ",
		"i will call you ",
		"you are called ",
		"you go by ",
	}

	for _, pat := range patterns {
		if strings.HasPrefix(lower, pat) {
			designation := strings.TrimSpace(prompt[len(pat):])
			designation = strings.TrimRight(designation, ".!?,;")
			if designation != "" && isValidName(designation) {
				return designation, true
			}
		}
	}
	return "", false
}

func InferStyle(text string) PreferenceStyle {
	lower := strings.ToLower(text)
	for _, kw := range conciseKeywords {
		if strings.Contains(lower, kw) {
			return StyleConcise
		}
	}
	for _, kw := range detailedKeywords {
		if strings.Contains(lower, kw) {
			return StyleDetailed
		}
	}
	for _, kw := range examplesKeywords {
		if strings.Contains(lower, kw) {
			return StyleExamples
		}
	}
	return StyleGeneral
}

func PreferenceComplianceScore(prefs []Preference, response string) float32 {
	if len(prefs) == 0 {
		return 0.5
	}

	wordCount := len(strings.Fields(response))
	score := float32(0.5)
	matched := false

	for _, p := range prefs {
		switch p.Style {
		case StyleConcise:
			matched = true
			if wordCount <= 20 {
				score += 0.3
			} else if wordCount <= 50 {
				score += 0.1
			} else {
				score -= 0.3
			}
		case StyleDetailed:
			matched = true
			if wordCount >= 100 {
				score += 0.3
			} else if wordCount >= 50 {
				score += 0.1
			} else {
				score -= 0.3
			}
		case StyleExamples:
			matched = true
			lower := strings.ToLower(response)
			if strings.Contains(lower, "example") || strings.Contains(lower, "e.g.") ||
				strings.Contains(lower, "for instance") || strings.Contains(lower, "```") {
				score += 0.2
			} else {
				score -= 0.1
			}
		}
	}

	if !matched {
		return 0.5
	}
	return clamp(score)
}

func clamp(v float32) float32 {
	if v < 0 {
		return 0
	}
	if v > 1 {
		return 1
	}
	return v
}

func DetectRule(prompt string) bool {
	lower := strings.ToLower(strings.TrimSpace(prompt))
	for _, rp := range rulePatterns {
		if strings.Contains(lower, rp.pattern) {
			return true
		}
	}
	return false
}

func ExtractRule(prompt string) (string, string, bool) {
	lower := strings.ToLower(strings.TrimSpace(prompt))

	// Pattern: "when I say <trigger>, you say/respond with <response>"
	// Also: "if I say <trigger>, you say/respond with <response>"
	separators := []string{
		", you say ",
		", you respond with ",
		", you should say ",
		", you should respond with ",
		", respond with ",
		", reply with ",
		" you say ",
		" you respond with ",
		" respond with ",
	}
	prefixes := []string{
		"when i say ",
		"if i say ",
		"i say ",
	}

	for _, prefix := range prefixes {
		if !strings.HasPrefix(lower, prefix) {
			continue
		}
		rest := prompt[len(prefix):]
		restLower := lower[len(prefix):]

		for _, sep := range separators {
			idx := strings.Index(restLower, sep)
			if idx < 0 {
				continue
			}
			trigger := strings.TrimSpace(rest[:idx])
			response := strings.TrimSpace(rest[idx+len(sep):])
			// Clean up quotes and trailing punctuation
			trigger = strings.Trim(trigger, `"'`)
			respo

func FormatRulesBlock(rules []Rule) string {
	if len(rules) == 0 {
		return ""
	}
	var b strings.Builder
	b.WriteString("[BEHAVIORAL RULES]\n")
	b.WriteString("Follow these rules EXACTLY. They override all other behavior.\n")
	for _, r := range rules {
		b.WriteString(fmt.Sprintf("- If user says: %s → You respond with: %s\n", r.Trigger, r.Response))
	}
	return b.String()
}

func ProjectToPrompt(preferences []Preference, prefsNorm float32) string {
	if len(preferences) == 0 {
		return ""
	}

	// Confidence from prefs segment norm: 0 → no injection, >0.05 → inject
	// Exception: identity preferences always project regardless of norm
	confidence := float64(prefsNorm)
	hasIdentity := false
	for _, p := range preferences {
		if strings.HasPrefix(p.Text, "The user's name is") || strings.HasPrefix(p.Text, "The AI's designation is") {
			hasIdentity = true
			break
		}
	}
	if confidence < 0.05 && !hasIdentity {
		return ""
	}
	// Cap confidence display at 1.0
	if confidence > 1.0 {
		confidence = 1.0
	}

	var b strings.Builder
	b.WriteString("[ADAPTIVE STATE]\n")
	for _, p := range preferences {
		b.WriteString(fmt.Sprintf("- %s\n", p.Text))
	}
	b.WriteString(fmt.Sprintf("(confidence: %.0f%%)\n", math.Round(confidence*100)))
	return b.String()
}

func WrapPrompt(stateBlock, prompt string) string {
	if stateBlock == "" {
		return prompt
	}
	return stateBlock + "\n[USER PROMPT]\n" + prompt
}

func LoadFixture(path string) (*Fixture, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read fixture %s: %w", path, err)
	}
	var f Fixture
	if err := json.Unmarshal(data, &f); err != nil {
		return nil, fmt.Errorf("parse fixture %s: %w", path, err)
	}
	return &f, nil
}

func (s *FixtureStartState) ToStateRecord() state.StateRecord {
	return state.StateRecord{
		VersionID:   s.VersionID,
		StateVector: s.StateVector,
		SegmentMap:  s.SegmentMap,
	}
}

func (fi *FixtureInteraction) ToInteraction() Interaction {
	return Interaction{
		TurnID:       fi.TurnID,
		Prompt:       fi.Prompt,
		ResponseText: fi.ResponseText,
		Entropy:      fi.Entropy,
		Signals: update.Signals{
			SentimentScore:      fi.Signals.SentimentScore,
			NoveltyScore:        fi.Signals.NoveltyScore,
			CoherenceScore:      fi.Signals.CoherenceScore,
			RiskFlag:            fi.Signals.RiskFlag,
			UserCorrection:      fi.Signals.UserCorrection,
			ToolFailure:         fi.Signals.ToolFailure,
			ConstraintViolation: fi.Signals.ConstraintViolation,
		},
		Evidence: fi.Evidence,
	}
}

func (fc *FixtureConfig) ToReplayConfig() ReplayConfig {
	return ReplayConfig{
		UpdateConfig: update.UpdateConfig{
			LearningRate:           fc.UpdateConfig.LearningRate,
			DecayRate:              fc.UpdateConfig.DecayRate,
			MaxDeltaNormPerSegment: fc.UpdateConfig.MaxDeltaNormPerSegment,
		},
		GateConfig: gate.GateConfig{
			MaxDeltaNorm:   fc.GateConfig.MaxDeltaNorm,
			MaxStateNorm:   fc.GateConfig.MaxStateNorm,
			MinEntropyDrop: fc.GateConfig.MinEntropyDrop,
			RiskSegmentCap: fc.GateConfig.RiskSegmentCap,
		},
		EvalConfig: eval.EvalConfig{
			MaxStateNorm:    fc.EvalConfig.MaxStateNorm,
			MaxSegmentNorm:  fc.EvalConfig.MaxSegmentNorm,
			EntropyBaseline: fc.EvalConfig.EntropyBaseline,
		},
	}
}

func DefaultReplayConfig() ReplayConfig {
	return ReplayConfig{
		UpdateConfig: update.DefaultUpdateConfig(),
		GateConfig:   gate.DefaultGateConfig(),
		EvalConfig:   eval.DefaultEvalConfig(),
	}
}

func Replay(startState state.StateRecord, interactions []Interaction, config ReplayConfig) []ReplayResult {
	current := startState
	results := make([]ReplayResult, 0, len(interactions))

	gateInst := gate.NewGate(config.GateConfig)
	evalInst := eval.NewEvalHarness(config.EvalConfig)

	for _, inter := range interactions {
		ctx := update.UpdateContext{
			TurnID:       inter.TurnID,
			Prompt:       inter.Prompt,
			ResponseText: inter.ResponseText,
			Entropy:      inter.Entropy,
		}

		// 1. Update
		updateResult := update.Update(current, ctx, inter.Signals, inter.Evidence, config.UpdateConfig)

		// 2. No-op check
		if updateResult.Decision.Action == "no_op" {
			results = append(results, ReplayResult{
				TurnID:         inter.TurnID,
				Action:         "no_op",
				Reason:         updateResult.Decision.Reason,
				UpdateDecision: updateResult.Decision,
				UpdateMetrics:  updateResult.Metrics,
				FinalVersionID: current.VersionID,
			})
			continue
		}

		// 3. Gate
		gateDecisio

func Summarize(results []ReplayResult, finalState state.StateRecord) ReplaySummary {
	s := ReplaySummary{
		TotalTurns: len(results),
		FinalState: finalState,
	}
	for _, r := range results {
		switch r.Action {
		case "commit":
			s.Commits++
		case "gate_reject":
			s.GateRejects++
		case "eval_rollback":
			s.EvalRollbacks++
		case "no_op":
			s.NoOps++
		}
	}
	return s
}

func AdjustedThreshold(base float32, goalsNorm float32) float32 {
	reduction := goalsNorm * 0.05
	if reduction > 0.15 {
		reduction = 0.15
	}
	adjusted := base - reduction
	if adjusted < 0.1 {
		adjusted = 0.1
	}
	return adjusted
}

func (r *Retriever) Retrieve(ctx context.Context, prompt string, entropy float32) (GateResult, error) {
	result := GateResult{}

	// Gate 1: entropy check (skipped when AlwaysRetrieve is set)
	if !r.config.AlwaysRetrieve && entropy < r.config.EntropyThreshold {
		result.Gate1Passed = false
		result.Reason = fmt.Sprintf("gate1: entropy %.4f < threshold %.4f", entropy, r.config.EntropyThreshold)
		return result, nil
	}
	result.Gate1Passed = true

	// Gate 2: similarity search (threshold enforced server-side)
	searchResults, err := r.codec.Search(ctx, prompt, r.config.TopK, r.config.SimilarityThreshold)
	if err != nil {
		return result, fmt.Errorf("retrieval search: %w", err)
	}

	// Convert codec results to EvidenceRecords
	gate2Results := make([]EvidenceRecord, len(searchResults))
	for i, sr := range searchResults {
		gate2Results[i] = EvidenceRecord{
			ID:           sr.ID,
			Text:         sr.Text,
			Score:        sr.Score,
			MetadataJSON: sr.MetadataJSON,
		}
	}
	result.Gate2Count 

func (r *Retriever) consistencyCheck(results []EvidenceRecord) []EvidenceRecord {
	seen := make(map[string]bool)
	var valid []EvidenceRecord

	for _, rec := range results {
		// Skip empty text
		if rec.Text == "" {
			continue
		}
		// Skip overlong text
		if r.config.MaxEvidenceLen > 0 && len(rec.Text) > r.config.MaxEvidenceLen {
			continue
		}
		// Skip duplicate IDs
		if seen[rec.ID] {
			continue
		}
		seen[rec.ID] = true
		valid = append(valid, rec)
	}

	return valid
}

func (r *Retriever) topicCoherenceFilter(prompt string, results []EvidenceRecord) []EvidenceRecord {
	minShared := r.config.MinSharedKeywords
	if minShared <= 0 {
		minShared = 1
	}
	promptTokens := tokenize(prompt)
	if len(promptTokens) == 0 {
		// No content words in prompt — skip filter to avoid false rejections
		return results
	}

	var valid []EvidenceRecord
	for _, rec := range results {
		evidenceTokens := tokenize(rec.Text)
		if sharedKeywords(promptTokens, evidenceTokens) >= minShared {
			valid = append(valid, rec)
		}
	}
	return valid
}

func tokenize(text string) []string {
	words := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
		return !unicode.IsLetter(r)
	})
	seen := make(map[string]bool)
	var tokens []string
	for _, w := range words {
		if len(w) < 2 || stopwords[w] || seen[w] {
			continue
		}
		seen[w] = true
		tokens = append(tokens, w)
	}
	return tokens
}

func sharedKeywords(a, b []string) int {
	set := make(map[string]bool, len(a))
	for _, t := range a {
		set[t] = true
	}
	count := 0
	for _, t := range b {
		if set[t] {
			count++
		}
	}
	return count
}

func DefaultConfig() RetrievalConfig {
	return RetrievalConfig{
		AlwaysRetrieve:      true,
		EntropyThreshold:    0.5,
		SimilarityThreshold: 0.5,
		TopK:                5,
		MaxEvidenceLen:       2000,
		MinSharedKeywords:   1,
	}
}

func (p *Producer) Produce(ctx context.Context, input ProduceInput) update.Signals {
	return update.Signals{
		SentimentScore:      p.sentimentScore(input),
		CoherenceScore:      p.coherenceScore(ctx, input),
		NoveltyScore:        p.noveltyScore(input),
		RiskFlag:            p.riskFlag(input),
		UserCorrection:      input.UserCorrect,
		ToolFailure:         false,
		ConstraintViolation: false,
	}
}

func (p *Producer) sentimentScore(input ProduceInput) float32 {
	tokens := tokenize(input.ResponseText)
	if len(tokens) == 0 {
		return 0
	}
	unique := make(map[string]struct{}, len(tokens))
	for _, t := range tokens {
		unique[t] = struct{}{}
	}
	diversity := float32(len(unique)) / float32(len(tokens))
	confidence := 1.0 - clamp(input.Entropy)
	return clamp(diversity * confidence)
}

func (p *Producer) coherenceScore(ctx context.Context, input ProduceInput) float32 {
	if p.embedder == nil {
		return 0
	}
	promptEmb, err := p.embedder.Embed(ctx, input.Prompt)
	if err != nil {
		return 0
	}
	responseEmb, err := p.embedder.Embed(ctx, input.ResponseText)
	if err != nil {
		return 0
	}
	return clamp(cosineSimilarity(promptEmb, responseEmb))
}

func (p *Producer) noveltyScore(input ProduceInput) float32 {
	// Tier 1: retrieval-inverse
	if len(input.Retrieved) > 0 {
		var maxScore float32
		for _, ev := range input.Retrieved {
			if ev.Score > maxScore {
				maxScore = ev.Score
			}
		}
		return clamp(1 - maxScore)
	}
	// Tier 2: logit variance
	if len(input.Logits) > 0 {
		v := logitVariance(input.Logits)
		return clamp(float32(math.Tanh(float64(v))))
	}
	// Tier 3: entropy fallback
	return clamp(input.Entropy)
}

func cosineSimilarity(a, b []float32) float32 {
	if len(a) != len(b) || len(a) == 0 {
		return 0
	}
	var dot, normA, normB float64
	for i := range a {
		dot += float64(a[i]) * float64(b[i])
		normA += float64(a[i]) * float64(a[i])
		normB += float64(b[i]) * float64(b[i])
	}
	denom := math.Sqrt(normA) * math.Sqrt(normB)
	if denom == 0 {
		return 0
	}
	return float32(dot / denom)
}

func logitVariance(logits []float32) float32 {
	if len(logits) == 0 {
		return 0
	}
	var sum float64
	for _, v := range logits {
		sum += float64(v)
	}
	mean := sum / float64(len(logits))
	var variance float64
	for _, v := range logits {
		d := float64(v) - mean
		variance += d * d
	}
	return float32(variance / float64(len(logits)))
}