def ingest_document(doc_path, metadata):
    # Extract text preserving structure
    content = extract_with_structure(doc_path)

    # Smart chunking strategy
    chunks = smart_chunking(
        content,
        chunk_size=512,        # Tokens, not characters
        overlap=50,            # Overlap for context continuity
        respect_boundaries=True  # Don't split sentences/paragraphs
    )

    # Enrich with metadata for filtering and ranking
    for chunk in chunks:
        chunk.metadata = {
            **metadata,
            'source': doc_path,
            'chunk_id': chunk.id,
            'parent_doc_id': doc.id,
            'timestamp': now(),
            'version': doc.version,
            'access_level': doc.access_level  # For security
        }

    # Batch embed and store
    embeddings = embed_batch(chunks)
    vector_db.upsert(chunks, embeddings)

def retrieve(query, top_k=10, filters=None):
    # Parallel retrieval from multiple sources
    vector_results = vector_search(
        query,
        k=top_k * 2,
        filters=filters  # Pre-filter by metadata
    )

    keyword_results = bm25_search(
        query,
        k=top_k * 2,
        filters=filters
    )

    # Combine and deduplicate
    combined = merge_results(vector_results, keyword_results)

    # Re-rank using cross-encoder for final ranking
    reranked = cross_encoder_rerank(
        query,
        combined,
        top_k=top_k
    )

    return reranked

def assemble_context(query, chunks, max_tokens=4000):
    context_parts = []
    token_count = 0

    for i, chunk in enumerate(chunks):
        # Accurate token counting
        chunk_tokens = count_tokens(chunk.text)

        if token_count + chunk_tokens > max_tokens:
            break

        # Format with citation metadata
        context_parts.append(
            f"[Source {i+1}: {chunk.metadata['source']}, "
            f"Page {chunk.metadata.get('page', 'N/A')}]\n"
            f"{chunk.text}\n"
        )
        token_count += chunk_tokens

    return "\n---\n".join(context_parts)

system_prompt = """
You are an assistant that answers questions based on provided context.

CRITICAL RULES:
1. Only use information from the provided context
2. Cite sources using [Source X] format inline
3. If context doesn't contain the answer, explicitly say "I don't have enough information"
4. Do not make up information or hallucinate facts
5. When sources contradict, present both perspectives

Context:
{context}

Question: {query}

Provide a clear, concise answer with inline citations.
"""

def validate_response(response, retrieved_chunks):
    # Extract cited sources from response
    cited_sources = extract_citations(response)

    # Verify all citations exist in retrieved chunks
    valid_citations = all(
        source in retrieved_chunks for source in cited_sources
    )

    if not valid_citations:
        log_warning("Invalid citations detected")
        # Option: Regenerate or flag for review

    # Add clickable links to sources
    response_with_links = add_source_links(response, retrieved_chunks)

    return response_with_links

class RAGEvaluator:
    def evaluate(self, test_set):
        results = {
            'retrieval_metrics': {
                'precision_at_k': [],
                'recall_at_k': [],
                'mrr': []  # Mean Reciprocal Rank
            },
            'generation_metrics': {
                'answer_relevance': [],
                'answer_correctness': [],
                'citation_accuracy': [],
                'hallucination_rate': []
            },
            'operational_metrics': {
                'latency_p50': [],
                'latency_p95': [],
                'cost_per_query': [],
                'error_rate': []
            }
        }

        for example in test_set:
            # Measure retrieval quality
            retrieved = retrieve(example.query)
            results['retrieval_metrics']['precision_at_k'].append(
                precision_at_k(retrieved, example.relevant_docs, k=10)
            )

            # Measure generation quality
            answer = generate(example.query, retrieved)
            results['generation_metrics']['answer_relevance'].append(
                llm_as_judge(example.query, answer)
            )

            # Validate citations
            results['generation_metrics']['citation_accuracy'].append(
                validate_citations(answer, retrieved)
            )

        return aggregate_metrics(results)

def rag_pipeline(query):
    with tracer.start_span("rag_query") as span:
        span.set_attribute("query_length", len(query))

        # Retrieval phase
        with tracer.start_span("retrieval"):
            start = time.time()
            chunks = retrieve(query)
            retrieval_latency = time.time() - start

            span.set_attribute("num_chunks_retrieved", len(chunks))
            span.set_attribute("retrieval_latency_ms", retrieval_latency * 1000)

        # Generation phase
        with tracer.start_span("generation"):
            start = time.time()
            response = generate(query, chunks)
            generation_latency = time.time() - start

            span.set_attribute("response_length", len(response))
            span.set_attribute("generation_latency_ms", generation_latency * 1000)

        # Cost tracking
        embedding_cost = calculate_cost(len(query), model="embedding")
        llm_cost = calculate_cost(
            count_tokens(chunks) + len(response),
            model="llm"
        )
        total_cost = embedding_cost + llm_cost

        log_metrics({
            'total_latency': retrieval_latency + generation_latency,
            'cost_per_query': total_cost,
            'num_chunks': len(chunks)
        })

        return response

def test_add():
    assert add(2, 3) == 5  # ✅ Deterministic

def test_summarize():
    summary = llm.summarize(document)
    assert summary == ???  # ❌ What's the "correct" output?

class TestSetBuilder:
    def create_test_set(self, source='production', size=500):
        """
        Create representative test set from production data
        """
        # Sample diverse queries
        queries = self.sample_queries(
            source=source,
            size=size,
            strategy='stratified',  # Ensure diversity
            criteria={
                'query_length': ['short', 'medium', 'long'],
                'query_type': ['factual', 'analytical', 'creative'],
                'difficulty': ['easy', 'medium', 'hard']
            }
        )

        # Generate or collect ground truth
        test_examples = []
        for query in queries:
            example = {
                'input': query,
                'context': self.get_context(query),
                'expected_output': self.get_ground_truth(query),
                'metadata': self.classify_query(query)
            }
            test_examples.append(example)

        return test_examples

    def get_ground_truth(self, query):
        """
        Obtain reference answer
        """
        # Option 1: Human labeling
        if query.requires_expert:
            return human_labeler.label(query)

        # Option 2: Use production data (with human in loop)
        if query.has_positive_feedback:
            return production_db.get_response(query)

        # Option 3: Generate with best available model
        return gpt4.generate_reference(query)

test_set_composition = {
    'total': 500,

    # By query type
    'by_type': {
        'factual': 200,        # "What is X?"
        'analytical': 150,     # "Why does X happen?"
        'creative': 100,       # "Generate ideas for X"
        'procedural': 50,      # "How do I X?"
    },

    # By difficulty
    'by_difficulty': {
        'easy': 200,   # Clear answer, well-known topic
        'medium': 200, # Requires reasoning, less common
        'hard': 100,   # Complex, ambiguous, rare
    },

    # By expected failure modes
    'edge_cases': {
        'ambiguous_queries': 50,
        'out_of_scope': 25,
        'adversarial': 25,
        'multilingual': 25,
        'very_long_context': 25,
    }
}

golden_set = {
    'size': 50,  # Smaller, curated
    'quality': 'expert-labeled',
    'purpose': 'regression testing',
    'update_frequency': 'quarterly',

    # Run before every deployment
    'pass_threshold': {
        'accuracy': 0.85,
        'no_regressions': True,  # All previously passing must still pass
    }
}

class ReferencedMetrics:
    def exact_match(self, predicted, reference):
        """
        Exact string match (rarely useful for LLMs)
        """
        return predicted.strip() == reference.strip()

    def semantic_similarity(self, predicted, reference):
        """
        Embedding-based similarity
        """
        pred_emb = embed(predicted)
        ref_emb = embed(reference)
        return cosine_similarity(pred_emb, ref_emb)

    def rouge_score(self, predicted, reference):
        """
        Overlap-based metric (good for summarization)
        """
        from rouge import Rouge
        rouge = Rouge()
        scores = rouge.get_scores(predicted, reference)[0]

        return {
            'rouge-1': scores['rouge-1']['f'],  # Unigram overlap
            'rouge-2': scores['rouge-2']['f'],  # Bigram overlap
            'rouge-l': scores['rouge-l']['f'],  # Longest common subsequence
        }

    def bleu_score(self, predicted, reference):
        """
        N-gram precision (good for translation)
        """
        from nltk.translate.bleu_score import sentence_bleu
        reference_tokens = [reference.split()]
        predicted_tokens = predicted.split()
        return sentence_bleu(reference_tokens, predicted_tokens)

    def bertscore(self, predicted, reference):
        """
        Contextual embedding similarity
        """
        from bert_score import score
        P, R, F1 = score([predicted], [reference], lang='en')
        return F1.item()

class ReferenceFreeMetrics:
    def perplexity(self, text):
        """
        How "surprising" is the text?
        Lower = more fluent
        """
        return model.perplexity(text)

    def coherence_score(self, text):
        """
        Is the text logically consistent?
        """
        sentences = sent_tokenize(text)
        embeddings = [embed(s) for s in sentences]

        # Average similarity between consecutive sentences
        coherence = np.mean([
            cosine_similarity(embeddings[i], embeddings[i+1])
            for i in range(len(embeddings)-1)
        ])

        return coherence

    def toxicity_score(self, text):
        """
        Does the text contain harmful content?
        """
        return toxicity_classifier.predict(text)

    def factual_consistency(self, text, context):
        """
        Is the text consistent with the context?
        (For RAG applications)
        """
        # Use NLI model
        premise = context
        hypothesis = text
        result = nli_model.predict(premise, hypothesis)

        return result['entailment_score']

class RAGMetrics:
    def retrieval_precision_at_k(self, retrieved_docs, relevant_docs, k=10):
        """
        What fraction of retrieved docs are relevant?
        """
        retrieved_k = retrieved_docs[:k]
        relevant_retrieved = len(set(retrieved_k) & set(relevant_docs))
        return relevant_retrieved / k

    def retrieval_recall_at_k(self, retrieved_docs, relevant_docs, k=10):
        """
        What fraction of relevant docs were retrieved?
        """
        retrieved_k = retrieved_docs[:k]
        relevant_retrieved = len(set(retrieved_k) & set(relevant_docs))
        return relevant_retrieved / len(relevant_docs)

    def citation_accuracy(self, generated_text, cited_sources, retrieved_docs):
        """
        Are citations valid and accurate?
        """
        # Extract citations from text
        citations = extract_citations(generated_text)

        # Check if each citation exists
        valid = sum(1 for c in citations if c in retrieved_docs)

        return valid / len(citations) if citations else 0

    def answer_relevance(self, question, answer):
        """
        Does the answer address the question?
        """
        # Use sentence similarity
        q_emb = embed(question)
        a_emb = embed(answer)
        return cosine_similarity(q_emb, a_emb)

    def context_utilization(self, answer, context):
        """
        How much of the context was used?
        """
        # Find sentences in answer that appear in context
        answer_sents = sent_tokenize(answer)
        context_sents = sent_tokenize(context)

        used = sum(1 for a_sent in answer_sents
                  if any(similarity(a_sent, c_sent) > 0.8
                        for c_sent in context_sents))

        return used / len(answer_sents)

class LLMJudge:
    def __init__(self, judge_model='gpt-4'):
        self.judge = judge_model

    def evaluate_relevance(self, question, answer):
        """
        Is the answer relevant to the question?
        """
        prompt = f"""
        Evaluate if the answer is relevant to the question.

        Question: {question}
        Answer: {answer}

        Rate relevance on a scale of 1-5:
        1 - Completely irrelevant
        2 - Slightly relevant
        3 - Moderately relevant
        4 - Mostly relevant
        5 - Highly relevant

        Provide ONLY the number, nothing else.
        """

        score = self.judge.complete(prompt, temperature=0)
        return int(score.strip())

    def evaluate_correctness(self, question, answer, reference):
        """
        Is the answer factually correct?
        """
        prompt = f"""
        Evaluate if the answer is factually correct compared to the reference.

        Question: {question}
        Reference Answer: {reference}
        Generated Answer: {answer}

        Rate correctness on a scale of 1-5:
        1 - Completely incorrect
        2 - Mostly incorrect
        3 - Partially correct
        4 - Mostly correct
        5 - Completely correct

        Provide ONLY the number, nothing else.
        """

        score = self.judge.complete(prompt, temperature=0)
        return int(score.strip())

    def evaluate_with_reasoning(self, question, answer, criteria):
        """
        Get both score and explanation
        """
        prompt = f"""
        Evaluate the answer based on these criteria:
        {criteria}

        Question: {question}
        Answer: {answer}

        Provide your evaluation in this format:
        Score: [1-5]
        Reasoning: [Brief explanation]
        """

        response = self.judge.complete(prompt, temperature=0)

        # Parse response
        score = extract_score(response)
        reasoning = extract_reasoning(response)

        return {'score': score, 'reasoning': reasoning}

def comprehensive_evaluation(test_example):
    """
    Evaluate on all relevant dimensions
    """
    question = test_example['input']
    generated = generate_answer(question)
    reference = test_example['expected_output']
    context = test_example['context']

    scores = {
        # Factual accuracy
        'correctness': llm_judge.evaluate_correctness(
            question, generated, reference
        ),

        # Relevance
        'relevance': llm_judge.evaluate_relevance(
            question, generated
        ),

        # Completeness
        'completeness': llm_judge.evaluate_completeness(
            question, generated, reference
        ),

        # Coherence
        'coherence': coherence_score(generated),

        # Conciseness (length appropriateness)
        'conciseness': evaluate_length_appropriateness(generated),

        # Citation quality (for RAG)
        'citation_accuracy': citation_accuracy(
            generated, context
        ),

        # Safety
        'toxicity': toxicity_score(generated),

        # Semantic similarity to reference
        'similarity': semantic_similarity(generated, reference),

        # Performance
        'latency_ms': test_example['latency'],
        'cost_usd': test_example['cost'],
    }

    # Compute weighted overall score
    weights = {
        'correctness': 0.3,
        'relevance': 0.25,
        'completeness': 0.2,
        'coherence': 0.1,
        'conciseness': 0.05,
        'citation_accuracy': 0.1,
    }

    overall_score = sum(
        scores[metric] * weights[metric]
        for metric in weights
    )

    scores['overall'] = overall_score

    return scores

class HumanEvaluation:
    def create_evaluation_task(self, examples, evaluators):
        """
        Set up human evaluation
        """
        tasks = []

        for example in examples:
            task = {
                'question': example['input'],
                'answer_a': example['model_a_output'],
                'answer_b': example['model_b_output'],
                'evaluation_criteria': {
                    'correctness': 'Is the answer factually correct?',
                    'helpfulness': 'Would this help the user?',
                    'clarity': 'Is it easy to understand?',
                    'preference': 'Which answer is better overall?'
                }
            }
            tasks.append(task)

        # Distribute to evaluators
        return self.distribute_tasks(tasks, evaluators)

    def analyze_inter_rater_agreement(self, evaluations):
        """
        Check if human evaluators agree
        """
        from sklearn.metrics import cohen_kappa_score

        # Extract ratings from pairs of evaluators
        rater1 = [e['rater1_score'] for e in evaluations]
        rater2 = [e['rater2_score'] for e in evaluations]

        # Calculate agreement
        kappa = cohen_kappa_score(rater1, rater2)

        if kappa < 0.6:
            print("Warning: Low inter-rater agreement. Consider clarifying criteria.")

        return kappa

class EvaluationPipeline:
    def __init__(self, test_set, metrics):
        self.test_set = test_set
        self.metrics = metrics

    def run_evaluation(self, model_version):
        """
        Run complete evaluation
        """
        results = []

        for example in self.test_set:
            # Generate output
            start_time = time.time()
            output = model_version.generate(example['input'])
            latency = (time.time() - start_time) * 1000

            # Compute all metrics
            scores = {}
            for metric_name, metric_fn in self.metrics.items():
                scores[metric_name] = metric_fn(
                    predicted=output,
                    reference=example.get('expected_output'),
                    context=example.get('context'),
                    input=example['input']
                )

            scores['latency_ms'] = latency
            scores['cost_usd'] = estimate_cost(example['input'], output)

            results.append({
                'example': example,
                'output': output,
                'scores': scores
            })

        # Aggregate results
        return self.aggregate_results(results)

    def aggregate_results(self, results):
        """
        Compute summary statistics
        """
        aggregated = {}

        # Average scores across all examples
        for metric in results[0]['scores'].keys():
            values = [r['scores'][metric] for r in results]
            aggregated[metric] = {
                'mean': np.mean(values),
                'median': np.median(values),
                'std': np.std(values),
                'min': np.min(values),
                'max': np.max(values),
                'p95': np.percentile(values, 95),
            }

        # Identify failure cases
        aggregated['failures'] = [
            r for r in results
            if r['scores']['overall'] < 0.6
        ]

        return aggregated

class ABTest:
    def compare_models(self, model_a, model_b, test_set):
        """
        Statistical comparison of two models
        """
        # Run both models
        results_a = self.evaluate(model_a, test_set)
        results_b = self.evaluate(model_b, test_set)

        # Compare on each metric
        comparison = {}

        for metric in results_a['scores'].keys():
            scores_a = [r['scores'][metric] for r in results_a]
            scores_b = [r['scores'][metric] for r in results_b]

            # Paired t-test
            from scipy.stats import ttest_rel
            statistic, p_value = ttest_rel(scores_a, scores_b)

            # Effect size
            mean_a = np.mean(scores_a)
            mean_b = np.mean(scores_b)
            improvement = ((mean_b - mean_a) / mean_a) * 100

            comparison[metric] = {
                'model_a_mean': mean_a,
                'model_b_mean': mean_b,
                'improvement_pct': improvement,
                'p_value': p_value,
                'significant': p_value < 0.05
            }

        return comparison

    def recommend_winner(self, comparison, priorities):
        """
        Determine which model to deploy
        """
        # Weight metrics by priority
        weighted_score_a = 0
        weighted_score_b = 0

        for metric, priority in priorities.items():
            weighted_score_a += comparison[metric]['model_a_mean'] * priority
            weighted_score_b += comparison[metric]['model_b_mean'] * priority

        # Consider cost and latency
        if comparison['cost_usd']['improvement_pct'] < -20:  # 20% more expensive
            print("Warning: Model B is significantly more expensive")

        if comparison['latency_ms']['improvement_pct'] > 50:  # 50% slower
            print("Warning: Model B is significantly slower")

        # Make recommendation
        if weighted_score_b > weighted_score_a and comparison['correctness']['significant']:
            return 'model_b'
        return 'model_a'

evaluation_results = {
    'model': 'rag_v3',
    'test_set_size': 500,
    'evaluation_date': '2026-01-15',

    'metrics': {
        # Quality
        'correctness': {'mean': 0.87, 'p95': 0.95},
        'relevance': {'mean': 0.89, 'p95': 0.98},
        'completeness': {'mean': 0.82, 'p95': 0.92},
        'citation_accuracy': {'mean': 0.94, 'p95': 1.0},

        # Performance
        'latency_ms': {'mean': 1200, 'p95': 2800},
        'cost_per_query': {'mean': 0.032, 'p95': 0.085},

        # Safety
        'toxicity_rate': 0.002,  # 0.2%
        'pii_leakage_rate': 0.0,
    },

    'pass_rate': 0.84,  # 84% of queries scored > 0.7

    'failure_analysis': {
        'out_of_scope_queries': 38,
        'insufficient_context': 24,
        'ambiguous_questions': 18,
        'technical_errors': 12,
    },

    'comparison_to_baseline': {
        'correctness': '+8%',
        'latency': '-15%',
        'cost': '-22%',
    }
}

class InputGuardrails:
    def __init__(self):
        self.pii_detector = PIIDetector()
        self.content_moderator = ContentModerator()
        self.injection_detector = InjectionDetector()

    def validate_input(self, user_input, context):
        violations = []

        # 1. PII Detection
        pii_found = self.pii_detector.detect(user_input)
        if pii_found:
            violations.append({
                'type': 'PII_DETECTED',
                'severity': 'HIGH',
                'entities': pii_found,
                'action': 'REDACT'
            })
            user_input = self.pii_detector.redact(user_input)

        # 2. Content Moderation
        moderation = self.content_moderator.check(user_input)
        if moderation.flagged:
            violations.append({
                'type': 'CONTENT_VIOLATION',
                'severity': 'HIGH',
                'categories': moderation.categories,
                'action': 'BLOCK'
            })
            raise ContentPolicyViolation(moderation.categories)

        # 3. Prompt Injection Detection
        if self.injection_detector.is_injection(user_input):
            violations.append({
                'type': 'PROMPT_INJECTION',
                'severity': 'HIGH',
                'action': 'BLOCK'
            })
            raise PromptInjectionDetected()

        # 4. Data Classification Check
        if context.requires_approval and not context.approved:
            raise ApprovalRequired()

        # Log all violations
        if violations:
            log_security_event(violations)

        return user_input, violations

class OutputGuardrails:
    def validate_output(self, llm_output, context):
        checks = []

        # 1. Toxicity Check
        toxicity_score = self.toxicity_classifier(llm_output)
        if toxicity_score > 0.7:
            checks.append({
                'check': 'toxicity',
                'passed': False,
                'score': toxicity_score
            })
            return self.safe_fallback_response()

        # 2. Hallucination Detection (for RAG)
        if context.retrieved_docs:
            faithfulness = self.check_faithfulness(
                llm_output,
                context.retrieved_docs
            )
            if faithfulness < 0.6:
                checks.append({
                    'check': 'faithfulness',
                    'passed': False,
                    'score': faithfulness
                })
                llm_output = self.add_uncertainty_disclaimer(llm_output)

        # 3. PII Leakage
        if self.contains_pii(llm_output):
            checks.append({
                'check': 'pii_leakage',
                'passed': False
            })
            llm_output = self.redact_pii(llm_output)

        # 4. Citation Validation (for RAG)
        if '[Source:' in llm_output:
            valid_citations = self.validate_citations(
                llm_output,
                context.retrieved_docs
            )
            if not valid_citations:
                checks.append({
                    'check': 'citation_validity',
                    'passed': False
                })

        log_output_checks(checks)
        return llm_output

class AIAccessControl:
    RISK_LEVELS = {
        'HIGH': ['senior_leadership', 'legal', 'compliance'],
        'MEDIUM': ['team_lead', 'manager'],
        'LOW': ['all_employees']
    }

    def can_access(self, user, application):
        # Check role-based access
        required_roles = self.RISK_LEVELS.get(
            application.risk_level,
            ['all_employees']
        )

        if not any(role in user.roles for role in required_roles):
            log_access_denied(user, application)
            return False

        # Check if user completed AI training
        if not user.completed_ai_training:
            return False

        # Check rate limits
        if self.exceeds_rate_limit(user):
            return False

        log_access_granted(user, application)
        return True

    def exceeds_rate_limit(self, user):
        usage = get_user_usage(user.id, last_24_hours=True)
        limits = {
            'requests_per_day': 1000,
            'tokens_per_day': 100000,
            'cost_per_day': 50.00
        }

        return (
            usage.requests > limits['requests_per_day'] or
            usage.tokens > limits['tokens_per_day'] or
            usage.cost > limits['cost_per_day']
        )

class AIAuditLogger:
    def log_request(self, request):
        """
        Log every AI request for audit purposes
        """
        audit_record = {
            'timestamp': datetime.now().isoformat(),
            'request_id': request.id,

            # User info
            'user_id': request.user.id,
            'user_email': request.user.email,
            'user_role': request.user.role,

            # Application info
            'application': request.application.name,
            'risk_level': request.application.risk_level,

            # Request details
            'input_text': request.input[:500],  # Truncate for storage
            'input_tokens': request.input_tokens,
            'model': request.model,
            'prompt_version': request.prompt_version,

            # Response details
            'output_text': request.output[:500],
            'output_tokens': request.output_tokens,
            'latency_ms': request.latency_ms,
            'cost_usd': request.cost,

            # Safety checks
            'input_violations': request.input_violations,
            'output_checks': request.output_checks,

            # Metadata
            'ip_address': request.ip_address,
            'user_agent': request.user_agent
        }

        # Store in audit database
        audit_db.insert(audit_record)

        # Check for anomalies
        self.detect_anomalies(audit_record)

    def detect_anomalies(self, record):
        """
        Detect unusual patterns
        """
        # High token usage
        if record['input_tokens'] + record['output_tokens'] > 10000:
            alert('HIGH_TOKEN_USAGE', record)

        # Repeated violations
        user_violations = audit_db.count_violations(
            record['user_id'],
            last_7_days=True
        )
        if user_violations > 5:
            alert('REPEATED_VIOLATIONS', record)

        # Unusual access patterns
        if self.is_unusual_access(record):
            alert('UNUSUAL_ACCESS_PATTERN', record)

def generate_compliance_report(start_date, end_date):
    """
    Generate report for compliance teams
    """
    data = audit_db.query(start_date, end_date)

    report = {
        'period': f"{start_date} to {end_date}",

        'usage_summary': {
            'total_requests': len(data),
            'unique_users': len(set(r['user_id'] for r in data)),
            'applications_used': len(set(r['application'] for r in data)),
            'total_cost': sum(r['cost_usd'] for r in data)
        },

        'risk_breakdown': {
            'high_risk_requests': sum(1 for r in data if r['risk_level'] == 'HIGH'),
            'medium_risk_requests': sum(1 for r in data if r['risk_level'] == 'MEDIUM'),
            'low_risk_requests': sum(1 for r in data if r['risk_level'] == 'LOW')
        },

        'violations': {
            'pii_detected': sum(1 for r in data if 'PII_DETECTED' in r['input_violations']),
            'content_violations': sum(1 for r in data if 'CONTENT_VIOLATION' in r['input_violations']),
            'injection_attempts': sum(1 for r in data if 'PROMPT_INJECTION' in r['input_violations'])
        },

        'data_handling': {
            'pii_processed': count_pii_processed(data),
            'external_api_calls': sum(1 for r in data if r['model'].startswith('gpt-')),
            'private_deployments': sum(1 for r in data if 'azure' in r['model'])
        },

        'top_users': get_top_users_by_usage(data, limit=10),
        'top_applications': get_top_applications(data, limit=10)
    }

    return report

governance_metrics = {
    # Risk Management
    'incidents_per_month': 2,  # Target: < 5
    'mean_time_to_detect': 15,  # minutes, Target: < 30
    'mean_time_to_resolve': 120,  # minutes, Target: < 180

    # Compliance
    'policy_violations_per_1000_requests': 0.5,  # Target: < 1
    'audit_findings': 0,  # Target: 0 critical findings
    'training_completion_rate': 0.95,  # Target: > 90%

    # Adoption
    'approved_applications': 15,
    'active_users': 2500,
    'user_satisfaction': 4.2,  # Target: > 4.0

    # Efficiency
    'approval_turnaround_time': 5,  # days, Target: < 7
    'false_positive_rate': 0.03,  # Target: < 5%
}

class CostTracker:
    PRICING = {
        'gpt-4': {
            'input': 0.03,   # per 1K tokens
            'output': 0.06
        },
        'gpt-4-turbo': {
            'input': 0.01,
            'output': 0.03
        },
        'gpt-3.5-turbo': {
            'input': 0.0005,
            'output': 0.0015
        },
        'text-embedding-3-small': {
            'input': 0.00002,
            'output': 0
        }
    }

    def calculate_cost(self, model, input_tokens, output_tokens):
        pricing = self.PRICING[model]
        cost = (
            (input_tokens / 1000) * pricing['input'] +
            (output_tokens / 1000) * pricing['output']
        )
        return cost

    def analyze_request(self, request_log):
        breakdown = {
            'embedding': 0,
            'retrieval': 0,
            'generation': 0,
            'total': 0
        }

        # Embedding cost
        breakdown['embedding'] = self.calculate_cost(
            'text-embedding-3-small',
            request_log.query_tokens,
            0
        )

        # Generation cost
        breakdown['generation'] = self.calculate_cost(
            request_log.model,
            request_log.prompt_tokens,
            request_log.completion_tokens
        )

        breakdown['total'] = sum(breakdown.values())
        return breakdown

class ModelRouter:
    def __init__(self):
        self.cheap_model = 'gpt-3.5-turbo'
        self.expensive_model = 'gpt-4'

    def classify_complexity(self, query):
        """
        Classify query complexity using heuristics or a small classifier
        """
        signals = {
            'length': len(query.split()),
            'has_code': '```' in query or 'code' in query.lower(),
            'technical_terms': self.count_technical_terms(query),
            'requires_reasoning': any(kw in query.lower()
                for kw in ['why', 'how', 'explain', 'compare'])
        }

        # Simple scoring
        complexity_score = (
            signals['length'] / 100 +
            signals['has_code'] * 2 +
            signals['technical_terms'] * 0.5 +
            signals['requires_reasoning'] * 1
        )

        return 'complex' if complexity_score > 3 else 'simple'

    def route(self, query):
        complexity = self.classify_complexity(query)

        if complexity == 'simple':
            return self.cheap_model
        return self.expensive_model

import joblib
from sklearn.ensemble import RandomForestClassifier

class MLModelRouter:
    def __init__(self):
        self.classifier = joblib.load('model_router.pkl')
        self.vectorizer = joblib.load('vectorizer.pkl')

    def train(self, historical_queries):
        """
        Train on past queries labeled by whether
        GPT-4 performed better than GPT-3.5
        """
        X = self.vectorizer.fit_transform([
            q.text for q in historical_queries
        ])
        y = [
            q.needed_gpt4  # Binary: did this query need GPT-4?
            for q in historical_queries
        ]

        self.classifier.fit(X, y)
        joblib.dump(self.classifier, 'model_router.pkl')

    def route(self, query):
        X = self.vectorizer.transform([query])
        needs_gpt4 = self.classifier.predict(X)[0]

        return 'gpt-4' if needs_gpt4 else 'gpt-3.5-turbo'

prompt = f"""
You are a helpful assistant. You should answer questions helpfully.
Be helpful and provide good answers. Make sure your answers are helpful.

Question: {query}

Please provide a helpful answer:
"""
# Token count: ~50

prompt = f"""
Answer this question clearly and accurately.

Question: {query}

Answer:
"""
# Token count: ~20

def compress_context(chunks, max_tokens=2000):
    """
    Intelligently compress retrieved context
    """
    compressed_chunks = []
    token_count = 0

    for chunk in sorted(chunks, key=lambda c: c.relevance_score, reverse=True):
        # Remove redundant sentences
        chunk_text = remove_redundant_sentences(chunk.text)

        # Extract key sentences if still too long
        if token_count + estimate_tokens(chunk_text) > max_tokens:
            chunk_text = extract_key_sentences(
                chunk_text,
                budget=max_tokens - token_count
            )

        if token_count + estimate_tokens(chunk_text) <= max_tokens:
            compressed_chunks.append(chunk_text)
            token_count += estimate_tokens(chunk_text)
        else:
            break

    return "\n\n".join(compressed_chunks)

def llm_compress(long_context, budget_tokens):
    """
    Use cheap model to compress context for expensive model
    """
    compression_prompt = f"""
    Compress this text to ~{budget_tokens} tokens while retaining all key information.

    Text:
    {long_context}

    Compressed version:
    """

    compressed = gpt_3_5_turbo.complete(
        compression_prompt,
        max_tokens=budget_tokens
    )

    return compressed

class SemanticCache:
    def __init__(self, similarity_threshold=0.95):
        self.cache = {}  # {embedding: response}
        self.threshold = similarity_threshold

    def get(self, query):
        query_embedding = embed(query)

        # Check for similar queries
        for cached_embedding, response in self.cache.items():
            similarity = cosine_similarity(query_embedding, cached_embedding)

            if similarity >= self.threshold:
                return response

        return None

    def set(self, query, response):
        query_embedding = embed(query)
        self.cache[query_embedding] = response

class TieredCache:
    def __init__(self):
        self.exact_match = {}  # Redis: O(1) lookup
        self.semantic = SemanticCache()  # Approximate matches
        self.popular = {}  # Most frequent queries

    def get(self, query):
        # 1. Exact match (fastest, ~1ms)
        if query in self.exact_match:
            return self.exact_match[query]

        # 2. Semantic match (~10ms)
        semantic_match = self.semantic.get(query)
        if semantic_match:
            return semantic_match

        # 3. Popular queries (pre-computed)
        canonical_form = self.canonicalize(query)
        if canonical_form in self.popular:
            return self.popular[canonical_form]

        return None

    def set(self, query, response):
        self.exact_match[query] = response
        self.semantic.set(query, response)

        # Track popularity
        self.increment_popularity(query)

def adaptive_retrieval(query, min_chunks=3, max_chunks=10):
    """
    Retrieve more chunks only if needed
    """
    chunks = retrieve(query, k=min_chunks)

    # Check if we have enough information
    confidence = estimate_confidence(query, chunks)

    if confidence < 0.7 and len(chunks) < max_chunks:
        # Retrieve more
        chunks = retrieve(query, k=min_chunks * 2)
        confidence = estimate_confidence(query, chunks)

    return chunks

def estimate_confidence(query, chunks):
    """
    Estimate if chunks contain sufficient information
    """
    # Use a small model to assess coverage
    assessment_prompt = f"""
    Question: {query}

    Available information:
    {summarize_chunks(chunks)}

    Can this information answer the question? (yes/no)
    """

    response = cheap_model.complete(assessment_prompt)
    return 1.0 if 'yes' in response.lower() else 0.3

def deduplicate_chunks(chunks):
    """
    Remove redundant information from retrieved chunks
    """
    seen_content = set()
    unique_chunks = []

    for chunk in chunks:
        # Create fingerprint (sentence-level)
        sentences = sent_tokenize(chunk.text)
        fingerprint = frozenset(
            sentence.lower().strip()
            for sentence in sentences
        )

        # Check overlap
        overlap = len(fingerprint & seen_content) / len(fingerprint)

        if overlap < 0.5:  # Less than 50% overlap
            unique_chunks.append(chunk)
            seen_content.update(fingerprint)

    return unique_chunks

class BatchProcessor:
    def __init__(self, batch_size=10, max_wait_ms=100):
        self.batch_size = batch_size
        self.max_wait_ms = max_wait_ms
        self.queue = []

    async def process(self, query):
        """
        Add query to batch and wait for batch completion
        """
        future = asyncio.Future()
        self.queue.append((query, future))

        # Trigger batch if full
        if len(self.queue) >= self.batch_size:
            await self._process_batch()

        # Or wait for timeout
        try:
            return await asyncio.wait_for(
                future,
                timeout=self.max_wait_ms / 1000
            )
        except asyncio.TimeoutError:
            await self._process_batch()
            return await future

    async def _process_batch(self):
        if not self.queue:
            return

        batch = self.queue[:self.batch_size]
        self.queue = self.queue[self.batch_size:]

        # Create single prompt for batch
        batch_prompt = self.create_batch_prompt([q for q, _ in batch])

        # Single API call
        response = await llm.complete_async(batch_prompt)

        # Parse and distribute results
        results = self.parse_batch_response(response)

        for (query, future), result in zip(batch, results):
            future.set_result(result)

    def create_batch_prompt(self, queries):
        return f"""
        Answer these questions:

        1. {queries[0]}
        2. {queries[1]}
        ...

        Provide answers in order:
        1. [Answer to question 1]
        2. [Answer to question 2]
        ...
        """

def stream_with_early_stop(prompt, stop_conditions):
    """
    Stream tokens and stop when conditions are met
    """
    buffer = ""

    for token in llm.stream(prompt):
        buffer += token

        # Check stop conditions
        if any(condition(buffer) for condition in stop_conditions):
            break

    return buffer

# Example stop conditions
def has_complete_answer(text):
    """Stop if we have a complete answer"""
    # Look for conclusion markers
    return any(marker in text.lower() for marker in [
        'in summary',
        'in conclusion',
        'therefore',
    ]) and len(text) > 200

def has_citation(text):
    """Stop if we found a citation"""
    return '[Source:' in text

# Prepare training data
training_data = [
    {
        "messages": [
            {"role": "system", "content": "You extract action items from meetings."},
            {"role": "user", "content": meeting_transcript},
            {"role": "assistant", "content": extracted_action_items}
        ]
    }
    for meeting_transcript, extracted_action_items in labeled_data
]

# Fine-tune
fine_tuned_model = openai.FineTune.create(
    training_file=upload_training_data(training_data),
    model="gpt-3.5-turbo"
)

# Use fine-tuned model
response = openai.ChatCompletion.create(
    model=fine_tuned_model.id,
    messages=[
        {"role": "user", "content": new_meeting_transcript}
    ]
)

# Metrics to track
metrics = {
    # Costs
    'cost_total': 3500,
    'cost_per_query': 0.0175,
    'cost_by_model': {
        'gpt-4': 2100,
        'gpt-3.5-turbo': 1200,
        'fine-tuned': 200
    },

    # Efficiency
    'cache_hit_rate': 0.42,
    'avg_input_tokens': 1800,
    'avg_output_tokens': 300,

    # Quality
    'avg_quality_score': 4.2,
    'user_satisfaction': 4.3,

    # Volume
    'total_queries': 200000,
    'queries_per_day': 6700,
}

# Alert thresholds
alerts = {
    'daily_cost_exceeds': 150,
    'cost_per_query_exceeds': 0.02,
    'cache_hit_rate_below': 0.35,
    'quality_score_below': 4.0
}

def self_consistent_answer(question, n=5):
    answers = []

    for _ in range(n):
        response = llm.complete(
            f"Answer this question: {question}",
            temperature=0.7
        )
        answers.append(response)

    # Use LLM to synthesize the most consistent answer
    synthesis_prompt = f"""
    Here are {n} different answers to the same question:

    {format_answers(answers)}

    Identify the most consistent answer or synthesize the best answer from these responses.
    """

    return llm.complete(synthesis_prompt, temperature=0)

prompt = """
Problem: {problem}

Generate 3 different approaches to solve this problem:

Approach 1:
[Description of first approach]
Pros:
Cons:

Approach 2:
[Description of second approach]
Pros:
Cons:

Approach 3:
[Description of third approach]
Pros:
Cons:

Based on the analysis, which approach is best and why?
"""

# First draft
initial_prompt = """
Write a technical blog post about {topic}.
"""

draft = llm.complete(initial_prompt)

# Self-critique
critique_prompt = f"""
You wrote this blog post:

{draft}

Critique it according to these criteria:
1. Technical accuracy
2. Clarity for the target audience
3. Logical flow
4. Missing important points

Provide specific suggestions for improvement.
"""

critique = llm.complete(critique_prompt)

# Revision
revision_prompt = f"""
Original blog post:
{draft}

Critique:
{critique}

Revise the blog post addressing the critique.
"""

final = llm.complete(revision_prompt)