category_breakdown_analysis.py

#!/usr/bin/env python3
"""
Category Breakdown Analysis for LLM-LLM Configuration

Analyzes freelancer distribution, job distribution, and trends by skill category
in the LLM-LLM marketplace simulation.
"""

import json
from collections import defaultdict, Counter
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
import plotly.graph_objects as go
from src.marketplace.market_metrics_utils import calculate_freelancer_tier

# Output directory for analysis results
OUTPUT_DIR = Path("analysis_results/category_breakdown")
REPUTATION_TIER_CHART_FILENAME = "reputation_tier_distribution.png"


def load_simulation_data(file_path):
    """Load simulation data from JSON file."""
    try:
        with open(file_path, 'r') as f:
            simulation_data = json.load(f)
        print(f"✅ Loaded simulation data: {file_path}")
        return simulation_data
    except FileNotFoundError:
        print(f"❌ File not found: {file_path}")
        return None


def extract_clients_by_category(all_jobs):
    """Extract client information organized by job category."""
    clients_by_category = defaultdict(set)
    for job in all_jobs:
        client_id = job.get('client_id', 'unknown')
        job_category = job.get('category', 'Unknown')
        clients_by_category[job_category].add(client_id)
    
    all_clients = set()
    for client_set in clients_by_category.values():
        all_clients.update(client_set)
    
    return clients_by_category, all_clients


def print_data_overview(freelancers, all_clients, all_jobs, all_bids, hiring_outcomes, round_data):
    """Print overview of simulation data."""
    print("📊 Data Overview:")
    print(f"   - Freelancers: {len(freelancers)}")
    print(f"   - Clients: {len(all_clients)}")
    print(f"   - Jobs: {len(all_jobs)}")
    print(f"   - Bids: {len(all_bids)}")
    print(f"   - Hiring outcomes: {len(hiring_outcomes)}")
    print(f"   - Rounds: {len(round_data)}")


def analyze_freelancer_distribution(freelancers):
    """Analyze and return freelancer distribution by category."""
    freelancer_categories = defaultdict(list)
    freelancer_category_counts = Counter()
    
    for freelancer_id, freelancer in freelancers.items():
        freelancer_category = freelancer.get('category', 'Unknown')
        freelancer_categories[freelancer_category].append(freelancer_id)
        freelancer_category_counts[freelancer_category] += 1
    
    return freelancer_categories, freelancer_category_counts


def print_freelancer_distribution(freelancer_category_counts, total_freelancers):
    """Print freelancer distribution by category."""
    print("\n🧑‍💼 FREELANCER DISTRIBUTION BY CATEGORY")
    print("-" * 60)
    
    for category, count in freelancer_category_counts.most_common():
        percentage = (count / total_freelancers) * 100
        print(f"   {category:<25}: {count:3d} freelancers ({percentage:5.1f}%)")


def analyze_job_distribution(all_jobs):
    """Analyze and return job distribution by category."""
    job_categories = defaultdict(list)
    job_category_counts = Counter()
    
    for job in all_jobs:
        job_category = job.get('category', 'Unknown')
        job_categories[job_category].append(job['id'])
        job_category_counts[job_category] += 1
    
    return job_categories, job_category_counts


def print_job_distribution(job_category_counts, total_jobs):
    """Print job distribution by category."""
    print("\n💼 JOB DISTRIBUTION BY CATEGORY")
    print("-" * 60)
    
    for category, count in job_category_counts.most_common():
        percentage = (count / total_jobs) * 100
        print(f"   {category:<25}: {count:3d} jobs ({percentage:5.1f}%)")


def print_client_distribution(clients_by_category, all_clients):
    """Print client distribution by category."""
    print("\n🏢 CLIENT DISTRIBUTION BY CATEGORY")
    print("-" * 60)
    
    client_category_counts = Counter()
    for category, client_set in clients_by_category.items():
        client_category_counts[category] = len(client_set)
    
    for category, count in client_category_counts.most_common():
        percentage = (count / len(all_clients)) * 100
        print(f"   {category:<25}: {count:3d} clients ({percentage:5.1f}%)")


def group_bids_by_category(all_jobs, all_bids):
    """Group bids by job category."""
    bids_by_job_category = defaultdict(list)
    for bid in all_bids:
        job_id = bid.get('job_id')
        job_category = None
        for job in all_jobs:
            if job['id'] == job_id:
                job_category = job.get('category', 'Unknown')
                break
        if job_category:
            bids_by_job_category[job_category].append(bid)
    
    return bids_by_job_category


def calculate_fill_rate(category, all_jobs, hiring_outcomes, jobs_in_category):
    """Calculate fill rate for a specific category."""
    filled_job_ids = set()
    for outcome in hiring_outcomes:
        if outcome.get('selected_freelancer') not in [None, 'none']:
            job_id = outcome.get('job_id')
            for job in all_jobs:
                if job['id'] == job_id and job.get('category') == category:
                    filled_job_ids.add(job_id)
                    break
    
    filled_jobs = len(filled_job_ids)
    fill_rate = (filled_jobs / jobs_in_category) * 100 if jobs_in_category > 0 else 0
    
    return filled_jobs, fill_rate


def analyze_bidding_patterns(all_jobs, all_bids, job_category_counts, hiring_outcomes):
    """Analyze bidding patterns and metrics by category."""
    bids_by_job_category = group_bids_by_category(all_jobs, all_bids)
    
    category_metrics = {}
    for category in job_category_counts.keys():
        jobs_in_category = job_category_counts[category]
        bids_in_category = len(bids_by_job_category[category])
        avg_bids_per_job = bids_in_category / jobs_in_category if jobs_in_category > 0 else 0
        
        filled_jobs, fill_rate = calculate_fill_rate(category, all_jobs, hiring_outcomes, jobs_in_category)
        
        category_metrics[category] = {
            'jobs': jobs_in_category,
            'bids': bids_in_category,
            'avg_bids_per_job': avg_bids_per_job,
            'fill_rate': fill_rate,
            'filled_jobs': filled_jobs
        }
    
    return category_metrics


def print_bidding_patterns(category_metrics):
    """Print bidding patterns by category."""
    print("\n🎯 BIDDING PATTERNS BY CATEGORY")
    print("-" * 60)
    print(f"{'Category':<25} {'Jobs':<6} {'Bids':<6} {'Avg Bids/Job':<12} {'Fill Rate':<10}")
    print("-" * 70)
    
    for category, metrics in category_metrics.items():
        print(f"{category:<25} {metrics['jobs']:<6} {metrics['bids']:<6} "
              f"{metrics['avg_bids_per_job']:<12.2f} {metrics['fill_rate']:<10.1f}%")


def analyze_freelancer_success(freelancers, hiring_outcomes):
    """Analyze freelancer success rates by category."""
    hired_freelancers = set()
    freelancer_job_wins = defaultdict(int)
    
    for outcome in hiring_outcomes:
        selected = outcome.get('selected_freelancer')
        if selected and selected != 'none':
            hired_freelancers.add(selected)
            freelancer_job_wins[selected] += 1
    
    freelancer_success = defaultdict(lambda: {'total': 0, 'hired': 0, 'jobs_won': 0})
    for fid, freelancer in freelancers.items():
        cat = freelancer.get('category', 'Unknown')
        freelancer_success[cat]['total'] += 1
        if fid in hired_freelancers:
            freelancer_success[cat]['hired'] += 1
            freelancer_success[cat]['jobs_won'] += freelancer_job_wins[fid]
    
    return freelancer_success


def print_freelancer_success(freelancer_success):
    """Print freelancer success rates by category."""
    print("\n🏆 FREELANCER SUCCESS BY THEIR ACTUAL CATEGORY")
    print("-" * 80)
    print(f"{'Freelancer Category':<30} {'Total':<6} {'Hired':<6} {'Success Rate':<12} {'Jobs Won':<10}")
    print("-" * 80)
    
    for cat, metrics in sorted(freelancer_success.items()):
        total = metrics['total']
        hired = metrics['hired']
        success_rate = (hired / total) * 100 if total > 0 else 0
        jobs_won = metrics['jobs_won']
        print(f'{cat:<30} {total:<6} {hired:<6} {success_rate:<12.1f}% {jobs_won:<10}')


def run_additional_analyses(simulation_data, all_jobs, all_bids, freelancers, hiring_outcomes,
                            job_categories, job_category_counts):
    """Run all additional market analyses."""
    analyze_temporal_trends_by_category(all_jobs)
    analyze_cross_category_hiring(hiring_outcomes, all_jobs, freelancers)
    analyze_competition_patterns(all_jobs, all_bids, job_category_counts)
    analyze_cross_category_flows(hiring_outcomes, all_jobs, freelancers)
    analyze_reputation_tiers(simulation_data)
    analyze_rejection_patterns_with_charts(hiring_outcomes, all_bids)
    
    print("\n" + "="*60)
    print("REPUTATION SYSTEM IMPACT ANALYSIS")
    print("="*60)
    
    analyze_reputation_impact(simulation_data, OUTPUT_DIR)


def analyze_llm_llm_by_category():
    """Comprehensive category-based analysis of LLM-LLM configuration."""
    
    print("=" * 80)
    print("CATEGORY BREAKDOWN ANALYSIS - LLM-LLM CONFIGURATION")
    print("=" * 80)
    
    # Load data
    llm_llm_file = "results/simuleval/true_gpt_simulation_20250905_001404.json"
    simulation_data = load_simulation_data(llm_llm_file)
    if not simulation_data:
        return
    
    # Extract data components
    freelancers = simulation_data.get('freelancer_profiles', {})
    all_jobs = simulation_data.get('all_jobs', [])
    all_bids = simulation_data.get('all_bids', [])
    hiring_outcomes = simulation_data.get('hiring_outcomes', [])
    round_data = simulation_data.get('round_data', [])
    
    # Process clients
    clients_by_category, all_clients = extract_clients_by_category(all_jobs)
    
    # Print overview
    print_data_overview(freelancers, all_clients, all_jobs, all_bids, hiring_outcomes, round_data)
    
    # Analyze and print distributions
    freelancer_categories, freelancer_category_counts = analyze_freelancer_distribution(freelancers)
    print_freelancer_distribution(freelancer_category_counts, len(freelancers))
    
    job_categories, job_category_counts = analyze_job_distribution(all_jobs)
    print_job_distribution(job_category_counts, len(all_jobs))
    
    print_client_distribution(clients_by_category, all_clients)
    
    # Analyze and print bidding patterns
    category_metrics = analyze_bidding_patterns(all_jobs, all_bids, job_category_counts, hiring_outcomes)
    print_bidding_patterns(category_metrics)
    
    # Analyze and print freelancer success
    freelancer_success = analyze_freelancer_success(freelancers, hiring_outcomes)
    print_freelancer_success(freelancer_success)
    
    # Temporal trends and cross-category analysis
    print("\n📈 TEMPORAL TRENDS BY CATEGORY")
    print("-" * 60)
    
    print("\n🔄 CROSS-CATEGORY HIRING PATTERNS")
    print("-" * 60)
    
    # Generate visualizations
    create_category_visualizations(
        freelancer_category_counts,
        job_category_counts,
        category_metrics
    )
    
    create_sankey_flow_diagram(hiring_outcomes, all_jobs, freelancers)
    
    # Run additional analyses
    run_additional_analyses(
        simulation_data, all_jobs, all_bids, freelancers, hiring_outcomes,
        job_categories, job_category_counts
    )
    
    print("\n✅ Category breakdown analysis complete!")
    print(f"📊 Visualizations saved to: {OUTPUT_DIR}/")


def analyze_temporal_trends_by_category(all_jobs):
    """Analyze how different categories perform over time"""
    
    # Group jobs by round and category
    jobs_by_round_category = defaultdict(lambda: defaultdict(int))
    
    for job in all_jobs:
        job_round = job.get('round', 1)
        job_category = job.get('category', 'Unknown')
        jobs_by_round_category[job_round][job_category] += 1
    
    # Show trends for top 5 categories
    top_categories = Counter()
    for round_jobs in jobs_by_round_category.values():
        for category, count in round_jobs.items():
            top_categories[category] += count
    
    print("Job posting trends for top 5 categories:")
    print(f"{'Round':<6}", end='')
    top_5_categories = [cat for cat, _ in top_categories.most_common(5)]
    for category in top_5_categories:
        print(f"{category[:15]:<16}", end='')
    print()
    print("-" * (6 + 16 * 5))
    
    # Show first 10 rounds
    for round_num in sorted(jobs_by_round_category.keys())[:10]:
        print(f"{round_num:<6}", end='')
        for category in top_5_categories:
            count = jobs_by_round_category[round_num][category]
            print(f"{count:<16}", end='')
        print()

def collect_cross_category_hires(hiring_outcomes, all_jobs, freelancers):
    """Collect hiring data across job and freelancer categories."""
    cross_hires = defaultdict(lambda: defaultdict(int))
    
    for outcome in hiring_outcomes:
        selected = outcome.get('selected_freelancer')
        if selected and selected != 'none':
            job_id = outcome.get('job_id')
            
            # Find job category
            job_cat = None
            for job in all_jobs:
                if job['id'] == job_id:
                    job_cat = job.get('category', 'Unknown')
                    break
            
            # Find freelancer category
            freelancer_cat = freelancers.get(selected, {}).get('category', 'Unknown')
            
            if job_cat and freelancer_cat:
                cross_hires[job_cat][freelancer_cat] += 1
    
    return cross_hires


def print_cross_category_patterns(cross_hires, min_hires=10):
    """Print cross-category hiring patterns."""
    print("(Job Category → Freelancer Category)")
    print()
    
    for job_cat, freelancer_hires in cross_hires.items():
        total_hires = sum(freelancer_hires.values())
        if total_hires >= min_hires:
            print(f'{job_cat} jobs ({total_hires} total hires):')
            for freelancer_cat, count in sorted(freelancer_hires.items(), key=lambda x: x[1], reverse=True):
                if count > 0:
                    pct = (count / total_hires) * 100
                    match_type = '✓ Direct Match' if job_cat == freelancer_cat else '✗ Cross-category'
                    print(f'  → {freelancer_cat:<28} {count:2d} hires ({pct:4.1f}%) {match_type}')
            print()


def analyze_cross_category_hiring(hiring_outcomes, all_jobs, freelancers):
    """Analyze hiring patterns across different job-freelancer category combinations."""
    cross_hires = collect_cross_category_hires(hiring_outcomes, all_jobs, freelancers)
    print_cross_category_patterns(cross_hires)


def collect_hiring_flows(hiring_outcomes, all_jobs, freelancers):
    """Collect hiring flow data between freelancer and job categories."""
    flows = defaultdict(int)
    
    for outcome in hiring_outcomes:
        selected = outcome.get('selected_freelancer')
        if selected and selected != 'none':
            job_id = outcome.get('job_id')
            
            # Find job category
            job_cat = None
            for job in all_jobs:
                if job['id'] == job_id:
                    job_cat = job.get('category', 'Unknown')
                    break
            
            # Find freelancer category
            freelancer_cat = freelancers.get(selected, {}).get('category', 'Unknown')
            
            if job_cat and freelancer_cat:
                flows[(freelancer_cat, job_cat)] += 1
    
    return flows


def prepare_sankey_data(flows):
    """Prepare node labels and link data for Sankey diagram."""
    freelancer_categories = sorted(set(flow[0] for flow in flows.keys()))
    job_categories = sorted(set(flow[1] for flow in flows.keys()))
    
    # Create node labels
    node_labels = [f"Freelancer: {cat}" for cat in freelancer_categories] + \
                  [f"Job: {cat}" for cat in job_categories]
    
    # Create source, target, and value lists
    source = []
    target = []
    value = []
    
    for (freelancer_cat, job_cat), count in flows.items():
        if count > 0:
            source_idx = freelancer_categories.index(freelancer_cat)
            target_idx = len(freelancer_categories) + job_categories.index(job_cat)
            
            source.append(source_idx)
            target.append(target_idx)
            value.append(count)
    
    return {
        'freelancer_categories': freelancer_categories,
        'job_categories': job_categories,
        'node_labels': node_labels,
        'source': source,
        'target': target,
        'value': value
    }


def create_color_scheme(freelancer_categories, job_categories):
    """Create color scheme for Sankey diagram nodes and links."""
    color_palette = [
        '#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4', '#FFEAA7', 
        '#DDA0DD', '#98D8C8', '#F7DC6F', '#BB8FCE', '#85C1E9',
        '#F8C471', '#82E0AA', '#F1948A', '#AED6F1', '#A9DFBF'
    ]
    
    # Ensure we have enough colors
    while len(color_palette) < len(freelancer_categories):
        color_palette.extend(color_palette)
    
    # Create node colors
    node_colors = []
    
    # Freelancer nodes (left side)
    for i in range(len(freelancer_categories)):
        node_colors.append(color_palette[i % len(color_palette)])
    
    # Job nodes (right side) - use same colors as corresponding freelancer categories
    for cat in job_categories:
        if cat in freelancer_categories:
            idx = freelancer_categories.index(cat)
            node_colors.append(color_palette[idx % len(color_palette)])
        else:
            node_colors.append('#CCCCCC')
    
    return node_colors, color_palette


def create_link_colors(source, target, freelancer_categories, job_categories, color_palette):
    """Create colors for Sankey diagram links based on flow type."""
    link_colors = []
    
    for s, t in zip(source, target):
        freelancer_cat = freelancer_categories[s]
        job_cat = job_categories[t - len(freelancer_categories)]
        
        # Use the freelancer category color
        freelancer_idx = freelancer_categories.index(freelancer_cat)
        base_color = color_palette[freelancer_idx % len(color_palette)]
        
        # Convert hex to rgba
        hex_color = base_color.lstrip('#')
        r = int(hex_color[0:2], 16)
        g = int(hex_color[2:4], 16)
        b = int(hex_color[4:6], 16)
        
        # Adjust opacity based on match type
        if freelancer_cat == job_cat:
            link_colors.append(f'rgba({r}, {g}, {b}, 0.6)')  # Same category: more opaque
        else:
            link_colors.append(f'rgba({r}, {g}, {b}, 0.3)')  # Cross-category: more transparent
    
    return link_colors


def save_sankey_diagram(fig, output_path):
    """Save Sankey diagram as HTML and optionally as PNG."""
    sankey_file = output_path / "category_hiring_flows.html"
    fig.write_html(str(sankey_file))
    
    # Also save as PNG (requires kaleido)
    try:
        png_file = output_path / "category_hiring_flows.png"
        fig.write_image(str(png_file), width=1200, height=800, scale=2)
        print(f"📊 Sankey diagram saved as PNG: {png_file}")
    except Exception as e:
        print(f"⚠️  Could not save PNG (install kaleido for PNG export): {e}")
    
    print(f"📊 Interactive Sankey diagram saved: {sankey_file}")
    print("💡 Open the HTML file in your browser to interact with the diagram")


def print_hiring_flow_summary(flows):
    """Print summary statistics of hiring flows."""
    same_category_hires = sum(count for (f_cat, j_cat), count in flows.items() if f_cat == j_cat)
    cross_category_hires = sum(count for (f_cat, j_cat), count in flows.items() if f_cat != j_cat)
    total_hires = same_category_hires + cross_category_hires
    
    print("\n📈 Hiring Flow Summary:")
    print(f"   Same-category hires: {same_category_hires} ({same_category_hires/total_hires*100:.1f}%)")
    print(f"   Cross-category hires: {cross_category_hires} ({cross_category_hires/total_hires*100:.1f}%)")
    print(f"   Total hires: {total_hires}")


def create_sankey_flow_diagram(hiring_outcomes, all_jobs, freelancers):
    """Create a Sankey diagram showing freelancer category → job category hiring flows."""
    
    print("\n🌊 CREATING SANKEY FLOW DIAGRAM")
    print("-" * 60)
    
    # Create output directory
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    
    # Collect and prepare data
    flows = collect_hiring_flows(hiring_outcomes, all_jobs, freelancers)
    sankey_data = prepare_sankey_data(flows)
    node_colors, color_palette = create_color_scheme(
        sankey_data['freelancer_categories'],
        sankey_data['job_categories']
    )
    link_colors = create_link_colors(
        sankey_data['source'],
        sankey_data['target'],
        sankey_data['freelancer_categories'],
        sankey_data['job_categories'],
        color_palette
    )
    
    # Create the Sankey diagram
    fig = go.Figure(data=[go.Sankey(
        node=dict(
            pad=15,
            thickness=20,
            line=dict(color="black", width=0.5),
            label=sankey_data['node_labels'],
            color=node_colors
        ),
        link=dict(
            source=sankey_data['source'],
            target=sankey_data['target'],
            value=sankey_data['value'],
            color=link_colors
        )
    )])
    
    fig.update_layout(
        title_text="Freelancer Category → Job Category Hiring Flows<br><sub>Each category has a distinct color. Darker flows = Same category, Lighter flows = Cross-category</sub>",
        font_size=12,
        width=1200,
        height=800
    )
    
    # Save diagram and print summary
    save_sankey_diagram(fig, OUTPUT_DIR)
    print_hiring_flow_summary(flows)


def create_category_visualizations(freelancer_counts, job_counts, category_metrics):
    """Create visualizations for category analysis"""
    
    # Create output directory
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    
    # Set style
    plt.style.use('default')
    sns.set_palette("husl")
    
    # 1. Freelancer and Job Distribution
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
    
    # Freelancer distribution
    categories = list(freelancer_counts.keys())[:8]  # Top 8 categories
    counts = [freelancer_counts[cat] for cat in categories]
    
    ax1.bar(range(len(categories)), counts, alpha=0.7)
    ax1.set_title('Freelancer Distribution by Category', fontsize=14, fontweight='bold')
    ax1.set_xlabel('Category')
    ax1.set_ylabel('Number of Freelancers')
    ax1.set_xticks(range(len(categories)))
    ax1.set_xticklabels(categories, rotation=45, ha='right')
    ax1.grid(True, alpha=0.3)
    
    # Job distribution
    job_categories = list(job_counts.keys())[:8]  # Top 8 categories
    job_counts_list = [job_counts[cat] for cat in job_categories]
    
    ax2.bar(range(len(job_categories)), job_counts_list, alpha=0.7, color='orange')
    ax2.set_title('Job Distribution by Category', fontsize=14, fontweight='bold')
    ax2.set_xlabel('Category')
    ax2.set_ylabel('Number of Jobs')
    ax2.set_xticks(range(len(job_categories)))
    ax2.set_xticklabels(job_categories, rotation=45, ha='right')
    ax2.grid(True, alpha=0.3)
    
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / "freelancer_job_distribution.png", dpi=300, bbox_inches='tight')
    plt.close()
    
    # 2. Category Performance Metrics - Focus on Fill Rate and Competition Level Only
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
    
    categories = list(category_metrics.keys())
    fill_rates = [category_metrics[cat]['fill_rate'] for cat in categories]
    avg_bids = [category_metrics[cat]['avg_bids_per_job'] for cat in categories]
    
    # Fill rates
    ax1.bar(range(len(categories)), fill_rates, alpha=0.7, color='green')
    ax1.set_title('Fill Rate by Job Category', fontsize=14, fontweight='bold')
    ax1.set_ylabel('Fill Rate (%)')
    ax1.set_xticks(range(len(categories)))
    ax1.set_xticklabels(categories, rotation=45, ha='right')
    ax1.grid(True, alpha=0.3)
    
    # Average bids per job (Competition Level)
    ax2.bar(range(len(categories)), avg_bids, alpha=0.7, color='red')
    ax2.set_title('Competition Level by Job Category', fontsize=14, fontweight='bold')
    ax2.set_ylabel('Average Bids per Job')
    ax2.set_xticks(range(len(categories)))
    ax2.set_xticklabels(categories, rotation=45, ha='right')
    ax2.grid(True, alpha=0.3)
    
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / "category_performance_metrics.png", dpi=300, bbox_inches='tight')
    plt.close()
    
    print("📊 Visualizations saved:")
    print(f"   - {OUTPUT_DIR}/freelancer_job_distribution.png")
    print(f"   - {OUTPUT_DIR}/category_performance_metrics.png")


def analyze_competition_patterns(all_jobs, all_bids, job_category_counts):
    """Analyze competition patterns and bidding behavior across categories"""
    
    print("\n🏁 COMPETITION PATTERNS BY CATEGORY")
    print("-" * 60)
    
    competition_data = {}
    
    for category, job_count in job_category_counts.items():
        # Get jobs and bids for this category
        category_jobs = [job for job in all_jobs if job.get('category') == category]
        job_ids = [job['id'] for job in category_jobs]
        category_bids = [bid for bid in all_bids if bid.get('job_id') in job_ids]
        
        if not category_bids:
            continue
        
        total_bids = len(category_bids)
        avg_bids_per_job = total_bids / job_count if job_count > 0 else 0
        
        # Count unique bidders and repeat bidders
        freelancer_bid_counts = Counter(bid.get('freelancer_id') for bid in category_bids)
        unique_bidders = len(freelancer_bid_counts)
        
        # Calculate repeat bidder rate
        jobs_per_freelancer = defaultdict(set)
        for bid in category_bids:
            jobs_per_freelancer[bid.get('freelancer_id')].add(bid.get('job_id'))
        
        repeat_bidders = sum(1 for jobs_set in jobs_per_freelancer.values() if len(jobs_set) > 1)
        repeat_rate = repeat_bidders / unique_bidders if unique_bidders > 0 else 0
        
        # Check if bidding is well-distributed (no single freelancer dominates)
        max_bids_by_one = max(freelancer_bid_counts.values()) if freelancer_bid_counts else 0
        max_share = max_bids_by_one / total_bids if total_bids > 0 else 0
        
        competition_data[category] = {
            'avg_bids_per_job': avg_bids_per_job,
            'unique_bidders': unique_bidders,
            'repeat_rate': repeat_rate,
            'max_freelancer_share': max_share,
            'total_bids': total_bids
        }
    
    # Display results
    print(f"{'Category':<25} {'Bids/Job':<9} {'Unique':<7} {'Repeat%':<8} {'MaxShare%':<10}")
    print("-" * 70)
    
    for category, data in sorted(competition_data.items()):
        print(f"{category:<25} {data['avg_bids_per_job']:<9.2f} {data['unique_bidders']:<7} "
              f"{data['repeat_rate']*100:<8.1f} {data['max_freelancer_share']*100:<10.1f}")
    
    # Summary insights
    high_competition = [cat for cat, data in competition_data.items() if data['avg_bids_per_job'] > 6]
    well_distributed = [cat for cat, data in competition_data.items() if data['max_freelancer_share'] < 0.3]
    
    print("\n📊 Competition Insights:")
    print(f"High competition categories (>6 bids/job): {', '.join(high_competition)}")
    print(f"Well-distributed bidding (<30% max share): {len(well_distributed)}/{len(competition_data)} categories")
    
    return competition_data


def collect_category_flows(hiring_outcomes, all_jobs, freelancers):
    """Collect hiring flow data between categories."""
    flows = defaultdict(int)
    category_totals = defaultdict(int)
    
    for outcome in hiring_outcomes:
        selected = outcome.get('selected_freelancer')
        if selected and selected != 'none':
            job_id = outcome.get('job_id')
            
            # Find job and freelancer categories
            job_cat = None
            for job in all_jobs:
                if job['id'] == job_id:
                    job_cat = job.get('category', 'Unknown')
                    break
            
            freelancer_cat = freelancers.get(selected, {}).get('category', 'Unknown')
            
            if job_cat and freelancer_cat:
                flows[(freelancer_cat, job_cat)] += 1
                category_totals[freelancer_cat] += 1
    
    return flows, category_totals


def calculate_specialization_data(flows, category_totals):
    """Calculate specialization rates for each category."""
    specialization_data = {}
    for freelancer_cat, total_hires in category_totals.items():
        same_category_hires = flows.get((freelancer_cat, freelancer_cat), 0)
        specialization_rate = same_category_hires / total_hires if total_hires > 0 else 0
        cross_category_rate = 1 - specialization_rate
        
        specialization_data[freelancer_cat] = {
            'specialization_rate': specialization_rate,
            'cross_category_rate': cross_category_rate,
            'total_hires': total_hires
        }
    
    return specialization_data


def print_specialization_rates(specialization_data):
    """Print category specialization rates."""
    print("Category Specialization (% hired within same category):")
    print("-" * 60)
    
    for freelancer_cat, data in specialization_data.items():
        print(f"{freelancer_cat:<30}: {data['specialization_rate']*100:5.1f}% within category, "
              f"{data['cross_category_rate']*100:5.1f}% cross-category")


def find_significant_flows(flows, category_totals, threshold=0.05):
    """Find significant cross-category flows above the threshold."""
    significant_flows = []
    for (freelancer_cat, job_cat), count in flows.items():
        if freelancer_cat != job_cat:  # Only cross-category
            total_from_category = category_totals[freelancer_cat]
            if total_from_category > 0:
                flow_rate = count / total_from_category
                if flow_rate > threshold:
                    significant_flows.append((freelancer_cat, job_cat, flow_rate, count))
    
    significant_flows.sort(key=lambda x: x[2], reverse=True)
    return significant_flows


def print_significant_flows(significant_flows):
    """Print significant cross-category flows."""
    print("\nMajor Cross-Category Flows (>5% of freelancer category):")
    print("-" * 60)
    
    for freelancer_cat, job_cat, rate, count in significant_flows:
        print(f"{freelancer_cat:<20} → {job_cat:<20}: {rate*100:5.1f}% ({count} hires)")


def print_flow_summary(flows, category_totals):
    """Print overall flow summary statistics."""
    total_hires = sum(category_totals.values())
    same_category_total = sum(flows.get((cat, cat), 0) for cat in category_totals.keys())
    cross_category_total = total_hires - same_category_total
    
    print("\n📊 Overall Flow Summary:")
    print(f"Same-category hires: {same_category_total} ({same_category_total/total_hires*100:.1f}%)")
    print(f"Cross-category hires: {cross_category_total} ({cross_category_total/total_hires*100:.1f}%)")


def analyze_cross_category_flows(hiring_outcomes, all_jobs, freelancers):
    """Analyze cross-category hiring flows and patterns"""
    
    print("\n🔄 CROSS-CATEGORY HIRING FLOWS")
    print("-" * 60)
    
    flows, category_totals = collect_category_flows(hiring_outcomes, all_jobs, freelancers)
    specialization_data = calculate_specialization_data(flows, category_totals)
    
    print_specialization_rates(specialization_data)
    
    significant_flows = find_significant_flows(flows, category_totals)
    print_significant_flows(significant_flows)
    
    print_flow_summary(flows, category_totals)
    
    return specialization_data, significant_flows


def analyze_reputation_tiers(simulation_data):
    """Analyze final reputation tier distribution"""
    
    print("\n🏆 FINAL REPUTATION TIER DISTRIBUTION")
    print("-" * 60)
    
    # Get reputation data
    reputation_data = simulation_data.get('reputation_data', {})
    freelancer_reputation = reputation_data.get('freelancers', {})
    
    if not freelancer_reputation:
        print("No reputation data available")
        return
    
    # Count tiers
    tier_counts = Counter()
    for freelancer_id, rep_info in freelancer_reputation.items():
        tier = rep_info.get('tier', 'Unknown')
        tier_counts[tier] += 1
    
    total_freelancers = len(freelancer_reputation)
    
    print(f"Freelancer Reputation Tiers (Total: {total_freelancers}):")
    print("-" * 60)
    
    for tier, count in tier_counts.most_common():
        percentage = (count / total_freelancers) * 100
        print(f"   {tier:<15}: {count:3d} freelancers ({percentage:5.1f}%)")
    
    # Create pie chart
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    
    if tier_counts:
        plt.figure(figsize=(8, 8))
        tiers = list(tier_counts.keys())
        counts = list(tier_counts.values())
        colors = ['#FF9999', '#66B2FF', '#99FF99', '#FFD700'][:len(tiers)]
        
        plt.pie(counts, labels=tiers, autopct='%1.1f%%', startangle=90, colors=colors)
        plt.title('Final Freelancer Reputation Tier Distribution', fontsize=14, fontweight='bold')
        plt.axis('equal')
        
        plt.savefig(OUTPUT_DIR / REPUTATION_TIER_CHART_FILENAME, dpi=300, bbox_inches='tight')
        plt.close()
        
        print(f"📊 Reputation tier pie chart saved: {OUTPUT_DIR}/{REPUTATION_TIER_CHART_FILENAME}")
    
    return tier_counts


def analyze_rejection_patterns_with_charts(hiring_outcomes, all_bids):
    """Analyze rejection patterns and create pie charts"""
    
    print("\n🚫 REJECTION PATTERN ANALYSIS")
    print("-" * 60)
    
    # Analyze hiring outcomes
    hired_count = 0
    rejected_by_client = 0
    no_bids_received = 0
    rejection_reasons = []
    
    for outcome in hiring_outcomes:
        selected = outcome.get('selected_freelancer')
        reasoning = outcome.get('reasoning', '').lower()
        
        if selected and selected != 'none':
            hired_count += 1
        elif 'no bid' in reasoning or 'no bids' in reasoning:
            no_bids_received += 1
        else:
            rejected_by_client += 1
            rejection_reasons.append(outcome.get('reasoning', ''))
    
    total_jobs = len(hiring_outcomes)
    
    print(f"Job Outcome Distribution (Total: {total_jobs} jobs):")
    print("-" * 60)
    print(f"   Successful hires    : {hired_count:3d} ({hired_count/total_jobs*100:5.1f}%)")
    print(f"   Client rejections   : {rejected_by_client:3d} ({rejected_by_client/total_jobs*100:5.1f}%)")
    print(f"   No bids received    : {no_bids_received:3d} ({no_bids_received/total_jobs*100:5.1f}%)")
    
    # Analyze rejection reasons if available
    if rejection_reasons:
        print("\nSample rejection reasons:")
        for i, reason in enumerate(rejection_reasons[:3], 1):
            print(f"   {i}. \"{reason[:100]}{'...' if len(reason) > 100 else ''}\"")
    
    # Create outcome pie chart
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    
    # Job outcomes pie chart
    plt.figure(figsize=(10, 5))
    
    # First subplot: Job outcomes
    plt.subplot(1, 2, 1)
    outcomes = ['Successful Hires', 'Client Rejections', 'No Bids Received']
    counts = [hired_count, rejected_by_client, no_bids_received]
    colors = ['#99FF99', '#FF9999', '#FFCC99']
    
    plt.pie(counts, labels=outcomes, autopct='%1.1f%%', startangle=90, colors=colors)
    plt.title('Job Outcome Distribution', fontsize=12, fontweight='bold')
    
    # Second subplot: Success vs All Rejections
    plt.subplot(1, 2, 2)
    success_vs_rejection = ['Successful Hires', 'All Rejections']
    success_vs_counts = [hired_count, rejected_by_client + no_bids_received]
    colors2 = ['#99FF99', '#FF9999']
    
    plt.pie(success_vs_counts, labels=success_vs_rejection, autopct='%1.1f%%', startangle=90, colors=colors2)
    plt.title('Success vs Rejection Rate', fontsize=12, fontweight='bold')
    
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / "job_outcome_analysis.png", dpi=300, bbox_inches='tight')
    plt.close()
    
    print(f"📊 Job outcome charts saved: {OUTPUT_DIR}/job_outcome_analysis.png")
    
    # Bid success analysis
    total_bids = len(all_bids)
    if total_bids > 0:
        bid_success_rate = hired_count / total_bids * 100
        print("\n📈 Bid Success Metrics:")
        print(f"   Total bids placed   : {total_bids}")
        print(f"   Successful bids     : {hired_count}")
        print(f"   Bid success rate    : {bid_success_rate:.1f}%")
    
    return {
        'total_jobs': total_jobs,
        'hired_count': hired_count,
        'rejected_by_client': rejected_by_client,
        'no_bids_received': no_bids_received,
        'total_bids': total_bids
    }


def calculate_tier_progression(freelancers):
    """Calculate tier progression for all freelancers."""
    tier_counts = {'New': 0, 'Established': 0, 'Expert': 0, 'Elite': 0}
    freelancer_progression = []
    
    for freelancer_id, freelancer in freelancers.items():
        total_hired = freelancer.get('total_hired', 0)
        completed_jobs = freelancer.get('completed_jobs', 0)
        
        tier = calculate_freelancer_tier(freelancer)
        
        tier_counts[tier] += 1
        freelancer_progression.append({
            'id': freelancer_id,
            'name': freelancer.get('name', 'Unknown'),
            'tier': tier,
            'total_hired': total_hired,
            'completed_jobs': completed_jobs,
            'completion_rate': completed_jobs / total_hired if total_hired > 0 else 0
        })
    
    return tier_counts, freelancer_progression


def print_tier_progression(tier_counts):
    """Print tier progression statistics."""
    print("\n📊 FREELANCER TIER PROGRESSION:")
    total_freelancers = sum(tier_counts.values())
    for tier, count in tier_counts.items():
        percentage = (count / total_freelancers) * 100
        print(f"   {tier:12}: {count:3d} freelancers ({percentage:5.1f}%)")


def find_reputation_mentions(hiring_outcomes, freelancer_progression):
    """Find hiring decisions that mention reputation factors."""
    reputation_keywords = ['reputation', 'experience', 'success', 'track record', 
                          'established', 'expert', 'proven']
    reputation_mentions = []
    
    for outcome in hiring_outcomes:
        if outcome.get('selected_freelancer') and outcome.get('reasoning'):
            reasoning = outcome['reasoning'].lower()
            if any(keyword in reasoning for keyword in reputation_keywords):
                freelancer_id = outcome['selected_freelancer']
                freelancer_info = next((f for f in freelancer_progression if f['id'] == freelancer_id), None)
                
                reasoning_text = outcome['reasoning']
                truncated_reasoning = reasoning_text[:200] + "..." if len(reasoning_text) > 200 else reasoning_text
                
                reputation_mentions.append({
                    'job_id': outcome['job_id'],
                    'freelancer_id': freelancer_id,
                    'freelancer_tier': freelancer_info['tier'] if freelancer_info else 'Unknown',
                    'reasoning': truncated_reasoning
                })
    
    return reputation_mentions


def collect_tier_examples(reputation_mentions, max_per_tier=2):
    """Collect example hiring decisions for each tier."""
    tier_examples = {'New': [], 'Established': [], 'Expert': [], 'Elite': []}
    
    for mention in reputation_mentions:
        tier = mention['freelancer_tier']
        if tier in tier_examples and len(tier_examples[tier]) < max_per_tier:
            tier_examples[tier].append(mention)
    
    return tier_examples


def print_reputation_hiring_decisions(reputation_mentions, tier_examples):
    """Print reputation-based hiring decisions and examples."""
    print("\n🔍 SEARCHING FOR REPUTATION-BASED HIRING DECISIONS:")
    print(f"   Found {len(reputation_mentions)} hiring decisions mentioning reputation factors")
    
    for tier in ['Elite', 'Expert', 'Established', 'New']:
        if tier_examples[tier]:
            print(f"\n   {tier} Freelancer Hiring Examples:")
            for i, example in enumerate(tier_examples[tier], 1):
                print(f"   {i}. Job: {example['job_id']}")
                print(f"      Reasoning: {example['reasoning']}")


def print_promotion_statistics(tier_counts):
    """Print freelancer promotion statistics."""
    print("\n📈 HIRING SUCCESS RATES BY TIER:")
    total_freelancers = sum(tier_counts.values())
    promoted_freelancers = sum(tier_counts[tier] for tier in ['Established', 'Expert', 'Elite'])
    promotion_rate = (promoted_freelancers / total_freelancers) * 100
    
    print(f"   Freelancers who advanced beyond 'New': {promoted_freelancers}/{total_freelancers} ({promotion_rate:.1f}%)")
    
    return promotion_rate


def print_elite_achievements(freelancer_progression):
    """Print elite freelancer achievements."""
    elite_freelancers = [f for f in freelancer_progression if f['tier'] == 'Elite']
    
    if elite_freelancers:
        print("\n🏆 ELITE FREELANCER ACHIEVEMENTS:")
        for freelancer in elite_freelancers[:3]:  # Show top 3
            print(f"   {freelancer['name']}: {freelancer['total_hired']} jobs, {freelancer['completion_rate']:.1%} success rate")


def analyze_reputation_impact(simulation_data, output_dir):
    """Analyze the impact of reputation tiers on hiring success and provide examples"""
    
    freelancers = simulation_data['freelancer_profiles']
    hiring_outcomes = simulation_data['hiring_outcomes']
    
    tier_counts, freelancer_progression = calculate_tier_progression(freelancers)
    print_tier_progression(tier_counts)
    
    reputation_mentions = find_reputation_mentions(hiring_outcomes, freelancer_progression)
    tier_examples = collect_tier_examples(reputation_mentions)
    print_reputation_hiring_decisions(reputation_mentions, tier_examples)
    
    promotion_rate = print_promotion_statistics(tier_counts)
    print_elite_achievements(freelancer_progression)
    
    create_reputation_pie_chart(tier_counts, output_dir)
    
    return {
        'tier_distribution': tier_counts,
        'reputation_mentions': len(reputation_mentions),
        'promotion_rate': promotion_rate,
        'elite_count': tier_counts['Elite']
    }


def create_reputation_pie_chart(tier_counts, output_dir):
    """Create a professional pie chart for reputation tier distribution for the paper"""
    
    # Create papers/figures directory for paper figures
    paper_output_dir = Path("papers/figures")
    paper_output_dir.mkdir(parents=True, exist_ok=True)
    
    if not tier_counts or sum(tier_counts.values()) == 0:
        print("No reputation data available for pie chart")
        return
    
    # Calculate percentages
    total_freelancers = sum(tier_counts.values())
    data = []
    for tier in ['New', 'Established', 'Expert', 'Elite']:  # Ordered by progression
        count = tier_counts.get(tier, 0)
        if count > 0:
            percentage = (count / total_freelancers) * 100
            data.append((tier, percentage))
    
    if not data:
        print("No valid tier data for pie chart")
        return
    
    # Set style for consistent, professional appearance
    plt.style.use('default')
    plt.rcParams['font.size'] = 12
    plt.rcParams['font.family'] = 'serif'
    
    fig, ax = plt.subplots(figsize=(10, 6))
    
    labels, values = zip(*data)
    
    # Use progression colors (light to dark)
    colors = ['#ffcccc', '#ff9999', '#ff6666', '#cc0000'][:len(labels)]
    
    # Create pie chart with percentage labels (use legend to avoid overlap)
    wedges, texts, autotexts = ax.pie(values, autopct='%1.1f%%', 
                                      colors=colors, startangle=90)
    
    # Improve text formatting for percentages
    for autotext in autotexts:
        autotext.set_color('white')
        autotext.set_fontweight('bold')
        autotext.set_fontsize(10)
    
    # Add legend instead of direct labels to avoid overlap
    ax.legend(wedges, labels, title="Reputation Tiers", 
              loc="center left", bbox_to_anchor=(1, 0, 0.5, 1), fontsize=11)
    
    ax.set_title('Freelancer Reputation Tier Distribution\n(After 100 Rounds)', 
                 fontsize=14, fontweight='bold', pad=20)
    
    # Equal aspect ratio ensures that pie is drawn as a circle
    ax.axis('equal')
    
    plt.tight_layout()
    
    # Save to papers/figures for the paper
    paper_filename = paper_output_dir / REPUTATION_TIER_CHART_FILENAME
    plt.savefig(paper_filename, dpi=300, bbox_inches='tight')
    
    # Also save to analysis results
    analysis_filename = output_dir / REPUTATION_TIER_CHART_FILENAME
    plt.savefig(analysis_filename, dpi=300, bbox_inches='tight')
    
    plt.close()
    
    print(f"📊 Reputation tier pie chart saved for paper: {paper_filename}")
    print(f"📊 Reputation tier pie chart saved for analysis: {analysis_filename}")


if __name__ == "__main__":
    analyze_llm_llm_by_category()