generate_metrics_report.py

#!/usr/bin/env python3
"""Generate comprehensive metrics report with plots

Creates plots and a markdown report showing:
- Entropy (Shannon, high-order) across all simulations
- Computational cost (CPU time, memory) 
- Comparisons across step counts (5, 50, 500)
"""

import sys
sys.path.insert(0, '.')

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import json
from pathlib import Path
from datetime import datetime
from typing import Dict, Any, List
# Set style
plt.style.use('default')
plt.rcParams['figure.facecolor'] = 'white'
plt.rcParams['axes.grid'] = True
plt.rcParams['grid.alpha'] = 0.3

# Import simulations
from simulations.conway import Conway2D, Conway3D
from simulations.cellular_automata import Rule30, Rule110, LangtonLoops, Wireworld
from simulations.boids import Boids
from simulations.genetic_algorithms import GeneticAlgorithm
from simulations.tierra import Tierra
from simulations.symbiosis import BFFSymbiosis
from simulations.lenia import Lenia
from simulations.stringmol import Stringmol
from simulations.avida import Avida
from simulations.xenobots import Xenobots
from configs.analysis_config import DEFAULT_STEP_COUNTS

SIMULATIONS = {
    'conway_2d': (Conway2D, {'width': 50, 'height': 50, 'density': 0.3}),
    'conway_3d': (Conway3D, {'width': 20, 'height': 20, 'depth': 20, 'density': 0.2}),
    'rule30': (Rule30, {'width': 100, 'density': 0.5}),
    'rule110': (Rule110, {'width': 100, 'density': 0.5}),
    'langton_loops': (LangtonLoops, {'width': 50, 'height': 50}),
    'wireworld': (Wireworld, {'width': 50, 'height': 50, 'num_electrons': 5}),
    'genetic_algorithm': (GeneticAlgorithm, {'population_size': 30, 'genome_length': 15}),
    'tierra': (Tierra, {'memory_size': 5000, 'num_creatures': 5}),
    'lenia': (Lenia, {'width': 50, 'height': 50}),
    'stringmol': (Stringmol, {'num_strings': 15, 'max_length': 30}),
    'avida': (Avida, {'memory_size': 5000, 'num_creatures': 5}),
    'xenobots': (Xenobots, {'num_xenobots': 3, 'voxel_size': (8, 8, 7), 'world_size': 100}),
    'bff_brainfuck': (BFFSymbiosis, {'language': 'brainfuck', 'num_programs': 10, 'program_length': 50}),
    'bff_forth': (BFFSymbiosis, {'language': 'forth', 'num_programs': 10, 'program_length': 50}),
}

STEP_COUNTS = DEFAULT_STEP_COUNTS


def run_simulation_collect_metrics(sim_name: str, sim_class, config: Dict[str, Any], 
                                   steps: int) -> Dict[str, Any]:
    """Run simulation and collect all metrics"""
    try:
        sim = sim_class(config=config)
        sim.run(steps=steps, collect_metrics=True)
        
        metrics_summary = sim.get_metrics_summary()
        entropy_metrics = sim.metrics['entropy']
        computational_metrics = sim.metrics['computational']
        
        return {
            'simulation': sim_name,
            'steps': steps,
            'summary': metrics_summary,
            'time_series': {
                'step': list(range(len(entropy_metrics['shannon_entropy']))),
                'shannon_entropy': [float(x) for x in entropy_metrics['shannon_entropy']],
                'high_order_entropy': [float(x) for x in entropy_metrics.get('high_order_entropy', [])],
                'cpu_time': [float(x) for x in computational_metrics['cpu_time']],
                'memory_mb': [float(x) for x in computational_metrics['memory_usage']],
                'step_time': [float(x) for x in computational_metrics['step_times']],
            },
            'status': 'success'
        }
    except Exception as e:
        return {
            'simulation': sim_name,
            'steps': steps,
            'status': 'error',
            'error': str(e)
        }


def generate_plots(all_results: Dict[int, List[Dict[str, Any]]], output_dir: Path):
    """Generate all plots"""
    plots = {}
    
    # Collect data across all step counts
    all_data = []
    for steps, results in all_results.items():
        for r in results:
            if r['status'] == 'success':
                summary = r['summary']
                all_data.append({
                    'simulation': r['simulation'],
                    'steps': steps,
                    'avg_shannon_entropy': summary.get('avg_entropy', 0),
                    'final_shannon_entropy': summary.get('final_entropy', 0),
                    'avg_high_order_entropy': summary.get('avg_high_order_entropy', 0),
                    'final_high_order_entropy': summary.get('final_high_order_entropy', 0),
                    'total_cpu_time': summary.get('total_cpu_time', 0),
                    'avg_step_time': summary.get('avg_step_time', 0),
                    'max_memory_mb': summary.get('max_memory_mb', 0),
                })
    
    df = pd.DataFrame(all_data)
    
    # Define greenscale colors for step counts
    green_colors = ['#004d00', '#006600', '#008000', '#00b300']  # Dark to light green
    
    # 1. Shannon Entropy Comparison Across Step Counts
    fig, ax = plt.subplots(figsize=(14, 8))
    pivot_shannon = df.pivot_table(values='avg_shannon_entropy', index='simulation', columns='steps')
    pivot_shannon.plot(kind='bar', ax=ax, width=0.8, color=green_colors)
    ax.set_xlabel('Simulation', fontsize=12)
    ax.set_ylabel('Average Shannon Entropy (bits)', fontsize=12)
    ax.set_title('Shannon Entropy Across Step Counts', fontsize=14, weight='bold')
    ax.legend(title='Steps', labels=[f'{s} steps' for s in STEP_COUNTS])
    ax.tick_params(axis='x', rotation=45)
    ax.grid(True, alpha=0.3, axis='y')
    plt.tight_layout()
    plot_path = output_dir / 'shannon_entropy_comparison.png'
    plt.savefig(plot_path, dpi=150, bbox_inches='tight')
    plt.close()
    plots['shannon_entropy'] = plot_path.name
    print(f"  ✓ Generated: {plot_path.name}")
    
    # 2. High-Order Entropy Comparison
    fig, ax = plt.subplots(figsize=(14, 8))
    pivot_hoe = df.pivot_table(values='avg_high_order_entropy', index='simulation', columns='steps')
    pivot_hoe.plot(kind='bar', ax=ax, width=0.8, color=green_colors)
    ax.set_xlabel('Simulation', fontsize=12)
    ax.set_ylabel('Average High-Order Entropy (bits)', fontsize=12)
    ax.set_title('High-Order Entropy Across Step Counts', fontsize=14, weight='bold')
    ax.legend(title='Steps', labels=[f'{s} steps' for s in STEP_COUNTS])
    ax.tick_params(axis='x', rotation=45)
    ax.grid(True, alpha=0.3, axis='y')
    plt.tight_layout()
    plot_path = output_dir / 'high_order_entropy_comparison.png'
    plt.savefig(plot_path, dpi=150, bbox_inches='tight')
    plt.close()
    plots['high_order_entropy'] = plot_path.name
    print(f"  ✓ Generated: {plot_path.name}")
    
    # 3. Computational Cost (CPU Time)
    fig, ax = plt.subplots(figsize=(14, 8))
    pivot_cpu = df.pivot_table(values='total_cpu_time', index='simulation', columns='steps')
    pivot_cpu.plot(kind='bar', ax=ax, width=0.8, logy=True, color=green_colors)
    ax.set_xlabel('Simulation', fontsize=12)
    ax.set_ylabel('Total CPU Time (seconds, log scale)', fontsize=12)
    ax.set_title('Computational Cost Across Step Counts', fontsize=14, weight='bold')
    ax.legend(title='Steps', labels=[f'{s} steps' for s in STEP_COUNTS])
    ax.tick_params(axis='x', rotation=45)
    ax.grid(True, alpha=0.3, axis='y')
    plt.tight_layout()
    plot_path = output_dir / 'computational_cost_comparison.png'
    plt.savefig(plot_path, dpi=150, bbox_inches='tight')
    plt.close()
    plots['computational_cost'] = plot_path.name
    print(f"  ✓ Generated: {plot_path.name}")
    
    # 4. Memory Usage
    fig, ax = plt.subplots(figsize=(14, 8))
    pivot_memory = df.pivot_table(values='max_memory_mb', index='simulation', columns='steps')
    pivot_memory.plot(kind='bar', ax=ax, width=0.8, color=green_colors)
    ax.set_xlabel('Simulation', fontsize=12)
    ax.set_ylabel('Max Memory Usage (MB)', fontsize=12)
    ax.set_title('Memory Usage Across Step Counts', fontsize=14, weight='bold')
    ax.legend(title='Steps', labels=[f'{s} steps' for s in STEP_COUNTS])
    ax.tick_params(axis='x', rotation=45)
    ax.grid(True, alpha=0.3, axis='y')
    plt.tight_layout()
    plot_path = output_dir / 'memory_usage_comparison.png'
    plt.savefig(plot_path, dpi=150, bbox_inches='tight')
    plt.close()
    plots['memory'] = plot_path.name
    print(f"  ✓ Generated: {plot_path.name}")
    
    # 5. Shannon vs High-Order Entropy Scatter (for 500 steps)
    results_500 = [r for r in all_results.get(500, []) if r['status'] == 'success']
    if results_500:
        fig, ax = plt.subplots(figsize=(10, 8))
        shannon_vals = [r['summary'].get('avg_entropy', 0) for r in results_500]
        hoe_vals = [r['summary'].get('avg_high_order_entropy', 0) for r in results_500]
        sim_names = [r['simulation'] for r in results_500]
        
        # Use greenscale colormap
        from matplotlib.colors import LinearSegmentedColormap
        greens_cmap = LinearSegmentedColormap.from_list('greens', 
            ['#004d00', '#006600', '#008000', '#00b300', '#00cc00', '#00e600', '#00ff00'])
        
        scatter = ax.scatter(shannon_vals, hoe_vals, s=100, alpha=0.6, 
                           c=range(len(results_500)), cmap=greens_cmap, 
                           edgecolors='black', linewidth=1)
        
        for i, name in enumerate(sim_names):
            ax.annotate(name, (shannon_vals[i], hoe_vals[i]), 
                       fontsize=8, alpha=0.7, xytext=(5, 5), textcoords='offset points')
        
        max_val = max(max(shannon_vals), max(hoe_vals)) if hoe_vals else max(shannon_vals)
        ax.plot([0, max_val], [0, max_val], 'g--', alpha=0.3, linewidth=2, label='y=x')
        
        ax.set_xlabel('Average Shannon Entropy (bits)', fontsize=12)
        ax.set_ylabel('Average High-Order Entropy (bits)', fontsize=12)
        ax.set_title('Shannon vs High-Order Entropy (500 steps)', fontsize=14, weight='bold')
        ax.legend()
        ax.grid(True, alpha=0.3)
        plt.tight_layout()
        plot_path = output_dir / 'shannon_vs_high_order_entropy.png'
        plt.savefig(plot_path, dpi=150, bbox_inches='tight')
        plt.close()
        plots['shannon_vs_hoe'] = plot_path.name
        print(f"  ✓ Generated: {plot_path.name}")
    
    # 7. Time Series Examples (select a few interesting simulations)
    interesting_sims = ['conway_2d', 'tierra', 'avida', 'bff_brainfuck', 'lenia']
    fig, axes = plt.subplots(2, 3, figsize=(18, 10))
    axes = axes.flatten()
    
    for idx, sim_name in enumerate(interesting_sims[:6]):
        ax = axes[idx]
        for step_idx, steps in enumerate(STEP_COUNTS):
            results = all_results.get(steps, [])
            result = next((r for r in results if r['simulation'] == sim_name and r['status'] == 'success'), None)
            if result and 'time_series' in result:
                ts = result['time_series']
                # Sample for plotting
                sample_idx = list(range(0, len(ts['step']), max(1, len(ts['step']) // 50)))
                steps_sampled = [ts['step'][i] for i in sample_idx]
                hoe_sampled = [ts['high_order_entropy'][i] if i < len(ts['high_order_entropy']) else 0 
                              for i in sample_idx]
                ax.plot(steps_sampled, hoe_sampled, label=f'{steps} steps', 
                       alpha=0.7, linewidth=1.5, color=green_colors[step_idx])
        
        ax.set_title(sim_name.replace('_', ' ').title(), fontsize=11)
        ax.set_xlabel('Step')
        ax.set_ylabel('High-Order Entropy (bits)')
        ax.legend(fontsize=8)
        ax.grid(True, alpha=0.3)
    
    # Hide unused subplot
    if len(interesting_sims) < 6:
        axes[5].axis('off')
    
    plt.suptitle('High-Order Entropy Evolution Over Time', fontsize=14, weight='bold', y=1.02)
    plt.tight_layout()
    plot_path = output_dir / 'high_order_entropy_timeseries.png'
    plt.savefig(plot_path, dpi=150, bbox_inches='tight')
    plt.close()
    plots['timeseries'] = plot_path.name
    print(f"  ✓ Generated: {plot_path.name}")
    
    return plots


def generate_markdown_report(all_results: Dict[int, List[Dict[str, Any]]], 
                            plots: Dict[str, str], output_dir: Path):
    """Generate markdown report with plots and summary"""
    
    # Collect summary statistics
    all_data = []
    for steps, results in all_results.items():
        for r in results:
            if r['status'] == 'success':
                summary = r['summary']
                all_data.append({
                    'simulation': r['simulation'],
                    'steps': steps,
                    'avg_shannon_entropy': summary.get('avg_entropy', 0),
                    'avg_high_order_entropy': summary.get('avg_high_order_entropy', 0),
                    'total_cpu_time': summary.get('total_cpu_time', 0),
                    'max_memory_mb': summary.get('max_memory_mb', 0),
                })
    
    df = pd.DataFrame(all_data)
    
    md_content = f"""# Computational Life Metrics Report

Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

This report presents comprehensive metrics analysis across all artificial life simulations at multiple step counts ({', '.join(map(str, STEP_COUNTS))} steps).

## Overview

We analyze {len(SIMULATIONS)} different computational life forms across {len(STEP_COUNTS)} different step counts, measuring:
- **Entropy**: Shannon entropy and high-order entropy (captures information from relations between tokens)
- **Computational Cost**: CPU time and memory usage

## Metrics Plots

### Shannon Entropy

Shannon entropy measures the information content of simulation states.

![Shannon Entropy Comparison]({plots['shannon_entropy']})

### High-Order Entropy

High-order entropy = Shannon entropy - normalized Kolmogorov complexity. This metric captures information that can only be explained by relations between different characters/tokens, factoring out information from i.i.d. sampling.

Reference: "Computational Life: How Well-formed, Self-replicating Programs Emerge from Simple Interaction" (arXiv:2406.19108)

![High-Order Entropy Comparison]({plots['high_order_entropy']})

### Computational Cost

Total CPU time required for each simulation across different step counts.

![Computational Cost]({plots['computational_cost']})

### Memory Usage

Maximum memory usage during simulation execution.

![Memory Usage]({plots['memory']})

### Shannon vs High-Order Entropy

Relationship between Shannon entropy and high-order entropy at 500 steps.

![Shannon vs High-Order Entropy]({plots['shannon_vs_hoe']})

### High-Order Entropy Evolution

Time series showing how high-order entropy evolves over time for selected simulations.

![High-Order Entropy Time Series]({plots['timeseries']})

## Summary Statistics

### By Step Count

"""
    
    # Add statistics by step count
    for steps in STEP_COUNTS:
        step_data = df[df['steps'] == steps]
        if len(step_data) > 0:
            md_content += f"""
#### {steps} Steps

- **Average Shannon Entropy**: {step_data['avg_shannon_entropy'].mean():.4f} bits (std: {step_data['avg_shannon_entropy'].std():.4f})
- **Average High-Order Entropy**: {step_data['avg_high_order_entropy'].mean():.4f} bits (std: {step_data['avg_high_order_entropy'].std():.4f})
- **Total CPU Time**: {step_data['total_cpu_time'].sum():.4f} seconds
- **Max Memory**: {step_data['max_memory_mb'].max():.2f} MB

"""
    
    # Add top simulations by metric
    md_content += """
## Top Simulations by Metric

### Highest Shannon Entropy (500 steps)
"""

    data_500 = df[df['steps'] == 500].nlargest(5, 'avg_shannon_entropy')
    for _, row in data_500.iterrows():
        md_content += f"- **{row['simulation']}**: {row['avg_shannon_entropy']:.4f} bits\n"

    md_content += """
### Highest High-Order Entropy (500 steps)
"""

    data_500_hoe = df[df['steps'] == 500].nlargest(5, 'avg_high_order_entropy')
    for _, row in data_500_hoe.iterrows():
        md_content += f"- **{row['simulation']}**: {row['avg_high_order_entropy']:.4f} bits\n"

    md_content += """
### Most Computationally Expensive (500 steps)
"""

    data_500_cpu = df[df['steps'] == 500].nlargest(5, 'total_cpu_time')
    for _, row in data_500_cpu.iterrows():
        md_content += f"- **{row['simulation']}**: {row['total_cpu_time']:.4f} seconds\n"
    
    md_content += f"""
## Data Files

Complete data is available in:
- `metrics_all_steps.csv` - Summary metrics for all simulations and step counts
- `metrics_timeseries.csv` - Time series data (sampled)
- `metrics_all_data.json` - Complete dataset with full time series

## Notes

- High-order entropy can be negative when data is highly compressible (normalized Kolmogorov complexity > Shannon entropy)
- Computational cost scales approximately linearly with step count for most simulations
- Some simulations may fail due to implementation bugs
"""
    
    # Save markdown
    report_path = output_dir / 'METRICS_REPORT.md'
    with open(report_path, 'w') as f:
        f.write(md_content)
    
    print(f"  ✓ Generated: {report_path.name}")
    return report_path


def main():
    """Main function"""
    output_dir = Path('data/processed/metrics_report')
    output_dir.mkdir(parents=True, exist_ok=True)
    
    print("=" * 70)
    print("Generating Comprehensive Metrics Report")
    print("=" * 70)
    
    # Collect data
    all_results = {}
    for steps in STEP_COUNTS:
        print(f"\n{'=' * 70}")
        print(f"Running simulations at {steps} steps...")
        print(f"{'=' * 70}\n")
        
        step_results = []
        for sim_name, (sim_class, config) in SIMULATIONS.items():
            print(f"  [{sim_name}]", end=' ', flush=True)
            result = run_simulation_collect_metrics(sim_name, sim_class, config, steps)
            step_results.append(result)
            if result['status'] == 'success':
                print(f"✓")
            else:
                print(f"✗ {result.get('error', 'Unknown')[:50]}")
        
        all_results[steps] = step_results
    
    # Generate plots
    print(f"\n{'=' * 70}")
    print("Generating plots...")
    print(f"{'=' * 70}\n")
    plots = generate_plots(all_results, output_dir)
    
    # Generate markdown report
    print(f"\n{'=' * 70}")
    print("Generating markdown report...")
    print(f"{'=' * 70}\n")
    report_path = generate_markdown_report(all_results, plots, output_dir)
    
    print(f"\n{'=' * 70}")
    print(f"Report generated: {report_path}")
    print(f"{'=' * 70}\n")


if __name__ == '__main__':
    main()