visualization_reporting.py

"""
Data Visualization and Reporting Module for Enhanced Portfolio System
Comprehensive visualization and analysis capabilities
"""

import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.offline as pyo
from typing import Dict, List, Optional, Tuple
import os
from enhanced_portfolio_system import Investment, Stock, Bond, Portfolio, Investor
from database_file_manager import DatabaseManager, FileManager

# Set style for better-looking plots
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")

class PortfolioVisualizer:
    """
    Comprehensive portfolio visualization class
    Creates various charts and graphs for portfolio analysis
    """
    
    def __init__(self, output_dir: str = "/home/ubuntu/portfolio_charts"):
        self.output_dir = output_dir
        os.makedirs(output_dir, exist_ok=True)
        
        # Configure matplotlib for better appearance
        plt.rcParams['figure.figsize'] = (12, 8)
        plt.rcParams['font.size'] = 10
        plt.rcParams['axes.grid'] = True
        plt.rcParams['grid.alpha'] = 0.3
    
    def create_portfolio_pie_chart(self, portfolio: Portfolio, filename: str = "portfolio_allocation.png") -> str:
        """Create pie chart showing portfolio allocation by investment"""
        try:
            # Prepare data
            symbols = []
            values = []
            colors = []
            
            # Use a color palette
            color_palette = plt.cm.Set3(np.linspace(0, 1, len(portfolio.investments)))
            
            for i, investment in enumerate(portfolio.investments.values()):
                symbols.append(f"{investment.symbol}\\n({investment.investment_type.value})")
                values.append(investment.get_total_value())
                colors.append(color_palette[i])
            
            # Create pie chart
            fig, ax = plt.subplots(figsize=(10, 8))
            wedges, texts, autotexts = ax.pie(values, labels=symbols, autopct='%1.1f%%', 
                                            colors=colors, startangle=90)
            
            # Enhance appearance
            for autotext in autotexts:
                autotext.set_color('white')
                autotext.set_fontweight('bold')
            
            ax.set_title(f'Portfolio Allocation - {portfolio.name}', 
                        fontsize=16, fontweight='bold', pad=20)
            
            # Add total value annotation
            total_value = sum(values)
            plt.figtext(0.5, 0.02, f'Total Portfolio Value: ${total_value:,.2f}', 
                       ha='center', fontsize=12, style='italic')
            
            plt.tight_layout()
            filepath = os.path.join(self.output_dir, filename)
            plt.savefig(filepath, dpi=300, bbox_inches='tight')
            plt.close()
            
            return filepath
        except Exception as e:
            print(f"Error creating pie chart: {e}")
            return ""
    
    def create_performance_bar_chart(self, portfolio: Portfolio, filename: str = "performance_comparison.png") -> str:
        """Create bar chart comparing investment performance"""
        try:
            # Prepare data
            symbols = []
            earnings = []
            yearly_rates = []
            colors = []
            
            for investment in portfolio.investments.values():
                symbols.append(investment.symbol)
                earnings.append(investment.earnings_loss())
                yearly_rates.append(investment.yearly_earnings_loss_rate())
                # Color based on performance
                colors.append('green' if investment.earnings_loss() >= 0 else 'red')
            
            # Create subplots
            fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10))
            
            # Earnings/Loss bar chart
            bars1 = ax1.bar(symbols, earnings, color=colors, alpha=0.7)
            ax1.set_title('Total Earnings/Loss by Investment', fontsize=14, fontweight='bold')
            ax1.set_ylabel('Earnings/Loss ($)')
            ax1.axhline(y=0, color='black', linestyle='-', alpha=0.3)
            
            # Add value labels on bars
            for bar, value in zip(bars1, earnings):
                height = bar.get_height()
                ax1.text(bar.get_x() + bar.get_width()/2., height + (50 if height >= 0 else -100),
                        f'${value:,.0f}', ha='center', va='bottom' if height >= 0 else 'top')
            
            # Yearly return rate bar chart
            colors2 = ['green' if rate >= 0 else 'red' for rate in yearly_rates]
            bars2 = ax2.bar(symbols, yearly_rates, color=colors2, alpha=0.7)
            ax2.set_title('Yearly Return Rate by Investment', fontsize=14, fontweight='bold')
            ax2.set_ylabel('Yearly Return Rate (%)')
            ax2.set_xlabel('Investment Symbol')
            ax2.axhline(y=0, color='black', linestyle='-', alpha=0.3)
            
            # Add value labels on bars
            for bar, value in zip(bars2, yearly_rates):
                height = bar.get_height()
                ax2.text(bar.get_x() + bar.get_width()/2., height + (0.5 if height >= 0 else -1),
                        f'{value:.1f}%', ha='center', va='bottom' if height >= 0 else 'top')
            
            plt.tight_layout()
            filepath = os.path.join(self.output_dir, filename)
            plt.savefig(filepath, dpi=300, bbox_inches='tight')
            plt.close()
            
            return filepath
        except Exception as e:
            print(f"Error creating performance bar chart: {e}")
            return ""
    
    def create_risk_return_scatter(self, portfolios: List[Portfolio], filename: str = "risk_return_analysis.png") -> str:
        """Create scatter plot showing risk vs return for multiple portfolios"""
        try:
            fig, ax = plt.subplots(figsize=(10, 8))
            
            for portfolio in portfolios:
                summary = portfolio.get_portfolio_summary()
                if 'basic_metrics' in summary and 'risk_metrics' in summary:
                    risk = summary['risk_metrics']['volatility']
                    return_rate = summary['basic_metrics']['average_yearly_return']
                    
                    ax.scatter(risk, return_rate, s=100, alpha=0.7, label=portfolio.name)
                    ax.annotate(portfolio.name, (risk, return_rate), 
                              xytext=(5, 5), textcoords='offset points')
            
            ax.set_xlabel('Risk (Volatility %)', fontsize=12)
            ax.set_ylabel('Expected Return (%)', fontsize=12)
            ax.set_title('Risk vs Return Analysis', fontsize=14, fontweight='bold')
            ax.grid(True, alpha=0.3)
            ax.legend()
            
            # Add quadrant lines
            ax.axhline(y=0, color='black', linestyle='-', alpha=0.3)
            ax.axvline(x=0, color='black', linestyle='-', alpha=0.3)
            
            plt.tight_layout()
            filepath = os.path.join(self.output_dir, filename)
            plt.savefig(filepath, dpi=300, bbox_inches='tight')
            plt.close()
            
            return filepath
        except Exception as e:
            print(f"Error creating risk-return scatter plot: {e}")
            return ""
    
    def create_sector_analysis(self, portfolio: Portfolio, filename: str = "sector_analysis.png") -> str:
        """Create sector allocation analysis for stock investments"""
        try:
            # Collect sector data
            sector_data = {}
            for investment in portfolio.investments.values():
                if isinstance(investment, Stock):
                    sector = investment.sector
                    if sector not in sector_data:
                        sector_data[sector] = {'value': 0, 'count': 0, 'earnings': 0}
                    
                    sector_data[sector]['value'] += investment.get_total_value()
                    sector_data[sector]['count'] += 1
                    sector_data[sector]['earnings'] += investment.earnings_loss()
            
            if not sector_data:
                print("No sector data available for visualization")
                return ""
            
            # Create subplots
            fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))
            
            sectors = list(sector_data.keys())
            values = [sector_data[s]['value'] for s in sectors]
            counts = [sector_data[s]['count'] for s in sectors]
            earnings = [sector_data[s]['earnings'] for s in sectors]
            
            # Sector allocation pie chart
            ax1.pie(values, labels=sectors, autopct='%1.1f%%', startangle=90)
            ax1.set_title('Sector Allocation by Value', fontweight='bold')
            
            # Sector count bar chart
            ax2.bar(sectors, counts, color='skyblue', alpha=0.7)
            ax2.set_title('Number of Investments by Sector', fontweight='bold')
            ax2.set_ylabel('Number of Investments')
            plt.setp(ax2.xaxis.get_majorticklabels(), rotation=45)
            
            # Sector earnings bar chart
            colors = ['green' if e >= 0 else 'red' for e in earnings]
            ax3.bar(sectors, earnings, color=colors, alpha=0.7)
            ax3.set_title('Earnings/Loss by Sector', fontweight='bold')
            ax3.set_ylabel('Earnings/Loss ($)')
            ax3.axhline(y=0, color='black', linestyle='-', alpha=0.3)
            plt.setp(ax3.xaxis.get_majorticklabels(), rotation=45)
            
            # Sector performance comparison
            avg_returns = []
            for sector in sectors:
                sector_investments = [inv for inv in portfolio.investments.values() 
                                    if isinstance(inv, Stock) and inv.sector == sector]
                if sector_investments:
                    avg_return = sum(inv.yearly_earnings_loss_rate() for inv in sector_investments) / len(sector_investments)
                    avg_returns.append(avg_return)
                else:
                    avg_returns.append(0)
            
            colors = ['green' if r >= 0 else 'red' for r in avg_returns]
            ax4.bar(sectors, avg_returns, color=colors, alpha=0.7)
            ax4.set_title('Average Yearly Return by Sector', fontweight='bold')
            ax4.set_ylabel('Average Yearly Return (%)')
            ax4.axhline(y=0, color='black', linestyle='-', alpha=0.3)
            plt.setp(ax4.xaxis.get_majorticklabels(), rotation=45)
            
            plt.tight_layout()
            filepath = os.path.join(self.output_dir, filename)
            plt.savefig(filepath, dpi=300, bbox_inches='tight')
            plt.close()
            
            return filepath
        except Exception as e:
            print(f"Error creating sector analysis: {e}")
            return ""
    
    def create_time_series_chart(self, db_manager: DatabaseManager, symbols: List[str], 
                               filename: str = "price_trends.png") -> str:
        """Create time series chart for price trends"""
        try:
            fig, ax = plt.subplots(figsize=(14, 8))
            
            for symbol in symbols:
                market_data = db_manager.get_market_data(symbol)
                if market_data:
                    dates = [datetime.strptime(d['date'], '%Y-%m-%d') for d in market_data]
                    prices = [d['close_price'] for d in market_data]
                    
                    ax.plot(dates, prices, marker='o', markersize=3, label=symbol, linewidth=2)
            
            ax.set_xlabel('Date', fontsize=12)
            ax.set_ylabel('Price ($)', fontsize=12)
            ax.set_title('Price Trends Over Time', fontsize=14, fontweight='bold')
            ax.legend()
            ax.grid(True, alpha=0.3)
            
            # Format x-axis
            ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
            ax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))
            plt.xticks(rotation=45)
            
            plt.tight_layout()
            filepath = os.path.join(self.output_dir, filename)
            plt.savefig(filepath, dpi=300, bbox_inches='tight')
            plt.close()
            
            return filepath
        except Exception as e:
            print(f"Error creating time series chart: {e}")
            return ""
    
    def create_interactive_dashboard(self, portfolio: Portfolio, filename: str = "interactive_dashboard.html") -> str:
        """Create interactive dashboard using Plotly"""
        try:
            # Prepare data
            symbols = []
            values = []
            earnings = []
            yearly_rates = []
            investment_types = []
            
            for investment in portfolio.investments.values():
                symbols.append(investment.symbol)
                values.append(investment.get_total_value())
                earnings.append(investment.earnings_loss())
                yearly_rates.append(investment.yearly_earnings_loss_rate())
                investment_types.append(investment.investment_type.value)
            
            # Create subplots
            fig = make_subplots(
                rows=2, cols=2,
                subplot_titles=('Portfolio Allocation', 'Performance Comparison', 
                              'Earnings Distribution', 'Return Rates'),
                specs=[[{"type": "pie"}, {"type": "bar"}],
                       [{"type": "bar"}, {"type": "scatter"}]]
            )
            
            # Portfolio allocation pie chart
            fig.add_trace(
                go.Pie(labels=symbols, values=values, name="Allocation"),
                row=1, col=1
            )
            
            # Performance bar chart
            colors = ['green' if e >= 0 else 'red' for e in earnings]
            fig.add_trace(
                go.Bar(x=symbols, y=earnings, name="Earnings", marker_color=colors),
                row=1, col=2
            )
            
            # Earnings distribution
            fig.add_trace(
                go.Bar(x=symbols, y=values, name="Total Value", marker_color='blue'),
                row=2, col=1
            )
            
            # Return rates scatter
            fig.add_trace(
                go.Scatter(x=symbols, y=yearly_rates, mode='markers+lines', 
                          name="Yearly Return", marker_size=10),
                row=2, col=2
            )
            
            # Update layout
            fig.update_layout(
                title_text=f"Portfolio Dashboard - {portfolio.name}",
                title_x=0.5,
                height=800,
                showlegend=False
            )
            
            # Save interactive chart
            filepath = os.path.join(self.output_dir, filename)
            pyo.plot(fig, filename=filepath, auto_open=False)
            
            return filepath
        except Exception as e:
            print(f"Error creating interactive dashboard: {e}")
            return ""

class ReportGenerator:
    """
    Advanced report generation class
    Creates comprehensive analysis reports
    """
    
    def __init__(self, output_dir: str = "/home/ubuntu/portfolio_reports"):
        self.output_dir = output_dir
        os.makedirs(output_dir, exist_ok=True)
    
    def generate_comprehensive_report(self, investor: Investor, filename: str = "comprehensive_report.html") -> str:
        """Generate comprehensive HTML report"""
        try:
            filepath = os.path.join(self.output_dir, filename)
            
            # Get investor summary
            summary = investor.get_investor_summary()
            
            html_content = f"""
            <!DOCTYPE html>
            <html>
            <head>
                <title>Portfolio Analysis Report</title>
                <style>
                    body {{ font-family: Arial, sans-serif; margin: 40px; }}
                    .header {{ background-color: #f0f0f0; padding: 20px; border-radius: 5px; }}
                    .section {{ margin: 20px 0; }}
                    .metric {{ display: inline-block; margin: 10px; padding: 15px; 
                             background-color: #e8f4f8; border-radius: 5px; min-width: 150px; }}
                    .positive {{ color: green; }}
                    .negative {{ color: red; }}
                    table {{ border-collapse: collapse; width: 100%; }}
                    th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
                    th {{ background-color: #f2f2f2; }}
                </style>
            </head>
            <body>
                <div class="header">
                    <h1>Portfolio Analysis Report</h1>
                    <h2>Investor: {summary['investor_info']['name']}</h2>
                    <p>Email: {summary['investor_info']['email']}</p>
                    <p>Report Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
                </div>
                
                <div class="section">
                    <h3>Executive Summary</h3>
                    <div class="metric">
                        <strong>Total Portfolio Value</strong><br>
                        ${summary['total_value']:,.2f}
                    </div>
                    <div class="metric">
                        <strong>Number of Portfolios</strong><br>
                        {summary['portfolio_count']}
                    </div>
                    <div class="metric">
                        <strong>Total Investments</strong><br>
                        {summary['total_investments']}
                    </div>
                    <div class="metric">
                        <strong>Average Return</strong><br>
                        <span class="{'positive' if summary['metrics']['average_yearly_return'] >= 0 else 'negative'}">
                            {summary['metrics']['average_yearly_return']:.2f}%
                        </span>
                    </div>
                </div>
            """
            
            # Add portfolio details
            html_content += """
                <div class="section">
                    <h3>Portfolio Details</h3>
                    <table>
                        <tr>
                            <th>Portfolio</th>
                            <th>Investments</th>
                            <th>Total Value</th>
                            <th>Total Earnings</th>
                            <th>Return %</th>
                        </tr>
            """
            
            for portfolio in investor.portfolios.values():
                port_summary = portfolio.get_portfolio_summary()
                metrics = port_summary['basic_metrics']
                
                html_content += f"""
                        <tr>
                            <td>{portfolio.name}</td>
                            <td>{len(portfolio.investments)}</td>
                            <td>${metrics['total_value']:,.2f}</td>
                            <td class="{'positive' if metrics['total_earnings'] >= 0 else 'negative'}">
                                ${metrics['total_earnings']:,.2f}
                            </td>
                            <td class="{'positive' if metrics['total_return_percentage'] >= 0 else 'negative'}">
                                {metrics['total_return_percentage']:.2f}%
                            </td>
                        </tr>
                """
            
            html_content += """
                    </table>
                </div>
                
                <div class="section">
                    <h3>Risk Analysis</h3>
            """
            
            if 'risk_metrics' in summary:
                risk = summary['risk_metrics']
                html_content += f"""
                    <div class="metric">
                        <strong>Portfolio Volatility</strong><br>
                        {risk['volatility']:.2f}%
                    </div>
                    <div class="metric">
                        <strong>Sharpe Ratio</strong><br>
                        {risk['sharpe_ratio']:.2f}
                    </div>
                    <div class="metric">
                        <strong>Max Return</strong><br>
                        {risk['max_return']:.2f}%
                    </div>
                    <div class="metric">
                        <strong>Min Return</strong><br>
                        {risk['min_return']:.2f}%
                    </div>
                """
            
            html_content += """
                </div>
                
                <div class="section">
                    <h3>Investment Breakdown</h3>
                    <table>
                        <tr>
                            <th>Symbol</th>
                            <th>Type</th>
                            <th>Shares</th>
                            <th>Purchase Price</th>
                            <th>Current Value</th>
                            <th>Total Value</th>
                            <th>Earnings/Loss</th>
                            <th>Yearly Return %</th>
                        </tr>
            """
            
            # Add individual investments
            for portfolio in investor.portfolios.values():
                for investment in portfolio.investments.values():
                    earnings = investment.earnings_loss()
                    yearly_rate = investment.yearly_earnings_loss_rate()
                    
                    html_content += f"""
                        <tr>
                            <td>{investment.symbol}</td>
                            <td>{investment.investment_type.value}</td>
                            <td>{investment.shares}</td>
                            <td>${investment.purchase_price:.2f}</td>
                            <td>${investment.current_value:.2f}</td>
                            <td>${investment.get_total_value():,.2f}</td>
                            <td class="{'positive' if earnings >= 0 else 'negative'}">
                                ${earnings:,.2f}
                            </td>
                            <td class="{'positive' if yearly_rate >= 0 else 'negative'}">
                                {yearly_rate:.2f}%
                            </td>
                        </tr>
                    """
            
            html_content += """
                    </table>
                </div>
                
                <div class="section">
                    <h3>Formulas Used</h3>
                    <p><strong>Earnings/Loss Calculation:</strong><br>
                    (current_value - purchase_price) × number_of_shares</p>
                    
                    <p><strong>Yearly Earnings/Loss Rate:</strong><br>
                    ((((current_value - purchase_price) / purchase_price) / (current_date - purchase_date))) × 100</p>
                </div>
                
            </body>
            </html>
            """
            
            with open(filepath, 'w') as f:
                f.write(html_content)
            
            return filepath
        except Exception as e:
            print(f"Error generating comprehensive report: {e}")
            return ""

if __name__ == "__main__":
    print("Data Visualization and Reporting Module - Testing")
    print("=" * 55)
    
    # Create test data
    from enhanced_portfolio_system import Investor, Portfolio, Stock, Bond
    
    # Create investor with multiple investments
    investor = Investor("VIZ001", "Visualization Test User", "viz@email.com")
    portfolio = investor.create_portfolio("VIZ_PORT", "Visualization Portfolio")
    
    # Add diverse investments
    stocks = [
        Stock("STK001", "AAPL", 100, 150.0, 175.0, "01/15/2024", "Technology", 0.5),
        Stock("STK002", "GOOGL", 50, 2500.0, 2750.0, "02/20/2024", "Technology", 0.0),
        Stock("STK003", "JPM", 75, 120.0, 135.0, "03/10/2024", "Financial", 2.5),
        Stock("STK004", "JNJ", 60, 160.0, 155.0, "01/25/2024", "Healthcare", 2.8),
    ]
    
    bonds = [
        Bond("BND001", "US10Y", 10, 1000.0, 980.0, "03/10/2024", 4.5, "03/10/2034", "AAA"),
        Bond("BND002", "CORP", 5, 1000.0, 1050.0, "02/15/2024", 5.2, "02/15/2029", "BBB"),
    ]
    
    # Add all investments
    for stock in stocks:
        portfolio.add_investment(stock)
    for bond in bonds:
        portfolio.add_investment(bond)
    
    # Test visualizations
    visualizer = PortfolioVisualizer()
    report_gen = ReportGenerator()
    
    print("Creating portfolio visualizations...")
    
    # Create various charts
    pie_chart = visualizer.create_portfolio_pie_chart(portfolio)
    performance_chart = visualizer.create_performance_bar_chart(portfolio)
    sector_chart = visualizer.create_sector_analysis(portfolio)
    dashboard = visualizer.create_interactive_dashboard(portfolio)
    
    print("Generating comprehensive report...")
    report = report_gen.generate_comprehensive_report(investor)
    
    print(f"Visualizations created in: {visualizer.output_dir}")
    print(f"Reports generated in: {report_gen.output_dir}")
    print("Data visualization and reporting module completed successfully!")