Portfolio Optimization

services/portfolio-optimizer/
├── src/
│   ├── main.py                        # FastAPI application entry point
│   ├── optimizer/
│   │   ├── mean_variance_optimizer.py # Markowitz mean-variance optimization
│   │   ├── risk_parity_optimizer.py   # Risk parity optimization
│   │   ├── min_variance_optimizer.py  # Minimum variance optimization
│   │   ├── target_vol_optimizer.py    # Target volatility optimization
│   │   └── black_litterman_optimizer.py # Black-Litterman model
│   ├── engine/
│   │   ├── portfolio_engine.py        # Core portfolio optimization engine
│   │   ├── risk_calculator.py         # Risk metrics calculation
│   │   ├── correlation_analyzer.py    # Correlation and covariance analysis
│   │   └── rebalancing_scheduler.py   # Automated rebalancing scheduler
│   ├── api/
│   │   ├── optimizer_api.py           # Portfolio optimization endpoints
│   │   └── rebalancing_api.py         # Rebalancing management endpoints
│   ├── models/
│   │   ├── portfolio_model.py         # Portfolio configuration models
│   │   ├── optimization_model.py      # Optimization result models
│   │   └── rebalancing_model.py       # Rebalancing configuration models
│   ├── data/
│   │   ├── returns_loader.py          # Historical returns data loading
│   │   ├── performance_aggregator.py  # Strategy performance aggregation
│   │   └── data_validator.py          # Data quality validation
│   ├── config.py                      # Configuration management
│   └── requirements.txt               # Python dependencies
├── Dockerfile                         # Container definition
└── docker-compose.yml                 # Local development setup

import cvxpy as cp
import numpy as np
import pandas as pd
from scipy.optimize import minimize

class MeanVarianceOptimizer:
    def __init__(self, risk_aversion=1.0, target_return=None, 
                 max_weight=1.0, min_weight=0.0):
        self.risk_aversion = risk_aversion
        self.target_return = target_return
        self.max_weight = max_weight
        self.min_weight = min_weight
        self.optimization_history = []

    def calculate_expected_returns(self, returns_df, method='historical'):
        """Calculate expected returns using various methods"""
        if method == 'historical':
            return returns_df.mean().values
        elif method == 'exponential':
            # Exponential weighted average
            alpha = 0.94  # RiskMetrics decay factor
            weights = np.array([(1-alpha) * alpha**i for i in range(len(returns_df))])
            weights = weights / weights.sum()
            return np.average(returns_df.values, weights=weights, axis=0)
        elif method == 'shrinkage':
            # James-Stein shrinkage estimator
            sample_mean = returns_df.mean().values
            grand_mean = sample_mean.mean()
            n = len(sample_mean)
            sample_var = returns_df.var().values

            # Shrinkage factor
            shrinkage = (n-3) / (n-1) * sample_var.sum() / ((sample_mean - grand_mean)**2).sum()
            shrinkage = max(0, min(1, shrinkage))

            return shrinkage * grand_mean + (1-shrinkage) * sample_mean

    def calculate_covariance_matrix(self, returns_df, method='sample'):
        """Calculate covariance matrix using various methods"""
        if method == 'sample':
            return returns_df.cov().values
        elif method == 'exponential':
            # Exponential weighted covariance
            alpha = 0.94
            weights = np.array([(1-alpha) * alpha**i for i in range(len(returns_df))])
            weights = weights / weights.sum()

            mean_returns = np.average(returns_df.values, weights=weights, axis=0)
            demeaned_returns = returns_df.values - mean_returns

            cov_matrix = np.zeros((len(mean_returns), len(mean_returns)))
            for i in range(len(mean_returns)):
                for j in range(len(mean_returns)):
                    cov_matrix[i,j] = np.average(
                        demeaned_returns[:,i] * demeaned_returns[:,j], 
                        weights=weights
                    )
            return cov_matrix
        elif method == 'ledoit_wolf':
            # Ledoit-Wolf shrinkage estimator
            sample_cov = returns_df.cov().values
            n, p = returns_df.shape

            # Calculate shrinkage target (constant correlation)
            var_avg = np.trace(sample_cov) / p
            corr_avg = (np.sum(sample_cov) - np.trace(sample_cov)) / (p**2 - p)
            target = corr_avg * np.sqrt(np.outer(np.diag(sample_cov), np.diag(sample_cov)))
            np.fill_diagonal(target, np.diag(sample_cov))

            # Calculate shrinkage intensity
            pi_hat = np.sum([np.sum((returns_df.iloc[i:i+1].T @ returns_df.iloc[i:i+1] - sample_cov)**2) 
                           for i in range(n)]) / n
            gamma_hat = np.sum((target - sample_cov)**2)
            kappa_hat = pi_hat / gamma_hat
            shrinkage = max(0, min(1, kappa_hat / n))

            return shrinkage * target + (1-shrinkage) * sample_cov

    def optimize(self, returns_df, constraints=None):
        """Perform mean-variance optimization"""
        n_assets = len(returns_df.columns)

        # Calculate expected returns and covariance matrix
        expected_returns = self.calculate_expected_returns(returns_df)
        cov_matrix = self.calculate_covariance_matrix(returns_df)

        # Define optimization variables
        weights = cp.Variable(n_assets)

        # Define objective function
        portfolio_return = expected_returns @ weights
        portfolio_risk = cp.quad_form(weights, cov_matrix)

        if self.target_return is not None:
            # Target return optimization
            objective = cp.Minimize(portfolio_risk)
            constraints_list = [
                cp.sum(weights) == 1,
                portfolio_return >= self.target_return,
                weights >= self.min_weight,
                weights <= self.max_weight
            ]
        else:
            # Mean-variance optimization
            objective = cp.Maximize(portfolio_return - self.risk_aversion * portfolio_risk)
            constraints_list = [
                cp.sum(weights) == 1,
                weights >= self.min_weight,
                weights <= self.max_weight
            ]

        # Add custom constraints
        if constraints:
            constraints_list.extend(constraints)

        # Solve optimization problem
        problem = cp.Problem(objective, constraints_list)
        problem.solve()

        if problem.status == 'optimal':
            optimal_weights = weights.value
            portfolio_metrics = {
                'expected_return': portfolio_return.value,
                'volatility': np.sqrt(portfolio_risk.value),
                'sharpe_ratio': portfolio_return.value / np.sqrt(portfolio_risk.value) if portfolio_risk.value > 0 else 0,
                'weights': dict(zip(returns_df.columns, optimal_weights))
            }

            # Record optimization
            self.optimization_history.append({
                'timestamp': pd.Timestamp.now(),
                'method': 'mean_variance',
                'risk_aversion': self.risk_aversion,
                'target_return': self.target_return,
                'metrics': portfolio_metrics
            })

            return portfolio_metrics
        else:
            raise ValueError(f"Optimization failed: {problem.status}")

    def efficient_frontier(self, returns_df, n_points=50):
        """Generate efficient frontier"""
        min_return = returns_df.mean().min()
        max_return = returns_df.mean().max()
        target_returns = np.linspace(min_return, max_return, n_points)

        frontier_points = []
        for target_return in target_returns:
            try:
                self.target_return = target_return
                result = self.optimize(returns_df)
                frontier_points.append({
                    'return': result['expected_return'],
                    'volatility': result['volatility'],
                    'sharpe_ratio': result['sharpe_ratio']
                })
            except:
                continue

        return pd.DataFrame(frontier_points)

class RiskParityOptimizer:
    def __init__(self, target_risk_contrib=None, max_weight=1.0, min_weight=0.0):
        self.target_risk_contrib = target_risk_contrib
        self.max_weight = max_weight
        self.min_weight = min_weight
        self.optimization_history = []

    def calculate_risk_contribution(self, weights, cov_matrix):
        """Calculate risk contribution of each asset"""
        portfolio_variance = weights.T @ cov_matrix @ weights
        marginal_risk = cov_matrix @ weights
        risk_contribution = weights * marginal_risk / np.sqrt(portfolio_variance)
        return risk_contribution

    def risk_parity_objective(self, weights, cov_matrix, target_risk_contrib):
        """Objective function for risk parity optimization"""
        n_assets = len(weights)
        if target_risk_contrib is None:
            target_risk_contrib = np.ones(n_assets) / n_assets

        risk_contrib = self.calculate_risk_contribution(weights, cov_matrix)
        target_contrib = target_risk_contrib / target_risk_contrib.sum()

        # Sum of squared differences from target
        return np.sum((risk_contrib - target_contrib)**2)

    def optimize(self, returns_df, target_risk_contrib=None):
        """Perform risk parity optimization"""
        n_assets = len(returns_df.columns)
        cov_matrix = returns_df.cov().values

        # Initial weights (equal weight)
        initial_weights = np.ones(n_assets) / n_assets

        # Set target risk contribution
        if target_risk_contrib is None:
            target_risk_contrib = np.ones(n_assets) / n_assets

        # Define constraints
        constraints = [
            {'type': 'eq', 'fun': lambda w: np.sum(w) - 1}  # Weights sum to 1
        ]

        # Define bounds
        bounds = [(self.min_weight, self.max_weight) for _ in range(n_assets)]

        # Optimize
        result = minimize(
            self.risk_parity_objective,
            initial_weights,
            args=(cov_matrix, target_risk_contrib),
            method='SLSQP',
            constraints=constraints,
            bounds=bounds,
            options={'ftol': 1e-8, 'maxiter': 1000}
        )

        if result.success:
            optimal_weights = result.x
            portfolio_variance = optimal_weights.T @ cov_matrix @ optimal_weights
            portfolio_volatility = np.sqrt(portfolio_variance)
            expected_return = returns_df.mean().values @ optimal_weights

            portfolio_metrics = {
                'expected_return': expected_return,
                'volatility': portfolio_volatility,
                'sharpe_ratio': expected_return / portfolio_volatility if portfolio_volatility > 0 else 0,
                'weights': dict(zip(returns_df.columns, optimal_weights)),
                'risk_contribution': dict(zip(returns_df.columns, 
                    self.calculate_risk_contribution(optimal_weights, cov_matrix)))
            }

            # Record optimization
            self.optimization_history.append({
                'timestamp': pd.Timestamp.now(),
                'method': 'risk_parity',
                'target_risk_contrib': target_risk_contrib,
                'metrics': portfolio_metrics
            })

            return portfolio_metrics
        else:
            raise ValueError(f"Risk parity optimization failed: {result.message}")

    def risk_budget_optimization(self, returns_df, risk_budget):
        """Optimize with custom risk budget allocation"""
        n_assets = len(returns_df.columns)
        if len(risk_budget) != n_assets:
            raise ValueError("Risk budget length must match number of assets")

        return self.optimize(returns_df, target_risk_contrib=risk_budget)

class MinimumVarianceOptimizer:
    def __init__(self, max_weight=1.0, min_weight=0.0):
        self.max_weight = max_weight
        self.min_weight = min_weight
        self.optimization_history = []

    def optimize(self, returns_df, constraints=None):
        """Perform minimum variance optimization"""
        n_assets = len(returns_df.columns)
        cov_matrix = returns_df.cov().values

        # Define optimization variables
        weights = cp.Variable(n_assets)

        # Define objective function (minimize variance)
        portfolio_variance = cp.quad_form(weights, cov_matrix)
        objective = cp.Minimize(portfolio_variance)

        # Define constraints
        constraints_list = [
            cp.sum(weights) == 1,  # Weights sum to 1
            weights >= self.min_weight,  # Minimum weight constraint
            weights <= self.max_weight   # Maximum weight constraint
        ]

        # Add custom constraints
        if constraints:
            constraints_list.extend(constraints)

        # Solve optimization problem
        problem = cp.Problem(objective, constraints_list)
        problem.solve()

        if problem.status == 'optimal':
            optimal_weights = weights.value
            portfolio_variance = portfolio_variance.value
            portfolio_volatility = np.sqrt(portfolio_variance)
            expected_return = returns_df.mean().values @ optimal_weights

            portfolio_metrics = {
                'expected_return': expected_return,
                'volatility': portfolio_volatility,
                'sharpe_ratio': expected_return / portfolio_volatility if portfolio_volatility > 0 else 0,
                'weights': dict(zip(returns_df.columns, optimal_weights))
            }

            # Record optimization
            self.optimization_history.append({
                'timestamp': pd.Timestamp.now(),
                'method': 'minimum_variance',
                'metrics': portfolio_metrics
            })

            return portfolio_metrics
        else:
            raise ValueError(f"Minimum variance optimization failed: {problem.status}")

    def global_minimum_variance(self, returns_df):
        """Calculate global minimum variance portfolio (analytical solution)"""
        cov_matrix = returns_df.cov().values
        n_assets = len(cov_matrix)

        # Analytical solution for unconstrained minimum variance
        cov_inv = np.linalg.inv(cov_matrix)
        ones = np.ones(n_assets)

        # Optimal weights
        optimal_weights = cov_inv @ ones / (ones.T @ cov_inv @ ones)

        # Calculate metrics
        portfolio_variance = optimal_weights.T @ cov_matrix @ optimal_weights
        portfolio_volatility = np.sqrt(portfolio_variance)
        expected_return = returns_df.mean().values @ optimal_weights

        return {
            'expected_return': expected_return,
            'volatility': portfolio_volatility,
            'sharpe_ratio': expected_return / portfolio_volatility if portfolio_volatility > 0 else 0,
            'weights': dict(zip(returns_df.columns, optimal_weights))
        }

class PortfolioEngine:
    def __init__(self):
        self.optimizers = {
            'mean_variance': MeanVarianceOptimizer(),
            'risk_parity': RiskParityOptimizer(),
            'minimum_variance': MinimumVarianceOptimizer(),
            'target_volatility': TargetVolatilityOptimizer()
        }
        self.current_weights = {}
        self.optimization_history = []
        self.rebalancing_scheduler = RebalancingScheduler()

    def load_performance_data(self, strategy_ids, start_date, end_date):
        """Load historical performance data for optimization"""
        # Load from strategy performance service
        performance_data = {}

        for strategy_id in strategy_ids:
            # Get daily returns from strategy performance service
            returns = self.get_strategy_returns(strategy_id, start_date, end_date)
            performance_data[strategy_id] = returns

        return pd.DataFrame(performance_data)

    def optimize_portfolio(self, strategy_ids, optimization_method='mean_variance', 
                          **optimization_params):
        """Perform portfolio optimization"""
        # Load performance data
        returns_df = self.load_performance_data(strategy_ids, 
                                               start_date='2024-01-01', 
                                               end_date='2024-12-20')

        # Select optimizer
        if optimization_method not in self.optimizers:
            raise ValueError(f"Unknown optimization method: {optimization_method}")

        optimizer = self.optimizers[optimization_method]

        # Set optimizer parameters
        for param, value in optimization_params.items():
            if hasattr(optimizer, param):
                setattr(optimizer, param, value)

        # Perform optimization
        optimization_result = optimizer.optimize(returns_df)

        # Store result
        self.current_weights = optimization_result['weights']
        self.optimization_history.append({
            'timestamp': pd.Timestamp.now(),
            'method': optimization_method,
            'strategy_ids': strategy_ids,
            'result': optimization_result
        })

        return optimization_result

    def calculate_portfolio_metrics(self, weights, returns_df):
        """Calculate comprehensive portfolio metrics"""
        if not weights:
            return {}

        # Convert weights to array
        weight_array = np.array(list(weights.values()))

        # Calculate basic metrics
        expected_returns = returns_df.mean().values
        cov_matrix = returns_df.cov().values

        portfolio_return = expected_returns @ weight_array
        portfolio_variance = weight_array.T @ cov_matrix @ weight_array
        portfolio_volatility = np.sqrt(portfolio_variance)

        # Calculate Sharpe ratio (assuming risk-free rate = 0)
        sharpe_ratio = portfolio_return / portfolio_volatility if portfolio_volatility > 0 else 0

        # Calculate maximum drawdown
        cumulative_returns = (1 + returns_df).cumprod()
        portfolio_cumulative = (cumulative_returns * weight_array).sum(axis=1)
        running_max = portfolio_cumulative.expanding().max()
        drawdown = (portfolio_cumulative - running_max) / running_max
        max_drawdown = drawdown.min()

        # Calculate VaR and CVaR
        portfolio_returns = returns_df @ weight_array
        var_95 = np.percentile(portfolio_returns, 5)
        cvar_95 = portfolio_returns[portfolio_returns <= var_95].mean()

        # Calculate diversification ratio
        individual_volatilities = np.sqrt(np.diag(cov_matrix))
        weighted_individual_vol = np.sum(weight_array * individual_volatilities)
        diversification_ratio = weighted_individual_vol / portfolio_volatility

        return {
            'expected_return': portfolio_return,
            'volatility': portfolio_volatility,
            'sharpe_ratio': sharpe_ratio,
            'max_drawdown': max_drawdown,
            'var_95': var_95,
            'cvar_95': cvar_95,
            'diversification_ratio': diversification_ratio,
            'weights': weights
        }

    def schedule_rebalancing(self, strategy_ids, rebalancing_frequency='weekly', 
                           threshold=0.05):
        """Schedule automatic portfolio rebalancing"""
        return self.rebalancing_scheduler.schedule_rebalancing(
            strategy_ids=strategy_ids,
            frequency=rebalancing_frequency,
            threshold=threshold,
            optimization_method='mean_variance'
        )

    def get_optimization_history(self, limit=100):
        """Get optimization history"""
        return self.optimization_history[-limit:]

    def compare_optimization_methods(self, strategy_ids):
        """Compare different optimization methods"""
        returns_df = self.load_performance_data(strategy_ids, 
                                               start_date='2024-01-01', 
                                               end_date='2024-12-20')

        comparison_results = {}

        for method, optimizer in self.optimizers.items():
            try:
                result = optimizer.optimize(returns_df)
                comparison_results[method] = result
            except Exception as e:
                comparison_results[method] = {'error': str(e)}

        return comparison_results

{
  "optimization_id": "opt_12345",
  "strategy_ids": ["strategy_001", "strategy_002", "strategy_003"],
  "optimization_method": "mean_variance",
  "parameters": {
    "risk_aversion": 1.0,
    "target_return": null,
    "max_weight": 0.4,
    "min_weight": 0.05
  },
  "constraints": {
    "sector_limits": {"tech": 0.3, "finance": 0.4},
    "leverage_limit": 1.5,
    "concentration_limit": 0.25
  },
  "rebalancing": {
    "frequency": "weekly",
    "threshold": 0.05,
    "auto_execute": true
  },
  "created_at": "2024-12-20T10:30:15.123Z"
}

{
  "result_id": "result_12345",
  "optimization_id": "opt_12345",
  "optimization_method": "mean_variance",
  "weights": {
    "strategy_001": 0.35,
    "strategy_002": 0.40,
    "strategy_003": 0.25
  },
  "metrics": {
    "expected_return": 0.0012,
    "volatility": 0.015,
    "sharpe_ratio": 0.08,
    "max_drawdown": -0.025,
    "var_95": -0.018,
    "diversification_ratio": 1.15
  },
  "risk_contribution": {
    "strategy_001": 0.33,
    "strategy_002": 0.38,
    "strategy_003": 0.29
  },
  "timestamp": "2024-12-20T10:30:15.123Z"
}

{
  "rebalancing_id": "rebal_12345",
  "optimization_id": "opt_12345",
  "frequency": "weekly",
  "threshold": 0.05,
  "auto_execute": true,
  "execution_method": "gradual",
  "execution_days": 3,
  "notifications": {
    "email": true,
    "slack": true,
    "threshold_alert": 0.03
  },
  "status": "active",
  "last_rebalancing": "2024-12-13T10:30:15.123Z",
  "next_rebalancing": "2024-12-20T10:30:15.123Z"
}

Strategy Performance Data → Returns Calculation → Correlation Analysis → Optimization Engine → Weight Allocation
    ↓
Portfolio Metrics Calculation → Risk Assessment → Rebalancing Decision → Trade Generation → Execution Monitoring
    ↓
Performance Attribution → Strategy Analysis → Optimization Feedback → Method Selection → Continuous Improvement

POST   /api/v1/portfolio/optimize              # Perform portfolio optimization
GET    /api/v1/portfolio/optimize/{opt_id}     # Get optimization result
PUT    /api/v1/portfolio/optimize/{opt_id}     # Update optimization configuration
DELETE /api/v1/portfolio/optimize/{opt_id}     # Delete optimization configuration

GET    /api/v1/portfolio/weights               # Get current portfolio weights
GET    /api/v1/portfolio/metrics               # Get portfolio performance metrics
GET    /api/v1/portfolio/efficient-frontier    # Generate efficient frontier
POST   /api/v1/portfolio/compare-methods       # Compare optimization methods

POST   /api/v1/portfolio/rebalancing/schedule  # Schedule rebalancing
GET    /api/v1/portfolio/rebalancing/status    # Get rebalancing status
POST   /api/v1/portfolio/rebalancing/execute   # Execute rebalancing
GET    /api/v1/portfolio/rebalancing/history   # Get rebalancing history

/ws/portfolio/weights                           # Real-time weight updates
/ws/portfolio/metrics                           # Real-time metric updates
/ws/portfolio/rebalancing                       # Real-time rebalancing status
/ws/portfolio/alerts                            # Portfolio optimization alerts

Strategy Performance Service → Returns Data → Portfolio Optimizer → Optimal Weights → Account Allocation