#Django部署最佳实践 - 生产环境部署与运维

"""
典型的Django生产环境架构：

Internet
    ↓
Load Balancer (如：AWS ELB, Nginx)
    ↓
Reverse Proxy (Nginx)
    ↓
Application Servers (Gunicorn Workers)
    ↓
Database Cluster (PostgreSQL/MySQL)
    ↓
Cache Service (Redis/Memcached)
    ↓
Message Queue (RabbitMQ/Celery)
    ↓
File Storage (S3/MinIO)

组件职责：
- Nginx: 静态文件服务、负载均衡、SSL终止
- Gunicorn: WSGI应用服务器
- PostgreSQL: 数据存储
- Redis: 缓存、会话存储、消息队列
- Celery: 异步任务处理
"""

"""
Django部署核心原则：

1. 安全第一 (Security First)
   - HTTPS强制
   - 最小权限原则
   - 安全配置验证

2. 高可用性 (High Availability)
   - 多实例部署
   - 故障转移
   - 负载均衡

3. 可扩展性 (Scalability)
   - 水平扩展能力
   - 资源隔离
   - 微服务友好

4. 监控完善 (Comprehensive Monitoring)
   - 性能监控
   - 错误追踪
   - 日志收集

5. 自动化 (Automation)
   - CI/CD流水线
   - 容器化部署
   - 配置管理
"""

"""
环境类型及其特点：

Development (开发环境)
├── DEBUG = True
├── 本地数据库
├── 无生产数据
└── 详细的错误信息

Staging (预发布环境)
├── 生产配置副本
├── 模拟生产数据
├── 相似的硬件配置
└── 用于测试部署

Production (生产环境)
├── DEBUG = False
├── 优化的性能配置
├── 真实用户数据
└── 严格的安全措施

各环境配置差异：
- 数据库连接
- 缓存配置
- 邮件设置
- 第三方服务凭证
- 性能优化参数
"""

# 环境变量配置
import os
from decouple import config  # pip install python-decouple
from pathlib import Path

BASE_DIR = Path(__file__).resolve().parent.parent

# 基础配置
DEBUG = config('DEBUG', default=False, cast=bool)
SECRET_KEY = config('SECRET_KEY', default='your-secret-key-here')
ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='localhost').split(',')

# 数据库配置
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql',
        'NAME': config('DB_NAME', default='myapp'),
        'USER': config('DB_USER', default='myappuser'),
        'PASSWORD': config('DB_PASSWORD'),
        'HOST': config('DB_HOST', default='localhost'),
        'PORT': config('DB_PORT', default='5432'),
        'CONN_MAX_AGE': config('DB_CONN_MAX_AGE', default=60, cast=int),
    }
}

# 缓存配置
CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': config('REDIS_URL', default='redis://127.0.0.1:6379/1'),
        'OPTIONS': {
            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
        },
        'KEY_PREFIX': 'myapp',
        'TIMEOUT': 300,
    }
}

# 邮件配置
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_HOST = config('EMAIL_HOST', default='localhost')
EMAIL_PORT = config('EMAIL_PORT', default=587, cast=int)
EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=True, cast=bool)
EMAIL_HOST_USER = config('EMAIL_HOST_USER')
EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD')

# AWS S3配置（可选）
if config('USE_S3', default=False, cast=bool):
    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
    AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID')
    AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY')
    AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME')
    AWS_S3_REGION_NAME = config('AWS_S3_REGION_NAME', default='us-east-1')
    AWS_S3_CUSTOM_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com'

# gunicorn.conf.py
import multiprocessing

# Server socket
bind = "0.0.0.0:8000"
backlog = 2048

# Worker processes
workers = multiprocessing.cpu_count() * 2 + 1
worker_class = "sync"
worker_connections = 1000
max_requests = 1000
max_requests_jitter = 100
timeout = 30
keepalive = 2
max_keepalive_requests = 100
max_keepalive_time = 2

# Security
limit_request_line = 4094
limit_request_fields = 100
limit_request_field_size = 8190

# Debugging
preload_app = True
daemon = False
raw_env = [
    'DJANGO_SETTINGS_MODULE=myproject.settings.production',
]

# Logging
accesslog = "/var/log/myapp/gunicorn_access.log"
errorlog = "/var/log/myapp/gunicorn_error.log"
loglevel = "info"
access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)s'

# Process naming
proc_name = 'myapp_gunicorn'

# Server mechanics
pidfile = '/var/run/myapp/gunicorn.pid'
user = 'app'
group = 'app'
tmp_upload_dir = None

# SSL
keyfile = None
certfile = None

# advanced_gunicorn.conf.py
import multiprocessing
import os

# 服务器配置
bind = ["0.0.0.0:8000"]
backlog = 2048

# 工作进程配置
workers = multiprocessing.cpu_count() * 2 + 1
worker_class = "gevent"  # 或 "eventlet" 用于异步支持
worker_connections = 1000
max_requests = 1000
max_requests_jitter = 100
timeout = 30
graceful_timeout = 30
keepalive = 2
max_keepalive_requests = 100
max_keepalive_time = 2

# 资源限制
limit_request_line = 4094
limit_request_fields = 100
limit_request_field_size = 8190

# 预加载应用（提高性能，但不利于热重载）
preload_app = True

# 守护进程配置
daemon = False
pidfile = '/var/run/myapp/gunicorn.pid'
user = 'app'
group = 'app'

# 日志配置
accesslog = '/var/log/myapp/gunicorn_access.log'
errorlog = '/var/log/myapp/gunicorn_error.log'
loglevel = 'info'
access_log_format = (
    '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s '
    '"%(f)s" "%(a)s" %(D)s %(p)s'
)

# 应用配置
raw_env = [
    'DJANGO_SETTINGS_MODULE=myproject.settings.production',
]

# 工作进程生命周期
max_requests = 1000
max_requests_jitter = 100
reload = False

# 服务器性能优化
worker_tmp_dir = '/dev/shm'  # 使用内存临时目录

# gunicorn_tuning.py
"""
Gunicorn性能调优指南：

1. 工作进程数 (workers)
   - 公式: (2 × CPU核心数) + 1
   - 过多会导致上下文切换开销
   - 过少无法充分利用CPU

2. 工作进程类型 (worker_class)
   - sync: 同步阻塞，默认类型
   - gevent: 协程，适合I/O密集型
   - eventlet: 协程，类似gevent

3. 连接数 (worker_connections)
   - gevent/eventlet: 设置为1000+
   - sync: 通常不需要调整

4. 请求限制 (max_requests, max_requests_jitter)
   - 防止内存泄漏
   - 随机抖动避免同时重启

5. 超时设置 (timeout, graceful_timeout)
   - timeout: 请求处理超时
   - graceful_timeout: 优雅关闭超时
"""

# backup_manager.py
import subprocess
import os
import datetime
from django.conf import settings
import logging

logger = logging.getLogger(__name__)

class DatabaseBackupManager:
    """数据库备份管理器"""
    
    def __init__(self):
        self.db_name = settings.DATABASES['default']['NAME']
        self.db_user = settings.DATABASES['default']['USER']
        self.db_host = settings.DATABASES['default']['HOST']
        self.backup_dir = '/backups'
    
    def create_backup(self):
        """创建数据库备份"""
        timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
        backup_file = f"{self.backup_dir}/{self.db_name}_backup_{timestamp}.sql"
        
        try:
            # 创建备份目录
            os.makedirs(self.backup_dir, exist_ok=True)
            
            # 执行备份命令
            cmd = [
                'pg_dump',
                '-h', self.db_host,
                '-U', self.db_user,
                '-d', self.db_name,
                '-f', backup_file
            ]
            
            env = os.environ.copy()
            env['PGPASSWORD'] = settings.DATABASES['default']['PASSWORD']
            
            result = subprocess.run(cmd, env=env, capture_output=True, text=True)
            
            if result.returncode == 0:
                logger.info(f"数据库备份成功: {backup_file}")
                
                # 压缩备份文件
                self.compress_backup(backup_file)
                
                # 清理旧备份
                self.cleanup_old_backups()
                
                return backup_file
            else:
                logger.error(f"数据库备份失败: {result.stderr}")
                return None
                
        except Exception as e:
            logger.error(f"备份过程中发生错误: {str(e)}")
            return None
    
    def compress_backup(self, backup_file):
        """压缩备份文件"""
        import gzip
        import shutil
        
        compressed_file = f"{backup_file}.gz"
        
        with open(backup_file, 'rb') as f_in:
            with gzip.open(compressed_file, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
        
        # 删除原始备份文件
        os.remove(backup_file)
        logger.info(f"备份文件已压缩: {compressed_file}")
    
    def cleanup_old_backups(self, days_to_keep=7):
        """清理旧备份"""
        import glob
        from datetime import datetime, timedelta
        
        cutoff_date = datetime.now() - timedelta(days=days_to_keep)
        
        backup_pattern = f"{self.backup_dir}/{self.db_name}_backup_*.sql.gz"
        backup_files = glob.glob(backup_pattern)
        
        for backup_file in backup_files:
            file_time = datetime.fromtimestamp(os.path.getmtime(backup_file))
            if file_time < cutoff_date:
                os.remove(backup_file)
                logger.info(f"删除旧备份: {backup_file}")
    
    def restore_backup(self, backup_file):
        """恢复数据库备份"""
        try:
            cmd = [
                'psql',
                '-h', self.db_host,
                '-U', self.db_user,
                '-d', self.db_name,
                '-f', backup_file
            ]
            
            env = os.environ.copy()
            env['PGPASSWORD'] = settings.DATABASES['default']['PASSWORD']
            
            result = subprocess.run(cmd, env=env, capture_output=True, text=True)
            
            if result.returncode == 0:
                logger.info(f"数据库恢复成功: {backup_file}")
                return True
            else:
                logger.error(f"数据库恢复失败: {result.stderr}")
                return False
                
        except Exception as e:
            logger.error(f"恢复过程中发生错误: {str(e)}")
            return False

# 定时备份任务
from celery import shared_task

@shared_task
def scheduled_backup():
    """定时备份任务"""
    manager = DatabaseBackupManager()
    backup_file = manager.create_backup()
    
    if backup_file:
        # 发送备份成功通知
        from django.core.mail import send_mail
        send_mail(
            '数据库备份成功',
            f'数据库备份已创建: {backup_file}',
            'admin@yourdomain.com',
            ['admin@yourdomain.com'],
        )
    
    return backup_file

# cache_monitor.py
import redis
import time
import logging
from django.conf import settings

logger = logging.getLogger(__name__)

class CacheMonitor:
    """缓存监控器"""
    
    def __init__(self):
        self.redis_client = redis.Redis(
            host=settings.CACHES['default']['LOCATION'].split(':')[1],
            port=settings.CACHES['default']['LOCATION'].split(':')[2],
            db=0
        )
    
    def get_cache_stats(self):
        """获取缓存统计信息"""
        info = self.redis_client.info()
        
        stats = {
            'used_memory': info['used_memory_human'],
            'used_memory_peak': info['used_memory_peak_human'],
            'connected_clients': info['connected_clients'],
            'total_commands_processed': info['total_commands_processed'],
            'keyspace_hits': info['keyspace_hits'],
            'keyspace_misses': info['keyspace_misses'],
            'hit_rate': self.calculate_hit_rate(info),
            'expired_keys': info['expired_keys'],
            'evicted_keys': info['evicted_keys'],
        }
        
        return stats
    
    def calculate_hit_rate(self, info):
        """计算缓存命中率"""
        hits = info['keyspace_hits']
        misses = info['keyspace_misses']
        total = hits + misses
        
        if total == 0:
            return 0.0
        
        return round((hits / total) * 100, 2)
    
    def monitor_cache_health(self):
        """监控缓存健康状态"""
        stats = self.get_cache_stats()
        
        # 检查内存使用率
        memory_percent = (int(stats['used_memory'][:-1]) / 2) * 100  # 假设最大2GB
        if memory_percent > 80:
            logger.warning(f"Redis内存使用率过高: {memory_percent}%")
        
        # 检查命中率
        if stats['hit_rate'] < 70:
            logger.warning(f"Redis缓存命中率低: {stats['hit_rate']}%")
        
        # 检查驱逐键数量
        if stats['evicted_keys'] > 0:
            logger.warning(f"Redis有键被驱逐: {stats['evicted_keys']}")
        
        return stats
    
    def get_top_keys(self, count=10):
        """获取最常用的键"""
        # 这需要Redis配置开启KEYSPACE通知
        # notify-keyspace-events Ex
        pass
    
    def cleanup_expired_keys(self):
        """清理过期键"""
        # Redis会自动清理过期键，这里可以执行额外的清理操作
        pass

# 缓存性能测试
def benchmark_cache_performance():
    """缓存性能基准测试"""
    import time
    
    cache_monitor = CacheMonitor()
    redis_client = cache_monitor.redis_client
    
    # 测试写入性能
    start_time = time.time()
    for i in range(1000):
        redis_client.set(f'test_key_{i}', f'test_value_{i}')
    write_time = time.time() - start_time
    
    # 测试读取性能
    start_time = time.time()
    for i in range(1000):
        redis_client.get(f'test_key_{i}')
    read_time = time.time() - start_time
    
    # 测试删除性能
    start_time = time.time()
    for i in range(1000):
        redis_client.delete(f'test_key_{i}')
    delete_time = time.time() - start_time
    
    print(f"写入1000个键耗时: {write_time:.4f}秒")
    print(f"读取1000个键耗时: {read_time:.4f}秒") 
    print(f"删除1000个键耗时: {delete_time:.4f}秒")

# logging_config.py
"""
# settings.py 中的日志配置
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
            'style': '{',
        },
        'simple': {
            'format': '{levelname} {message}',
            'style': '{',
        },
        'json': {
            '()': 'pythonjsonlogger.jsonlogger.JsonFormatter',
            'format': '%(asctime)s %(name)s %(levelname)s %(filename)s %(lineno)d %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
        'file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/myapp/app.log',
            'maxBytes': 1024*1024*10,  # 10MB
            'backupCount': 5,
            'formatter': 'verbose',
        },
        'error_file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/myapp/error.log',
            'maxBytes': 1024*1024*10,  # 10MB
            'backupCount': 5,
            'level': 'ERROR',
            'formatter': 'verbose',
        },
        'performance_file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/myapp/performance.log',
            'maxBytes': 1024*1024*10,  # 10MB
            'backupCount': 5,
            'level': 'INFO',
            'formatter': 'verbose',
        },
    },
    'root': {
        'handlers': ['console', 'file'],
        'level': 'INFO',
    },
    'loggers': {
        'django': {
            'handlers': ['file'],
            'level': 'INFO',
            'propagate': False,
        },
        'django.request': {
            'handlers': ['error_file'],
            'level': 'ERROR',
            'propagate': False,
        },
        'performance': {
            'handlers': ['performance_file'],
            'level': 'INFO',
            'propagate': False,
        },
        'myapp': {
            'handlers': ['file', 'error_file'],
            'level': 'INFO',
            'propagate': False,
        },
    },
}
"""

# system_monitor.py
import psutil
import time
import logging
from threading import Thread
import smtplib
from email.mime.text import MIMEText
from django.conf import settings

logger = logging.getLogger(__name__)

class SystemMonitor:
    """系统监控器"""
    
    def __init__(self):
        self.alert_thresholds = {
            'cpu_percent': 80,
            'memory_percent': 85,
            'disk_percent': 90,
            'process_count': 500,
        }
        self.monitoring = False
    
    def get_system_stats(self):
        """获取系统统计信息"""
        stats = {
            'timestamp': time.time(),
            'cpu_percent': psutil.cpu_percent(interval=1),
            'memory_percent': psutil.virtual_memory().percent,
            'disk_percent': psutil.disk_usage('/').percent,
            'process_count': len(psutil.pids()),
            'load_average': psutil.getloadavg(),
            'network_io': psutil.net_io_counters(),
            'boot_time': psutil.boot_time(),
        }
        
        return stats
    
    def check_alerts(self, stats):
        """检查是否触发警报"""
        alerts = []
        
        for metric, threshold in self.alert_thresholds.items():
            if stats[metric] > threshold:
                alert_msg = f"ALERT: {metric} is {stats[metric]}%, threshold is {threshold}%"
                alerts.append(alert_msg)
                logger.warning(alert_msg)
        
        # 检查Django进程
        django_processes = [p for p in psutil.process_iter() 
                           if 'gunicorn' in p.name() or 'python' in p.name()]
        if len(django_processes) == 0:
            alert_msg = "ALERT: No Django processes running"
            alerts.append(alert_msg)
            logger.error(alert_msg)
        
        return alerts
    
    def send_alert_email(self, alerts):
        """发送警报邮件"""
        if not alerts:
            return
        
        subject = f"系统警报 - {len(alerts)} 个问题"
        body = "\n".join(alerts)
        
        try:
            msg = MIMEText(body)
            msg['Subject'] = subject
            msg['From'] = settings.SERVER_EMAIL
            msg['To'] = settings.ADMINS[0][1] if settings.ADMINS else settings.SERVER_EMAIL
            
            server = smtplib.SMTP(settings.EMAIL_HOST, settings.EMAIL_PORT)
            server.starttls()
            server.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
            server.send_message(msg)
            server.quit()
            
            logger.info("警报邮件发送成功")
        except Exception as e:
            logger.error(f"发送警报邮件失败: {str(e)}")
    
    def start_monitoring(self):
        """开始监控"""
        self.monitoring = True
        
        def monitor_loop():
            while self.monitoring:
                try:
                    stats = self.get_system_stats()
                    alerts = self.check_alerts(stats)
                    
                    if alerts:
                        self.send_alert_email(alerts)
                    
                    time.sleep(60)  # 每分钟检查一次
                    
                except Exception as e:
                    logger.error(f"监控循环错误: {str(e)}")
                    time.sleep(60)
        
        monitor_thread = Thread(target=monitor_loop, daemon=True)
        monitor_thread.start()
    
    def stop_monitoring(self):
        """停止监控"""
        self.monitoring = False

# 性能监控中间件
class PerformanceMonitoringMiddleware:
    """性能监控中间件"""
    
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        import time
        from django.db import connection
        
        start_time = time.time()
        start_queries = len(connection.queries)
        
        response = self.get_response(request)
        
        duration = time.time() - start_time
        query_count = len(connection.queries) - start_queries
        
        # 记录性能数据
        perf_logger = logging.getLogger('performance')
        perf_logger.info(
            f"REQUEST: {request.method} {request.path} "
            f"DURATION: {duration:.3f}s "
            f"QUERIES: {query_count} "
            f"STATUS: {response.status_code}"
        )
        
        # 性能警报
        if duration > 2.0:  # 超过2秒的请求
            perf_logger.warning(
                f"SLOW REQUEST: {request.path} took {duration:.3f}s"
            )
        
        if query_count > 50:  # 超过50个查询
            perf_logger.warning(
                f"HIGH QUERY COUNT: {request.path} had {query_count} queries"
            )
        
        return response

"""
本章核心要点：

1. 生产环境部署需要考虑安全性、可用性和性能
2. Docker容器化简化了部署和扩展
3. Nginx作为反向代理提供负载均衡和SSL终止
4. Gunicorn的配置直接影响应用性能
5. 数据库和缓存服务需要专门的生产配置
6. 监控和日志是运维的重要组成部分
7. CI/CD自动化提高了部署效率和可靠性
8. 定期备份和健康检查确保系统稳定性
"""