#Django文件上传与存储 - 完整的文件处理解决方案

"""
文件上传工作原理：

1. 客户端使用multipart/form-data编码上传文件
2. Django接收文件并创建UploadedFile对象
3. 文件被存储到临时位置或直接处理
4. 可以将文件保存到指定位置或存储服务
5. 在模型中保存文件路径信息

文件类型：
- UploadedFile: 基础上传文件类
- TemporaryUploadedFile: 临时上传文件
- InMemoryUploadedFile: 内存中的上传文件
"""

# 文件上传表单基础
from django import forms
from django.core.files.uploadedfile import UploadedFile

class FileUploadForm(forms.Form):
    """基础文件上传表单"""
    
    title = forms.CharField(max_length=100)
    file = forms.FileField(
        label='选择文件',
        help_text='支持的文件类型: PDF, DOC, XLS, JPG, PNG'
    )
    
    def clean_file(self):
        """验证上传的文件"""
        uploaded_file = self.cleaned_data['file']
        
        # 检查文件大小（例如限制为5MB）
        if uploaded_file.size > 5 * 1024 * 1024:  # 5MB
            raise forms.ValidationError('文件大小不能超过5MB')
        
        # 检查文件类型
        allowed_extensions = ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.jpg', '.jpeg', '.png']
        import os
        ext = os.path.splitext(uploaded_file.name)[1].lower()
        
        if ext not in allowed_extensions:
            raise forms.ValidationError(
                f'不支持的文件类型: {ext}。支持的类型: {", ".join(allowed_extensions)}'
            )
        
        return uploaded_file

# 模型表单中的文件字段
from django.db import models

class Document(models.Model):
    """文档模型"""
    title = models.CharField(max_length=200)
    file = models.FileField(upload_to='documents/%Y/%m/')  # 按年月组织
    uploaded_at = models.DateTimeField(auto_now_add=True)
    file_size = models.PositiveIntegerField(null=True, blank=True)  # 文件大小
    content_type = models.CharField(max_length=100, null=True, blank=True)  # MIME类型
    
    def save(self, *args, **kwargs):
        """保存时设置文件相关信息"""
        if self.file:
            self.file_size = self.file.size
            if hasattr(self.file, 'content_type'):
                self.content_type = self.file.content_type
        super().save(*args, **kwargs)

class DocumentForm(forms.ModelForm):
    """文档模型表单"""
    class Meta:
        model = Document
        fields = ['title', 'file']

# 文件上传视图
from django.shortcuts import render, redirect
from django.contrib import messages
from django.views.decorators.csrf import csrf_protect
from django.http import JsonResponse
import json

@csrf_protect
def upload_file_view(request):
    """文件上传视图"""
    if request.method == 'POST':
        form = FileUploadForm(request.POST, request.FILES)
        if form.is_valid():
            # 处理上传的文件
            uploaded_file = request.FILES['file']
            
            # 方法1: 直接保存到模型
            document = Document.objects.create(
                title=form.cleaned_data['title'],
                file=uploaded_file
            )
            
            messages.success(request, '文件上传成功！')
            return redirect('document_list')
        else:
            messages.error(request, '文件上传失败，请检查文件格式和大小。')
    else:
        form = FileUploadForm()
    
    return render(request, 'upload.html', {'form': form})

# AJAX文件上传
def ajax_upload_view(request):
    """AJAX文件上传视图"""
    if request.method == 'POST' and request.FILES:
        try:
            uploaded_file = request.FILES['file']
            
            # 验证文件
            if uploaded_file.size > 10 * 1024 * 1024:  # 10MB限制
                return JsonResponse({
                    'success': False,
                    'error': '文件太大，最大支持10MB'
                })
            
            # 保存文件
            document = Document.objects.create(
                title=uploaded_file.name,
                file=uploaded_file
            )
            
            return JsonResponse({
                'success': True,
                'file_id': document.id,
                'file_url': document.file.url,
                'file_name': uploaded_file.name
            })
        except Exception as e:
            return JsonResponse({
                'success': False,
                'error': str(e)
            })
    
    return JsonResponse({'success': False, 'error': '无效请求'})

# 进度条上传（使用FormData和JavaScript）
def progress_upload_view(request):
    """支持进度显示的文件上传"""
    if request.method == 'POST':
        # 这里可以集成进度跟踪逻辑
        # 通常需要结合JavaScript和WebSocket来实现实时进度
        uploaded_file = request.FILES['file']
        
        # 保存文件
        document = Document.objects.create(
            title=uploaded_file.name,
            file=uploaded_file
        )
        
        return JsonResponse({
            'success': True,
            'file_url': document.file.url
        })
    
    return render(request, 'progress_upload.html')

# Django文件处理核心组件
from django.core.files.base import File, ContentFile
from django.core.files.storage import default_storage, FileSystemStorage
from django.core.files.uploadedfile import (
    UploadedFile, TemporaryUploadedFile, InMemoryUploadedFile
)
import tempfile
import os

# 文件存储后端
"""
Django文件存储后端：
1. FileSystemStorage: 本地文件系统存储（默认）
2. S3Boto3Storage: AWS S3存储
3. GoogleCloudStorage: Google Cloud存储
4. AzureStorage: Microsoft Azure存储
"""

# 文件处理工具类
class FileProcessor:
    """文件处理工具类"""
    
    @staticmethod
    def get_file_info(uploaded_file):
        """获取文件信息"""
        return {
            'name': uploaded_file.name,
            'size': uploaded_file.size,
            'content_type': getattr(uploaded_file, 'content_type', ''),
            'charset': getattr(uploaded_file, 'charset', ''),
            'temporary_file_path': getattr(uploaded_file, 'temporary_file_path', None),
            'multiple_chunks': uploaded_file.multiple_chunks(),
            'chunks': list(uploaded_file.chunks()) if uploaded_file.multiple_chunks() else [uploaded_file.read()]
        }
    
    @staticmethod
    def save_file_to_path(uploaded_file, destination_path):
        """将上传文件保存到指定路径"""
        with open(destination_path, 'wb+') as destination:
            for chunk in uploaded_file.chunks():
                destination.write(chunk)
    
    @staticmethod
    def create_content_file(content, name):
        """从内容创建文件对象"""
        return ContentFile(content, name=name)
    
    @staticmethod
    def temporary_file_handling():
        """临时文件处理示例"""
        # 创建临时文件
        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
            temp_file.write(b'Some temporary content')
            temp_file_path = temp_file.name
        
        # 将临时文件移动到Django存储
        with open(temp_file_path, 'rb') as f:
            content = f.read()
            django_file = ContentFile(content)
            # 保存到模型
            # my_model.file.save('temp_file.txt', django_file)
        
        # 清理临时文件
        os.unlink(temp_file_path)

# 文件上传处理器
class FileUploadHandler:
    """文件上传处理器"""
    
    def __init__(self, request):
        self.request = request
    
    def handle_single_file(self, field_name='file'):
        """处理单个文件上传"""
        if field_name in self.request.FILES:
            uploaded_file = self.request.FILES[field_name]
            return self.process_file(uploaded_file)
        return None
    
    def handle_multiple_files(self, field_name='files'):
        """处理多个文件上传"""
        files = []
        if field_name in self.request.FILES:
            for uploaded_file in self.request.FILES.getlist(field_name):
                processed_file = self.process_file(uploaded_file)
                files.append(processed_file)
        return files
    
    def process_file(self, uploaded_file):
        """处理单个上传文件"""
        # 验证文件
        self.validate_file(uploaded_file)
        
        # 生成安全的文件名
        safe_filename = self.generate_safe_filename(uploaded_file.name)
        
        # 保存文件
        saved_path = default_storage.save(safe_filename, uploaded_file)
        
        return {
            'original_name': uploaded_file.name,
            'saved_name': safe_filename,
            'saved_path': saved_path,
            'size': uploaded_file.size,
            'content_type': getattr(uploaded_file, 'content_type', '')
        }
    
    def validate_file(self, uploaded_file):
        """验证上传文件"""
        # 检查文件大小
        max_size = 10 * 1024 * 1024  # 10MB
        if uploaded_file.size > max_size:
            raise ValueError(f'文件太大，最大支持 {max_size / (1024*1024)}MB')
        
        # 检查文件名安全性
        if '..' in uploaded_file.name or '/' in uploaded_file.name:
            raise ValueError('文件名包含非法字符')
    
    def generate_safe_filename(self, original_filename):
        """生成安全的文件名"""
        import uuid
        import os
        name, ext = os.path.splitext(original_filename)
        safe_name = f"{uuid.uuid4().hex}{ext.lower()}"
        return safe_name

# 文件存储接口和实现
from django.core.files.storage import Storage
from django.conf import settings
import os
from urllib.parse import urljoin

class CustomStorage(Storage):
    """自定义存储后端示例"""
    
    def __init__(self, location=None, base_url=None):
        self.location = location or getattr(settings, 'CUSTOM_FILE_STORAGE_LOCATION', '/custom_storage/')
        self.base_url = base_url or getattr(settings, 'CUSTOM_FILE_STORAGE_URL', '/custom-files/')
    
    def _save(self, name, content):
        """保存文件"""
        full_path = self.path(name)
        
        # 创建目录
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        
        # 写入文件
        with open(full_path, 'wb') as f:
            for chunk in content.chunks():
                f.write(chunk)
        
        return name
    
    def _open(self, name, mode='rb'):
        """打开文件"""
        return open(self.path(name), mode)
    
    def delete(self, name):
        """删除文件"""
        full_path = self.path(name)
        if os.path.exists(full_path):
            os.remove(full_path)
    
    def exists(self, name):
        """检查文件是否存在"""
        return os.path.exists(self.path(name))
    
    def url(self, name):
        """获取文件URL"""
        return urljoin(self.base_url, name)
    
    def size(self, name):
        """获取文件大小"""
        return os.path.getsize(self.path(name))
    
    def path(self, name):
        """获取文件系统路径"""
        return os.path.join(self.location, name)

# 存储工厂
class StorageFactory:
    """存储工厂类"""
    
    @staticmethod
    def get_storage(storage_type='filesystem', **kwargs):
        """根据类型获取存储实例"""
        if storage_type == 'filesystem':
            from django.core.files.storage import FileSystemStorage
            return FileSystemStorage(**kwargs)
        elif storage_type == 'custom':
            return CustomStorage(**kwargs)
        elif storage_type == 's3':
            try:
                from storages.backends.s3boto3 import S3Boto3Storage
                return S3Boto3Storage(**kwargs)
            except ImportError:
                raise ValueError("S3存储需要安装django-storages[boto3]")
        else:
            raise ValueError(f"不支持的存储类型: {storage_type}")

# 存储配置管理
class StorageConfig:
    """存储配置管理"""
    
    @staticmethod
    def get_media_storage():
        """获取媒体文件存储配置"""
        storage_type = getattr(settings, 'MEDIA_STORAGE_TYPE', 'filesystem')
        
        if storage_type == 'filesystem':
            return {
                'class': 'django.core.files.storage.FileSystemStorage',
                'location': settings.MEDIA_ROOT,
                'base_url': settings.MEDIA_URL
            }
        elif storage_type == 's3':
            return {
                'class': 'storages.backends.s3boto3.S3Boto3Storage',
                'bucket_name': settings.AWS_STORAGE_BUCKET_NAME,
                'region_name': settings.AWS_S3_REGION_NAME,
                'file_overwrite': False
            }
        else:
            return {
                'class': 'django.core.files.storage.FileSystemStorage',
                'location': settings.MEDIA_ROOT,
                'base_url': settings.MEDIA_URL
            }

# Django文件字段类型详解
from django.db import models
from django.core.files.storage import default_storage
import os

class FileFieldsExample(models.Model):
    """文件字段示例"""
    
    # 基础文件字段
    document = models.FileField(
        upload_to='documents/',
        max_length=500,
        help_text='上传文档文件'
    )
    
    # 图像字段（需要Pillow）
    image = models.ImageField(
        upload_to='images/',
        blank=True,
        null=True,
        help_text='上传图片文件'
    )
    
    # 自定义上传路径
    custom_file = models.FileField(
        upload_to=lambda instance, filename: f'uploads/{instance.user.id}/{filename}',
        blank=True,
        null=True
    )
    
    # 指定存储后端
    stored_file = models.FileField(
        upload_to='stored/',
        storage=default_storage,  # 可以指定自定义存储
        blank=True,
        null=True
    )
    
    # 限制文件类型
    pdf_only = models.FileField(
        upload_to='pdfs/',
        validators=[validate_pdf_file],  # 自定义验证器
        blank=True,
        null=True
    )

def validate_pdf_file(value):
    """PDF文件验证器"""
    import magic  # 需要python-magic包
    file_mime = magic.from_buffer(value.read(1024), mime=True)
    value.seek(0)  # 重置文件指针
    
    if file_mime != 'application/pdf':
        raise ValidationError('只允许上传PDF文件')

# 高级文件字段配置
class AdvancedFileFields(models.Model):
    """高级文件字段配置"""
    
    # 按日期组织文件
    dated_document = models.FileField(
        upload_to='documents/%Y/%m/%d/',  # 按年/月/日组织
        help_text='按日期组织的文档'
    )
    
    # 按用户ID组织
    user_document = models.FileField(
        upload_to=lambda instance, filename: f'user_docs/{instance.owner.id}/{filename}',
        help_text='按用户组织的文档'
    )
    
    # 按文件哈希组织（防重复）
    hashed_file = models.FileField(
        upload_to=lambda instance, filename: f'hashed/{hash_file_content(instance.file)}',
        help_text='按内容哈希组织的文件'
    )

def hash_file_content(uploaded_file):
    """根据文件内容生成哈希值"""
    import hashlib
    md5_hash = hashlib.md5()
    for chunk in uploaded_file.chunks():
        md5_hash.update(chunk)
    return f"{md5_hash.hexdigest()[:8]}/{uploaded_file.name}"

# 文件字段验证
class ValidatedFileForm(forms.ModelForm):
    """带验证的文件表单"""
    
    class Meta:
        model = FileFieldsExample
        fields = ['document', 'image']
    
    def clean_document(self):
        """验证文档文件"""
        document = self.cleaned_data.get('document')
        if document:
            # 检查文件大小
            if document.size > 10 * 1024 * 1024:  # 10MB
                raise forms.ValidationError('文档文件不能超过10MB')
            
            # 检查文件类型（基于扩展名）
            import os
            ext = os.path.splitext(document.name)[1].lower()
            allowed_exts = ['.pdf', '.doc', '.docx', '.txt', '.xls', '.xlsx']
            if ext not in allowed_exts:
                raise forms.ValidationError(
                    f'不允许的文档类型: {ext}。支持: {", ".join(allowed_exts)}'
                )
        
        return document
    
    def clean_image(self):
        """验证图像文件"""
        image = self.cleaned_data.get('image')
        if image:
            # 检查文件大小
            if image.size > 5 * 1024 * 1024:  # 5MB
                raise forms.ValidationError('图像文件不能超过5MB')
            
            # 检查图像尺寸
            from PIL import Image
            img = Image.open(image)
            width, height = img.size
            if width > 4000 or height > 4000:
                raise forms.ValidationError('图像尺寸过大，最大支持4000x4000像素')
        
        return image

# 高级文件表单处理
from django import forms
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
import os
import mimetypes

class AdvancedFileForm(forms.Form):
    """高级文件表单"""
    
    title = forms.CharField(max_length=200)
    description = forms.CharField(widget=forms.Textarea, required=False)
    
    # 多文件上传
    files = forms.FileField(
        widget=forms.ClearableFileInput(attrs={
            'multiple': True,
            'accept': '.pdf,.doc,.docx,.jpg,.jpeg,.png'
        }),
        help_text='可以选择多个文件',
        required=False
    )
    
    # 图像文件
    image = forms.ImageField(
        help_text='上传一张图片',
        required=False
    )
    
    def clean_files(self):
        """验证多个文件"""
        files = self.cleaned_data.get('files')
        if files:
            cleaned_files = []
            
            for uploaded_file in files if hasattr(files, '__iter__') else [files]:
                # 验证单个文件
                self.validate_single_file(uploaded_file)
                cleaned_files.append(uploaded_file)
            
            return cleaned_files
        
        return []
    
    def validate_single_file(self, uploaded_file):
        """验证单个文件"""
        # 检查文件大小
        max_size = 10 * 1024 * 1024  # 10MB
        if uploaded_file.size > max_size:
            raise ValidationError(_(f'文件 {uploaded_file.name} 太大，最大支持 {max_size // (1024*1024)}MB'))
        
        # 检查文件类型
        allowed_types = {
            'application/pdf': '.pdf',
            'application/msword': '.doc',
            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
            'image/jpeg': '.jpg',
            'image/png': '.png',
            'text/plain': '.txt'
        }
        
        # 获取MIME类型
        mime_type = mimetypes.guess_type(uploaded_file.name)[0]
        if not mime_type:
            # 如果无法猜测，读取文件头
            import magic
            mime_type = magic.from_buffer(uploaded_file.read(1024), mime=True)
            uploaded_file.seek(0)  # 重置文件指针
        
        if mime_type not in allowed_types:
            raise ValidationError(_(f'不支持的文件类型: {mime_type}'))
        
        # 检查扩展名是否匹配
        _, ext = os.path.splitext(uploaded_file.name.lower())
        expected_ext = allowed_types.get(mime_type)
        if expected_ext and ext != expected_ext:
            raise ValidationError(_(f'文件扩展名与内容类型不匹配'))

# 文件上传进度处理
class ProgressFileForm(forms.Form):
    """支持进度显示的文件表单"""
    
    title = forms.CharField(max_length=200)
    file = forms.FileField()
    
    def save_with_progress(self, progress_callback=None):
        """带进度回调的文件保存"""
        uploaded_file = self.cleaned_data['file']
        
        # 创建临时文件来处理大文件
        import tempfile
        import shutil
        
        temp_file_path = None
        try:
            # 创建临时文件
            with tempfile.NamedTemporaryFile(delete=False) as temp_file:
                temp_file_path = temp_file.name
                
                # 分块写入并报告进度
                total_size = uploaded_file.size
                bytes_written = 0
                
                for chunk in uploaded_file.chunks():
                    temp_file.write(chunk)
                    bytes_written += len(chunk)
                    
                    if progress_callback:
                        progress = (bytes_written / total_size) * 100
                        progress_callback(progress)
            
            # 将临时文件移动到最终位置
            final_path = default_storage.save(uploaded_file.name, open(temp_file_path, 'rb'))
            
            # 创建数据库记录
            from .models import Document
            document = Document.objects.create(
                title=self.cleaned_data['title'],
                file=final_path
            )
            
            return document
            
        finally:
            # 清理临时文件
            if temp_file_path and os.path.exists(temp_file_path):
                os.unlink(temp_file_path)

# 动态文件表单
class DynamicFileForm(forms.Form):
    """动态文件表单"""
    
    def __init__(self, *args, **kwargs):
        # 从kwargs中获取文件字段数量
        num_files = kwargs.pop('num_files', 1)
        super().__init__(*args, **kwargs)
        
        # 动态添加文件字段
        for i in range(num_files):
            self.fields[f'file_{i}'] = forms.FileField(
                label=f'文件 {i+1}',
                required=False
            )
    
    def clean(self):
        """验证所有动态文件字段"""
        cleaned_data = super().clean()
        
        for field_name, field_value in cleaned_data.items():
            if field_name.startswith('file_') and field_value:
                # 验证单个文件
                self.validate_dynamic_file(field_value)
        
        return cleaned_data
    
    def validate_dynamic_file(self, uploaded_file):
        """验证动态添加的文件"""
        # 实现文件验证逻辑
        if uploaded_file.size > 5 * 1024 * 1024:  # 5MB限制
            raise ValidationError('文件太大')

# settings.py - 基础文件存储配置
import os
from pathlib import Path

BASE_DIR = Path(__file__).resolve().parent.parent

# 媒体文件配置
MEDIA_URL = '/media/'  # 媒体文件URL前缀
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')  # 媒体文件存储根目录

# 静态文件配置
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATICFILES_DIRS = [
    os.path.join(BASE_DIR, 'static'),
]

# 文件上传配置
FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440  # 2.5MB，超过此大小的文件将保存到临时文件
DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440  # 2.5MB，POST数据的最大内存大小
FILE_UPLOAD_TEMP_DIR = os.path.join(BASE_DIR, 'tmp')  # 临时文件目录

# 创建必要的目录
os.makedirs(MEDIA_ROOT, exist_ok=True)
os.makedirs(FILE_UPLOAD_TEMP_DIR, exist_ok=True)

# 存储后端配置
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'  # 用于云存储

# AWS S3存储配置示例（需要django-storages）
"""
AWS_ACCESS_KEY_ID = 'your-access-key'
AWS_SECRET_ACCESS_KEY = 'your-secret-key'
AWS_STORAGE_BUCKET_NAME = 'your-bucket-name'
AWS_S3_REGION_NAME = 'us-east-1'
AWS_S3_CUSTOM_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com'

# 静态文件S3配置
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'

# 媒体文件S3配置
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
"""

# 高级存储配置类
from django.conf import settings
from django.core.files.storage import get_storage_class
from django.utils.module_loading import import_string
import os

class StorageManager:
    """存储管理器"""
    
    def __init__(self):
        self._storages = {}
    
    def get_storage(self, storage_alias='default'):
        """获取存储实例"""
        if storage_alias in self._storages:
            return self._storages[storage_alias]
        
        # 从设置中获取存储配置
        storage_configs = getattr(settings, 'CUSTOM_FILE_STORAGES', {})
        
        if storage_alias in storage_configs:
            config = storage_configs[storage_alias]
            storage_class = get_storage_class(config['class'])
            storage_instance = storage_class(**config.get('options', {}))
        else:
            # 默认存储
            storage_class = get_storage_class(settings.DEFAULT_FILE_STORAGE)
            storage_instance = storage_class()
        
        self._storages[storage_alias] = storage_instance
        return storage_instance
    
    def save_file(self, file_obj, filename, storage_alias='default'):
        """使用指定存储保存文件"""
        storage = self.get_storage(storage_alias)
        return storage.save(filename, file_obj)
    
    def get_file_url(self, filename, storage_alias='default'):
        """获取文件URL"""
        storage = self.get_storage(storage_alias)
        return storage.url(filename)

# 存储配置示例
CUSTOM_FILE_STORAGES = {
    'default': {
        'class': 'django.core.files.storage.FileSystemStorage',
        'options': {
            'location': os.path.join(settings.BASE_DIR, 'media'),
            'base_url': '/media/',
        }
    },
    'temp': {
        'class': 'django.core.files.storage.FileSystemStorage',
        'options': {
            'location': os.path.join(settings.BASE_DIR, 'temp_uploads'),
            'base_url': '/temp-media/',
        }
    },
    'secure': {
        'class': 'django.core.files.storage.FileSystemStorage',
        'options': {
            'location': os.path.join(settings.BASE_DIR, 'secure_files'),
            'base_url': '/secure-files/',  # 实际URL可能需要保护
        }
    }
}

# 使用存储管理器
storage_manager = StorageManager()

# 存储策略配置
class StorageStrategy:
    """存储策略"""
    
    @staticmethod
    def get_strategy(file_type, file_size):
        """根据文件类型和大小选择存储策略"""
        if file_type.startswith('image/'):
            if file_size < 1024 * 1024:  # 1MB以下
                return 'default'
            else:
                return 'high_capacity'
        elif file_type.startswith('video/'):
            return 'high_capacity'
        elif file_type == 'application/pdf':
            if file_size > 10 * 1024 * 1024:  # 10MB以上
                return 'cloud'
            else:
                return 'default'
        else:
            return 'default'

# 文件组织策略
class FileOrganizer:
    """文件组织策略"""
    
    @staticmethod
    def organize_by_date(filename, date=None):
        """按日期组织文件"""
        from datetime import datetime
        if date is None:
            date = datetime.now()
        
        year = date.strftime('%Y')
        month = date.strftime('%m')
        day = date.strftime('%d')
        
        return f"{year}/{month}/{day}/{filename}"
    
    @staticmethod
    def organize_by_user(filename, user_id):
        """按用户组织文件"""
        return f"user_{user_id}/{filename}"
    
    @staticmethod
    def organize_by_content_type(filename, content_type):
        """按内容类型组织文件"""
        type_mapping = {
            'image/jpeg': 'images',
            'image/png': 'images', 
            'image/gif': 'images',
            'application/pdf': 'documents',
            'application/msword': 'documents',
            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'documents',
            'video/mp4': 'videos',
            'audio/mpeg': 'audio'
        }
        
        folder = type_mapping.get(content_type, 'others')
        return f"{folder}/{filename}"
    
    @staticmethod
    def organize_by_size(filename, file_size):
        """按文件大小组织文件"""
        if file_size < 1024 * 1024:  # 1MB
            size_folder = 'small'
        elif file_size < 10 * 1024 * 1024:  # 10MB
            size_folder = 'medium'
        else:
            size_folder = 'large'
        
        return f"{size_folder}/{filename}"

# 综合文件路径生成器
class SmartUploadPath:
    """智能上传路径生成器"""
    
    def __init__(self, base_path='', organization_strategies=None):
        self.base_path = base_path
        self.strategies = organization_strategies or [
            FileOrganizer.organize_by_date,
            FileOrganizer.organize_by_content_type,
        ]
    
    def __call__(self, instance, filename):
        """生成上传路径"""
        # 获取文件信息
        content_type = getattr(instance, 'content_type', '')
        file_size = getattr(instance, 'file_size', 0)
        
        # 应用组织策略
        organized_path = filename
        for strategy in self.strategies:
            if callable(strategy):
                if strategy.__name__ == 'organize_by_content_type':
                    organized_path = strategy(organized_path, content_type)
                elif strategy.__name__ == 'organize_by_size':
                    organized_path = strategy(organized_path, file_size)
                elif strategy.__name__ == 'organize_by_date':
                    organized_path = strategy(organized_path)
                elif strategy.__name__ == 'organize_by_user':
                    user_id = getattr(instance, 'user_id', 1)
                    organized_path = strategy(organized_path, user_id)
        
        return f"{self.base_path}{organized_path}"

# 使用示例
smart_path = SmartUploadPath('uploads/', [
    FileOrganizer.organize_by_date,
    FileOrganizer.organize_by_content_type,
])

class SmartFileModel(models.Model):
    """使用智能路径的文件模型"""
    title = models.CharField(max_length=200)
    file = models.FileField(upload_to=smart_path)
    content_type = models.CharField(max_length=100, blank=True)
    file_size = models.PositiveIntegerField(null=True, blank=True)
    uploaded_at = models.DateTimeField(auto_now_add=True)

# 存储性能优化配置
import threading
from concurrent.futures import ThreadPoolExecutor
import asyncio
from django.core.files.storage import default_storage

class OptimizedStorage:
    """优化的存储类"""
    
    def __init__(self, storage_backend=None):
        self.storage = storage_backend or default_storage
        self._thread_local = threading.local()
        self.executor = ThreadPoolExecutor(max_workers=4)
    
    def save_async(self, name, content):
        """异步保存文件"""
        future = self.executor.submit(self.storage.save, name, content)
        return future
    
    def save_batch(self, file_pairs):
        """批量保存文件"""
        futures = []
        for name, content in file_pairs:
            future = self.executor.submit(self.storage.save, name, content)
            futures.append(future)
        
        results = []
        for future in futures:
            results.append(future.result())
        
        return results
    
    def get_cached_url(self, name, cache_timeout=3600):
        """获取带缓存的文件URL"""
        from django.core.cache import cache
        
        cache_key = f"file_url_{name}"
        cached_url = cache.get(cache_key)
        
        if cached_url is None:
            cached_url = self.storage.url(name)
            cache.set(cache_key, cached_url, timeout=cache_timeout)
        
        return cached_url

# 缓存存储装饰器
def cached_storage_method(method):
    """存储方法缓存装饰器"""
    def wrapper(self, name, *args, **kwargs):
        from django.core.cache import cache
        
        cache_key = f"storage_{method.__name__}_{name}"
        cached_result = cache.get(cache_key)
        
        if cached_result is not None:
            return cached_result
        
        result = method(self, name, *args, **kwargs)
        cache.set(cache_key, result, timeout=300)  # 缓存5分钟
        
        return result
    return wrapper

class CachedStorage:
    """带缓存的存储类"""
    
    def __init__(self, storage_backend=None):
        self.storage = storage_backend or default_storage
    
    @cached_storage_method
    def url(self, name):
        """获取文件URL（带缓存）"""
        return self.storage.url(name)
    
    @cached_storage_method 
    def size(self, name):
        """获取文件大小（带缓存）"""
        return self.storage.size(name)
    
    def exists(self, name):
        """检查文件是否存在（带缓存）"""
        from django.core.cache import cache
        
        cache_key = f"storage_exists_{name}"
        cached_result = cache.get(cache_key)
        
        if cached_result is not None:
            return cached_result
        
        result = self.storage.exists(name)
        cache.set(cache_key, result, timeout=300)
        
        return result

# 存储监控
class StorageMonitor:
    """存储监控类"""
    
    def __init__(self):
        self.stats = {
            'files_saved': 0,
            'files_deleted': 0,
            'total_size': 0,
            'average_size': 0
        }
        self.lock = threading.Lock()
    
    def record_save(self, file_size):
        """记录文件保存"""
        with self.lock:
            self.stats['files_saved'] += 1
            self.stats['total_size'] += file_size
            self.stats['average_size'] = self.stats['total_size'] / self.stats['files_saved']
    
    def record_delete(self, file_size):
        """记录文件删除"""
        with self.lock:
            self.stats['files_deleted'] += 1
            self.stats['total_size'] -= file_size
            if self.stats['files_saved'] > self.stats['files_deleted']:
                self.stats['average_size'] = (self.stats['total_size'] + file_size) / (self.stats['files_saved'] - self.stats['files_deleted'])
    
    def get_stats(self):
        """获取统计信息"""
        return self.stats.copy()

# 全局存储监控器
storage_monitor = StorageMonitor()

# 文件安全验证
import os
import magic  # 需要python-magic包
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _

class SecureFileValidator:
    """安全文件验证器"""
    
    def __init__(self, allowed_extensions=None, allowed_mimes=None, max_size=None):
        self.allowed_extensions = allowed_extensions or [
            '.jpg', '.jpeg', '.png', '.gif',  # 图像
            '.pdf', '.doc', '.docx', '.txt',  # 文档
            '.mp3', '.wav', '.ogg',          # 音频
            '.mp4', '.avi', '.mov'           # 视频
        ]
        self.allowed_mimes = allowed_mimes or [
            'image/jpeg', 'image/png', 'image/gif',
            'application/pdf',
            'application/msword',
            'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
            'text/plain',
            'audio/mpeg', 'audio/wav', 'audio/ogg',
            'video/mp4', 'video/x-msvideo', 'video/quicktime'
        ]
        self.max_size = max_size or 10 * 1024 * 1024  # 10MB
    
    def validate(self, uploaded_file):
        """执行完整验证"""
        self.validate_size(uploaded_file)
        self.validate_extension(uploaded_file)
        self.validate_mime_type(uploaded_file)
        self.validate_content(uploaded_file)
        self.validate_filename(uploaded_file)
    
    def validate_size(self, uploaded_file):
        """验证文件大小"""
        if uploaded_file.size > self.max_size:
            raise ValidationError(
                _('文件大小超过限制 (%(size)s MB)'),
                params={'size': self.max_size / (1024*1024)},
            )
    
    def validate_extension(self, uploaded_file):
        """验证文件扩展名"""
        _, ext = os.path.splitext(uploaded_file.name.lower())
        if ext not in self.allowed_extensions:
            raise ValidationError(
                _('不允许的文件扩展名: %(ext)s'),
                params={'ext': ext},
            )
    
    def validate_mime_type(self, uploaded_file):
        """验证MIME类型（双重检查）"""
        # 基于文件内容的真实MIME类型
        real_mime = magic.from_buffer(uploaded_file.read(1024), mime=True)
        uploaded_file.seek(0)  # 重置文件指针
        
        if real_mime not in self.allowed_mimes:
            raise ValidationError(
                _('不允许的文件类型: %(mime)s'),
                params={'mime': real_mime},
            )
        
        # 基于扩展名的MIME类型
        import mimetypes
        ext_mime = mimetypes.guess_type(uploaded_file.name)[0]
        if ext_mime and ext_mime not in self.allowed_mimes:
            raise ValidationError(
                _('扩展名对应的文件类型不允许: %(mime)s'),
                params={'mime': ext_mime},
            )
    
    def validate_content(self, uploaded_file):
        """验证文件内容（防恶意代码）"""
        # 检查文件头
        header = uploaded_file.read(1024)
        uploaded_file.seek(0)  # 重置
        
        # 检查是否包含HTML/JS标签（常见攻击向量）
        suspicious_patterns = [
            b'<script', b'javascript:', b'vbscript:', b'<iframe',
            b'<object', b'<embed', b'<?php', b'<?', b'<%',  # 服务端包含
        ]
        
        header_lower = header.lower()
        for pattern in suspicious_patterns:
            if pattern in header_lower:
                raise ValidationError(_('文件可能包含恶意代码'))
    
    def validate_filename(self, uploaded_file):
        """验证文件名安全性"""
        filename = uploaded_file.name
        
        # 检查路径遍历攻击
        if '..' in filename or '~' in filename:
            raise ValidationError(_('文件名包含非法字符'))
        
        # 检查危险扩展名
        dangerous_extensions = [
            '.exe', '.bat', '.com', '.scr', '.vbs', '.js', '.jar',
            '.pif', '.html', '.htm', '.php', '.asp', '.aspx'
        ]
        
        _, ext = os.path.splitext(filename.lower())
        if ext in dangerous_extensions:
            raise ValidationError(_('文件扩展名可能存在安全风险'))

# 使用安全验证器的表单
class SecureFileForm(forms.Form):
    """安全文件表单"""
    
    title = forms.CharField(max_length=200)
    file = forms.FileField()
    
    def __init__(self, *args, **kwargs):
        self.validator = SecureFileValidator(
            max_size=5 * 1024 * 1024,  # 5MB
            allowed_extensions=['.jpg', '.jpeg', '.png', '.pdf', '.doc', '.docx']
        )
        super().__init__(*args, **kwargs)
    
    def clean_file(self):
        """安全验证上传文件"""
        uploaded_file = self.cleaned_data.get('file')
        if uploaded_file:
            self.validator.validate(uploaded_file)
        return uploaded_file

# 模型级别的安全验证
class SecureDocument(models.Model):
    """安全文档模型"""
    title = forms.CharField(max_length=200)
    file = models.FileField(upload_to='secure_docs/')
    uploaded_at = models.DateTimeField(auto_now_add=True)
    verified = models.BooleanField(default=False)  # 是否通过安全检查
    
    def save(self, *args, **kwargs):
        """保存前进行安全检查"""
        if self.file:
            validator = SecureFileValidator()
            # 验证上传的文件
            if hasattr(self.file, 'file'):  # 如果是已上传的文件
                validator.validate(self.file.file)
        
        super().save(*args, **kwargs)
        
        # 异步安全扫描
        self.async_security_scan()
    
    def async_security_scan(self):
        """异步安全扫描"""
        # 这里可以集成病毒扫描、内容审核等
        from django.core.mail import mail_admins
        import threading
        
        def scan_task():
            try:
                # 模拟安全扫描
                import time
                time.sleep(1)  # 模拟扫描时间
                
                # 如果发现安全问题，通知管理员
                # mail_admins("安全警告", f"发现可疑文件: {self.file.name}")
                
                # 更新验证状态
                self.verified = True
                self.save(update_fields=['verified'])
                
            except Exception as e:
                # 记录错误
                import logging
                logger = logging.getLogger(__name__)
                logger.error(f"安全扫描失败: {e}")
        
        # 在后台线程中执行扫描
        thread = threading.Thread(target=scan_task)
        thread.daemon = True
        thread.start()

# 文件访问控制
from django.http import HttpResponse, Http404, HttpResponseForbidden
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.views.generic import View
from django.contrib.auth.mixins import LoginRequiredMixin
import os
from django.conf import settings

class SecureFileDownloadView(LoginRequiredMixin, View):
    """安全文件下载视图"""
    
    def get(self, request, file_path):
        """安全下载文件"""
        # 验证用户权限
        if not self.has_permission(request.user, file_path):
            return HttpResponseForbidden("无权访问此文件")
        
        # 构建安全的文件路径
        safe_path = self.get_safe_file_path(file_path)
        
        if not safe_path or not os.path.exists(safe_path):
            raise Http404("文件不存在")
        
        # 检查文件类型，防止执行脚本
        if self.is_dangerous_file(safe_path):
            return HttpResponseForbidden("禁止下载此类型文件")
        
        # 读取并返回文件
        with open(safe_path, 'rb') as f:
            response = HttpResponse(f.read(), content_type='application/octet-stream')
            response['Content-Disposition'] = f'attachment; filename="{os.path.basename(safe_path)}"'
            return response
    
    def has_permission(self, user, file_path):
        """检查用户是否有权限访问文件"""
        # 实现权限检查逻辑
        # 例如：检查文件是否属于用户，或用户是否有相应角色
        from .models import Document
        
        try:
            document = Document.objects.get(file=file_path)
            # 检查文档是否属于当前用户或用户有相应权限
            return (document.owner == user or 
                   user.is_staff or 
                   user.has_perm('myapp.download_document'))
        except Document.DoesNotExist:
            return False
    
    def get_safe_file_path(self, file_path):
        """获取安全的文件系统路径"""
        # 防止路径遍历攻击
        if '..' in file_path or file_path.startswith('/'):
            return None
        
        # 构建完整路径
        full_path = os.path.join(settings.MEDIA_ROOT, file_path)
        
        # 规范化路径
        normalized_path = os.path.normpath(full_path)
        
        # 确保路径在媒体目录内
        if not normalized_path.startswith(os.path.normpath(settings.MEDIA_ROOT)):
            return None
        
        return normalized_path
    
    def is_dangerous_file(self, file_path):
        """检查是否为危险文件类型"""
        dangerous_extensions = [
            '.exe', '.bat', '.com', '.scr', '.vbs', '.js', '.jar',
            '.pif', '.hta', '.msi', '.cmd', '.ps1', '.sh'
        ]
        
        _, ext = os.path.splitext(file_path.lower())
        return ext in dangerous_extensions

# 带权限检查的模型
class AccessControlledFile(models.Model):
    """访问控制文件模型"""
    title = models.CharField(max_length=200)
    file = models.FileField(upload_to='controlled/')
    owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
    access_level = models.CharField(
        max_length=20,
        choices=[
            ('private', '私有'),
            ('shared', '共享'),
            ('public', '公开'),
        ],
        default='private'
    )
    allowed_users = models.ManyToManyField('auth.User', related_name='allowed_files', blank=True)
    allowed_groups = models.ManyToManyField('auth.Group', blank=True)
    uploaded_at = models.DateTimeField(auto_now_add=True)
    
    def can_access(self, user):
        """检查用户是否有访问权限"""
        if self.access_level == 'public':
            return True
        
        if user == self.owner:
            return True
        
        if self.access_level == 'shared':
            if user in self.allowed_users.all():
                return True
            if user.groups.filter(id__in=self.allowed_groups.values_list('id', flat=True)).exists():
                return True
        
        return user.is_staff

# 访问控制装饰器
def require_file_permission(permission_level='read'):
    """文件权限装饰器"""
    def decorator(view_func):
        def wrapper(request, *args, **kwargs):
            file_id = kwargs.get('file_id') or request.GET.get('file_id')
            
            if file_id:
                try:
                    from .models import AccessControlledFile
                    file_obj = AccessControlledFile.objects.get(id=file_id)
                    
                    if not file_obj.can_access(request.user):
                        return HttpResponseForbidden("无权访问此文件")
                    
                    if permission_level == 'download' and not file_obj.can_download(request.user):
                        return HttpResponseForbidden("无权下载此文件")
                        
                except AccessControlledFile.DoesNotExist:
                    return Http404("文件不存在")
            
            return view_func(request, *args, **kwargs)
        return wrapper
    return decorator

# 临时访问链接
import uuid
from datetime import datetime, timedelta

class TemporaryAccessLink(models.Model):
    """临时访问链接"""
    file = models.ForeignKey(AccessControlledFile, on_delete=models.CASCADE)
    access_key = models.UUIDField(default=uuid.uuid4, unique=True)
    created_by = models.ForeignKey('auth.User', on_delete=models.CASCADE)
    expires_at = models.DateTimeField()
    max_downloads = models.PositiveIntegerField(default=1)
    downloads_count = models.PositiveIntegerField(default=0)
    is_active = models.BooleanField(default=True)
    
    @classmethod
    def create_link(cls, file, creator, hours=24, max_downloads=1):
        """创建临时访问链接"""
        return cls.objects.create(
            file=file,
            created_by=creator,
            expires_at=datetime.now() + timedelta(hours=hours),
            max_downloads=max_downloads
        )
    
    def is_valid(self):
        """检查链接是否有效"""
        return (self.is_active and 
                self.expires_at > datetime.now() and
                self.downloads_count < self.max_downloads)
    
    def use_link(self):
        """使用链接（增加下载计数）"""
        if self.is_valid():
            self.downloads_count += 1
            if self.downloads_count >= self.max_downloads:
                self.is_active = False
            self.save()
            return True
        return False

# 临时链接视图
class TemporaryDownloadView(View):
    """临时链接下载视图"""
    
    def get(self, request, access_key):
        try:
            link = TemporaryAccessLink.objects.get(access_key=access_key)
            
            if not link.is_valid():
                return HttpResponseForbidden("链接已失效")
            
            # 使用链接
            if not link.use_link():
                return HttpResponseForbidden("无法使用链接")
            
            # 返回文件
            file_path = link.file.file.path
            with open(file_path, 'rb') as f:
                response = HttpResponse(f.read(), content_type='application/octet-stream')
                response['Content-Disposition'] = f'attachment; filename="{os.path.basename(file_path)}"'
                return response
                
        except TemporaryAccessLink.DoesNotExist:
            return Http404("链接不存在")

# AWS S3存储集成
"""
pip install django-storages[boto3]
pip install boto3
"""

# settings.py 配置
"""
# AWS配置
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', 'us-east-1')
AWS_S3_CUSTOM_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com'

# S3设置
AWS_S3_OBJECT_PARAMETERS = {
    'CacheControl': 'max-age=86400',
}
AWS_DEFAULT_ACL = 'public-read'
AWS_S3_VERIFY = True
AWS_S3_REGION_NAME = AWS_S3_REGION_NAME

# 静态文件S3存储
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'

# 媒体文件S3存储
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
"""

# S3存储配置类
class S3StorageConfig:
    """S3存储配置"""
    
    @staticmethod
    def get_public_media_settings():
        """公共媒体文件设置"""
        return {
            'AWS_STORAGE_BUCKET_NAME': os.environ.get('AWS_PUBLIC_BUCKET'),
            'AWS_S3_CUSTOM_DOMAIN': f'{os.environ.get("AWS_PUBLIC_BUCKET")}.s3.amazonaws.com',
            'AWS_DEFAULT_ACL': 'public-read',
            'AWS_S3_OBJECT_PARAMETERS': {
                'CacheControl': 'max-age=86400',
            },
        }
    
    @staticmethod
    def get_private_media_settings():
        """私有媒体文件设置"""
        return {
            'AWS_STORAGE_BUCKET_NAME': os.environ.get('AWS_PRIVATE_BUCKET'),
            'AWS_DEFAULT_ACL': 'private',
            'AWS_S3_FILE_OVERWRITE': False,
            'AWS_S3_OBJECT_PARAMETERS': {
                'CacheControl': 'max-age=60',
            },
        }

# S3文件操作类
import boto3
from botocore.exceptions import ClientError
from django.core.files.storage import default_storage

class S3FileManager:
    """S3文件管理器"""
    
    def __init__(self):
        self.s3_client = boto3.client(
            's3',
            aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
            aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
            region_name=settings.AWS_S3_REGION_NAME
        )
        self.bucket_name = settings.AWS_STORAGE_BUCKET_NAME
    
    def upload_file(self, file_obj, s3_key, extra_args=None):
        """上传文件到S3"""
        try:
            self.s3_client.upload_fileobj(
                file_obj,
                self.bucket_name,
                s3_key,
                ExtraArgs=extra_args or {
                    'ContentType': getattr(file_obj, 'content_type', 'binary/octet-stream'),
                    'ACL': 'public-read'
                }
            )
            return f"https://{self.bucket_name}.s3.{settings.AWS_S3_REGION_NAME}.amazonaws.com/{s3_key}"
        except ClientError as e:
            raise Exception(f"S3上传失败: {e}")
    
    def download_file(self, s3_key, local_path):
        """从S3下载文件"""
        try:
            self.s3_client.download_file(self.bucket_name, s3_key, local_path)
        except ClientError as e:
            raise Exception(f"S3下载失败: {e}")
    
    def generate_presigned_url(self, s3_key, expiration=3600):
        """生成预签名URL"""
        try:
            return self.s3_client.generate_presigned_url(
                'get_object',
                Params={'Bucket': self.bucket_name, 'Key': s3_key},
                ExpiresIn=expiration
            )
        except ClientError as e:
            raise Exception(f"生成预签名URL失败: {e}")
    
    def delete_file(self, s3_key):
        """删除S3文件"""
        try:
            self.s3_client.delete_object(Bucket=self.bucket_name, Key=s3_key)
            return True
        except ClientError as e:
            raise Exception(f"S3删除失败: {e}")
    
    def get_file_info(self, s3_key):
        """获取文件信息"""
        try:
            response = self.s3_client.head_object(Bucket=self.bucket_name, Key=s3_key)
            return {
                'size': response['ContentLength'],
                'last_modified': response['LastModified'],
                'content_type': response['ContentType'],
                'etag': response['ETag'].strip('"')
            }
        except ClientError as e:
            raise Exception(f"获取文件信息失败: {e}")

# S3优化的模型字段
from storages.backends.s3boto3 import S3Boto3Storage

class PublicMediaStorage(S3Boto3Storage):
    """公共媒体存储"""
    location = 'media'
    default_acl = 'public-read'
    file_overwrite = False
    signature_version = 's3v4'

class PrivateMediaStorage(S3Boto3Storage):
    """私有媒体存储"""
    location = 'private'
    default_acl = 'private'
    file_overwrite = False
    signature_version = 's3v4'

# 使用S3存储的模型
class S3Document(models.Model):
    """S3文档模型"""
    title = models.CharField(max_length=200)
    public_file = models.FileField(
        upload_to='public_docs/',
        storage=PublicMediaStorage(),
        blank=True,
        null=True
    )
    private_file = models.FileField(
        upload_to='private_docs/',
        storage=PrivateMediaStorage(),
        blank=True,
        null=True
    )
    uploaded_at = models.DateTimeField(auto_now_add=True)

# S3信号处理器
@receiver(post_save, sender=S3Document)
def optimize_s3_file(sender, instance, **kwargs):
    """优化S3文件处理"""
    s3_manager = S3FileManager()
    
    # 为新上传的文件设置优化参数
    if instance.public_file:
        # 可以在这里进行文件优化，如压缩、转码等
        pass

    if instance.private_file:
        # 设置私有文件的额外安全参数
        pass