Python下载实战技巧全解析

Python下载实战技巧技术文章大纲

核心模块与库概述

• requests库：简单高效的HTTP请求库，适合绝大多数下载场景
• urllib标准库：Python内置模块，无需额外安装
• aiohttp：异步下载解决方案，提升大规模下载效率
• wget模块：模拟命令行wget功能的Python实现

基础下载方法

HTTP/HTTPS文件下载基础实现

import requests
url = 'https://example.com/file.zip'
response = requests.get(url)
with open('local_file.zip', 'wb') as f:
    f.write(response.content)

处理下载进度显示

from tqdm import tqdm
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
with open('file.zip', 'wb') as f, tqdm(
    total=total_size, unit='B', unit_scale=True
) as pbar:
    for data in response.iter_content(1024):
        f.write(data)
        pbar.update(len(data))

高级下载技巧

断点续传实现方案

headers = {}
if os.path.exists('partial.file'):
    file_size = os.path.getsize('partial.file')
    headers = {'Range': f'bytes={file_size}-'}

response = requests.get(url, headers=headers, stream=True)
mode = 'ab' if 'content-range' in response.headers else 'wb'

多线程分块下载技术

import concurrent.futures

def download_chunk(url, start, end, filename):
    headers = {'Range': f'bytes={start}-{end}'}
    response = requests.get(url, headers=headers)
    with open(filename, 'r+b') as f:
        f.seek(start)
        f.write(response.content)

特殊场景处理

处理大文件内存优化

with requests.get(url, stream=True) as r:
    r.raise_for_status()
    with open('large_file.iso', 'wb') as f:
        for chunk in r.iter_content(chunk_size=8192):
            f.write(chunk)

绕过Cloudflare等防护系统

import cloudscraper
scraper = cloudscraper.create_scraper()
html = scraper.get(url).text

异常处理与调试

常见错误代码处理方案

try:
    response = requests.get(url, timeout=10)
    response.raise_for_status()
except requests.exceptions.HTTPError as err:
    print(f"HTTP错误: {err}")
except requests.exceptions.Timeout:
    print("请求超时")
except requests.exceptions.RequestException as err:
    print(f"请求异常: {err}")

重试机制实现

from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

session = requests.Session()
retries = Retry(total=5, backoff_factor=1)
session.mount('https://', HTTPAdapter(max_retries=retries))

性能优化策略

连接池配置优化

adapter = requests.adapters.HTTPAdapter(
    pool_connections=100,
    pool_maxsize=100,
    max_retries=3
)
session.mount('https://', adapter)

异步下载加速方案

import aiohttp
import asyncio

async def async_download(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            content = await response.read()
            with open('async_file.bin', 'wb') as f:
                f.write(content)

安全注意事项

SSL证书验证配置

requests.get(url, verify='/path/to/certificate.pem')

敏感文件下载保护

import tempfile
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
    response = requests.get(url)
    tmp_file.write(response.content)
    secure_path = tmp_file.name

实用工具集成

下载管理器类封装

class DownloadManager:
    def __init__(self, max_workers=5):
        self.session = requests.Session()
        self.executor = ThreadPoolExecutor(max_workers)
        
    def download(self, url, dest):
        future = self.executor.submit(self._download, url, dest)
        return future

日志记录与监控

import logging
logging.basicConfig(filename='download.log', level=logging.INFO)
logger = logging.getLogger(__name__)

def download_with_log(url):
    try:
        response = requests.get(url)
        logger.info(f"成功下载 {url}")
    except Exception as e:
        logger.error(f"下载失败 {url}: {str(e)}")