#Python urllib 模块使用指南

from urllib.request import urlopen

# 发送一个无参数的 GET 请求
response = urlopen('https://www.python.org')
# 读取返回的字节流，并解码为 UTF-8 文本
print(response.read().decode('utf-8'))

from urllib.parse import urlencode
from urllib.request import urlopen

# 准备要提交的参数
data_dict = {'word': 'hello urllib', 'author': 'germey'}
# 用 urlencode 转换成 URL 编码的字符串，再编码成 UTF-8 字节
data_bytes = bytes(urlencode(data_dict), encoding='utf-8')

# 只要传入了 data 参数，就会自动变成 POST 请求
response = urlopen('https://httpbin.org/post', data=data_bytes)
print(response.read().decode('utf-8'))

import socket
import urllib.error
from urllib.request import urlopen

try:
    # 故意设置一个极短的超时时间来测试异常
    response = urlopen('https://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
    # 判断是不是超时错误
    if isinstance(e.reason, socket.timeout):
        print('⚠️ 请求超时，请检查网络或适当增加 timeout')

from urllib.request import Request, urlopen

# 1. 用 Request 构造请求
req = Request('https://python.org')
# 2. 动态添加请求头（伪装成浏览器非常重要！）
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
# 3. 发送请求
response = urlopen(req)
print(response.status)  # 输出状态码 200

Request(
    url,
    data=None,               # POST 时传入字节流
    headers={},              # 请求头字典，也可以后续用 add_header 添加
    origin_req_host=None,    # 发起请求的主机（通常自动生成）
    unverifiable=False,      # 很少用到，默认即可
    method=None              # 显式指定请求方法：GET / POST / PUT / DELETE 等
)

from urllib.request import (
    HTTPPasswordMgrWithDefaultRealm,
    HTTPBasicAuthHandler,
    build_opener
)

# 1. 创建密码管理器
password_mgr = HTTPPasswordMgrWithDefaultRealm()
# 2. 添加认证信息
password_mgr.add_password(
    realm=None,   # 一般网站没有指定 realm，留空即可
    uri='https://ssr3.scrape.center/',
    user='admin',
    passwd='admin'
)
# 3. 创建认证 Handler
auth_handler = HTTPBasicAuthHandler(password_mgr)
# 4. 用 Handler 构建自定义 Opener
opener = build_opener(auth_handler)
# 5. 发送带认证信息的请求
response = opener.open('https://ssr3.scrape.center/')
print(response.read().decode('utf-8'))

from urllib.request import ProxyHandler, build_opener

# 1. 配置代理字典（HTTP 和 HTTPS 需要分开写）
proxy_dict = {
    'http': 'http://127.0.0.1:8080',
    'https': 'https://127.0.0.1:8080'
}
# 2. 创建代理 Handler
proxy_handler = ProxyHandler(proxy_dict)
# 3. 构建 Opener
opener = build_opener(proxy_handler)
# 4. 通过代理发送请求
try:
    response = opener.open('https://www.baidu.com', timeout=3)
    print('✅ 代理连接成功')
except Exception as e:
    print(f'❌ 代理连接失败：{e}')

from http.cookiejar import CookieJar, MozillaCookieJar
from urllib.request import HTTPCookieProcessor, build_opener

# -- 方式1：只在内存中保存 Cookies（临时会话）
cookie_jar = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie_jar)
opener = build_opener(cookie_handler)
opener.open('https://www.baidu.com')

print('内存中的 Cookies：')
for cookie in cookie_jar:
    print(f'  {cookie.name}: {cookie.value}')

# -- 方式2：把 Cookies 持久化到本地文件
# MozillaCookieJar 兼容浏览器格式的 cookies.txt
local_cookie = MozillaCookieJar('baidu_cookies.txt')
local_handler = HTTPCookieProcessor(local_cookie)
local_opener = build_opener(local_handler)
local_opener.open('https://www.baidu.com')
# 保存到文件，忽略过期和临时性限制
local_cookie.save(ignore_discard=True, ignore_expires=True)
print('\n✅ Cookies 已保存到 baidu_cookies.txt')

from urllib.request import urlopen
from urllib.error import HTTPError, URLError

try:
    response = urlopen('https://httpbin.org/status/404', timeout=3)
except HTTPError as e:
    # HTTP 类错误，例如 404 Not Found、500 Internal Server Error
    print(f'❌ HTTP错误：状态码 {e.code}，原因 {e.reason}')
    # 也可以进一步查看响应头
    # print(e.headers)
except URLError as e:
    # URL 类错误，例如域名不存在、网络不可达、超时等
    print(f'❌ URL错误：{e.reason}')
else:
    # 没有异常时才会执行到这里
    print('✅ 请求成功')

from urllib.parse import urlparse

url = 'https://www.baidu.com/index.html;user?id=5&name=test#comment'
result = urlparse(url)
print(result)
# 输出：ParseResult(scheme='https', netloc='www.baidu.com', path='/index.html', params='user', query='id=5&name=test', fragment='comment')

# 通过属性获取指定部分
print(f'协议：{result.scheme}')
print(f'域名：{result.netloc}')
print(f'路径：{result.path}')

from urllib.parse import urlunparse, urljoin

# 用 urlunparse 拼接
data = ['https', 'www.baidu.com', '/index.html', '', 'id=6', 'new_comment']
print(urlunparse(data))  # 输出：https://www.baidu.com/index.html?id=6#new_comment

# urljoin 处理相对链接
base_url = 'https://www.example.com/blog/'
rel_url1 = 'article1.html'
rel_url2 = '../about.html'
print(urljoin(base_url, rel_url1))  # https://www.example.com/blog/article1.html
print(urljoin(base_url, rel_url2))  # https://www.example.com/about.html

from urllib.parse import urlencode, quote, unquote

# urlencode：把字典转换成 URL 查询字符串
params = {'keyword': 'Python爬虫', 'page': 1, 'size': 10}
base_url = 'https://www.example.com/search?'
full_url = base_url + urlencode(params)
print(full_url)  # 中文会被自动编码

# quote / unquote：单独处理字符串的编码和解码
chinese = 'Python入门到放弃'
encoded = quote(chinese)
decoded = unquote(encoded)
print(f'编码后：{encoded}')
print(f'解码后：{decoded}')