#Python urllib 模块使用指南

from urllib.request import urlopen

# 发起无参数的 GET 请求
response = urlopen('https://www.python.org')
# 读取响应字节流，转为 UTF-8 文本
print(response.read().decode('utf-8'))

from urllib.parse import urlencode
from urllib.request import urlopen

# 构造字典参数
data_dict = {'word': 'hello urllib', 'author': 'germey'}
# 1. 用 urlencode 转为 URL 编码字符串
# 2. 再用 bytes() 转为 UTF-8 字节流
data_bytes = bytes(urlencode(data_dict), encoding='utf-8')

# 带 data 参数会自动转为 POST 请求
response = urlopen('https://httpbin.org/post', data=data_bytes)
print(response.read().decode('utf-8'))

import socket
import urllib.error
from urllib.request import urlopen

try:
    # 设置 0.1 秒超时（故意设短测试异常）
    response = urlopen('https://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
    # 判断异常原因是否为超时
    if isinstance(e.reason, socket.timeout):
        print('⚠️ 请求超时，请检查网络或延长 timeout')

from urllib.request import Request, urlopen

# 1. 基础构造
req = Request('https://python.org')
# 2. 动态添加请求头（伪装成浏览器很重要！）
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
# 3. 发起请求
response = urlopen(req)
print(response.status)  # 输出响应码 200

Request(
    url,
    data=None,       # POST 字节流
    headers={},      # 请求头字典，也可以用 add_header 动态加
    origin_req_host=None,  # 请求发起的主机（一般自动生成）
    unverifiable=False,    # 很少用到的参数，默认 False
    method=None      # 显式指定请求方法：GET/POST/PUT/DELETE 等
)

from urllib.request import (
    HTTPPasswordMgrWithDefaultRealm,
    HTTPBasicAuthHandler,
    build_opener
)

# 1. 创建密码管理器
password_mgr = HTTPPasswordMgrWithDefaultRealm()
# 2. 添加验证信息
password_mgr.add_password(
    realm=None,  # 一般网站没有指定，留空即可
    uri='https://ssr3.scrape.center/',
    user='admin',
    passwd='admin'
)
# 3. 创建验证 Handler
auth_handler = HTTPBasicAuthHandler(password_mgr)
# 4. 用 Handler 构建自定义 Opener
opener = build_opener(auth_handler)
# 5. 发起验证请求
response = opener.open('https://ssr3.scrape.center/')
print(response.read().decode('utf-8'))

from urllib.request import ProxyHandler, build_opener

# 1. 配置代理字典（HTTP/HTTPS 分开写）
proxy_dict = {
    'http': 'http://127.0.0.1:8080',
    'https': 'https://127.0.0.1:8080'
}
# 2. 创建代理 Handler
proxy_handler = ProxyHandler(proxy_dict)
# 3. 构建 Opener
opener = build_opener(proxy_handler)
# 4. 发起代理请求
try:
    response = opener.open('https://www.baidu.com', timeout=3)
    print('✅ 代理连接成功')
except Exception as e:
    print(f'❌ 代理连接失败：{e}')

from http.cookiejar import CookieJar, MozillaCookieJar
from urllib.request import HTTPCookieProcessor, build_opener

# --- 情况1：内存中保存 Cookies（临时会话）
cookie_jar = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie_jar)
opener = build_opener(cookie_handler)
# 第一次请求（比如登录后），Cookies 会自动存入 cookie_jar
opener.open('https://www.baidu.com')
# 打印内存中的 Cookies
print('内存中的 Cookies：')
for cookie in cookie_jar:
    print(f'  {cookie.name}: {cookie.value}')

# --- 情况2：保存 Cookies 到本地文件（持久化会话）
# 用 MozillaCookieJar 兼容浏览器的 cookies.txt 格式
local_cookie = MozillaCookieJar('baidu_cookies.txt')
local_handler = HTTPCookieProcessor(local_cookie)
local_opener = build_opener(local_handler)
# 发起请求并保存
local_opener.open('https://www.baidu.com')
local_cookie.save(ignore_discard=True, ignore_expires=True)  # 忽略过期和临时 Cookie
print('\n✅ 已保存到 baidu_cookies.txt')

from urllib.request import urlopen
from urllib.error import HTTPError, URLError

try:
    response = urlopen('https://httpbin.org/status/404', timeout=3)
except HTTPError as e:
    # HTTP 错误：比如 404 未找到、500 服务器错误
    print(f'❌ HTTP错误：状态码 {e.code}，原因 {e.reason}')
    # 可以获取响应头
    # print(e.headers)
except URLError as e:
    # URL 错误：比如域名不存在、网络不通、超时
    print(f'❌ URL错误：{e.reason}')
else:
    # 无异常时执行
    print('✅ 请求成功')

from urllib.parse import urlparse

url = 'https://www.baidu.com/index.html;user?id=5&name=test#comment'
result = urlparse(url)
print(result)
# 输出：ParseResult(scheme='https', netloc='www.baidu.com', path='/index.html', params='user', query='id=5&name=test', fragment='comment')

# 直接用属性访问对应部分
print(f'协议：{result.scheme}')
print(f'域名：{result.netloc}')
print(f'路径：{result.path}')

from urllib.parse import urlunparse, urljoin

# --- urlunparse
data = ['https', 'www.baidu.com', '/index.html', '', 'id=6', 'new_comment']
print(urlunparse(data))  # 输出：https://www.baidu.com/index.html?id=6#new_comment

# --- urljoin（处理 HTML 中的相对链接非常有用）
base_url = 'https://www.example.com/blog/'
rel_url1 = 'article1.html'
rel_url2 = '../about.html'
print(urljoin(base_url, rel_url1))  # https://www.example.com/blog/article1.html
print(urljoin(base_url, rel_url2))  # https://www.example.com/about.html

from urllib.parse import urlencode, quote, unquote

# --- urlencode
params = {'keyword': 'Python爬虫', 'page': 1, 'size': 10}
base_url = 'https://www.example.com/search?'
full_url = base_url + urlencode(params)
print(full_url)  # keyword 会自动编码

# --- quote/unquote
chinese = 'Python入门到放弃'
encoded = quote(chinese)
decoded = unquote(encoded)
print(f'编码后：{encoded}')
print(f'解码后：{decoded}')