python3 urllib

官方文档

官方文档：https://docs.python.org/3/library/urllib.html

获取页面内容

第一种方式

import urllib.request

url = 'https://www.baidu.com/'

r = urllib.request.urlopen(url)

print(r)  # <http.client.HTTPResponse object at 0x00000201E6C66CF8>

print(r.read().decode('utf-8'))

另一种方式

import urllib.request

url = 'http://www.cnblogs.com/0bug/'

req = urllib.request.Request(url)

res = urllib.request.urlopen(req)

print(res.read().decode('utf-8'))

发送内容

import urllib.request

import urllib.parse

url = 'http://httpbin.org/post'

data = bytes(urllib.parse.urlencode({'name': 'lcg'}), encoding='utf-8')

r = urllib.request.urlopen(url, data=data)

print(r.read().decode('utf-8'))

设置超时时间

设置超时时间

import urllib.request

url = 'http://www.cnblogs.com/0bug/'

r = urllib.request.urlopen(url, timeout=1)  # 设置超时时间

print(r.read().decode('utf8'))

异常处理

import urllib.request

import urllib.error

import socket

url = 'http://www.cnblogs.com/0bug/'

try:

    r = urllib.request.urlopen(url, timeout=0.01)  # 设置超时时间

    print(r.read().decode('utf8'))

except urllib.error.URLError as e:

    if isinstance(e.reason, socket.timeout):

        print('请求超时')

响应码、响应头

import urllib.request

url = 'http://www.cnblogs.com/0bug/'

r = urllib.request.urlopen(url)

print(r.status)  # 200

print(r.getheaders())  # [(('Content-Type', 'text/html; charset=utf-8'),......]

print(r.getheader('Content-Type'))  # text/html; charset=utf-8

构造请求信息

import urllib.request

import urllib.parse

url = 'http://www.cnblogs.com/0bug/'

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36',

    'Host': 'www.cnblogs.com'

}

dic = {'name': 'lcg'}

data = bytes(urllib.parse.urlencode(dic), encoding='utf-8')

req = urllib.request.Request(url=url, data=data, headers=headers, method='POST')

res = urllib.request.urlopen(req)

print(res.read().decode('utf-8'))

另一种添加请求头的方式

import urllib.request

import urllib.parse

url = 'http://www.cnblogs.com/0bug/'

dic = {'name': 'lcg'}

data = bytes(urllib.parse.urlencode(dic), encoding='utf-8')

req = urllib.request.Request(url=url, data=data, method='POST')

req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/53...')

res = urllib.request.urlopen(req)

print(res.read().decode('utf-8'))

代理

import urllib.request

url = 'http://www.cnblogs.com/0bug/'

proxy_handler = urllib.request.ProxyHandler({

    'http': 'http://122.114.31.177:808',

    'https': 'https://124.133.75.183:8118'

})

opener = urllib.request.build_opener(proxy_handler)

r = opener.open(url)

print(r.read().decode('utf-8'))