爬虫系列之requests

爬取百度内容：

 import requests

 url = "https://www.baidu.com"

 if __name__ == '__main__':

     try:

         kv = {'user-agent': 'Mozilla/5.0'}

         r = requests.get(url, headers=kv)

         r.raise_for_status()  #返回状态值，如果不是200，则抛出异常

         r.encoding = r.apparent_encoding

         print(r.text)

         #print(r.request.headers)

     except:

         print("爬虫失败")

在URL中填上http://www.baidu.com/s?wd=keyword，keyword就是我们要百度搜索的内容，在requests中有params参数，可以把参数追加到URL中。

 import requests

 url = "http://www.baidu.com/s"

 keyword = "python"

 if __name__ == '__main__':

     try:

         kv = {'user-agent': 'Mozilla/5.0'}

         wd = {'wd': keyword}

         r = requests.get(url, headers=kv, params=wd)

         print(r.request.url)

         r.raise_for_status()

         r.encoding = r.apparent_encoding

         print(len(r.text))

     except:

         print("爬虫失败")

爬取图片

 import requests

 import os

 url = "http://image.nationalgeographic.com.cn/2017/0211/20170211061910157.jpg"

 kv = {'header': 'Mozilla/5.0'}

 root = "D://pic_save//"

 path = root + url.split('/')[-1]

 if __name__ == '__main__':

     try:

         if not os.path.exists(root):

             os.mkdir(root)

         if not os.path.exists(path):

             r = requests.get(url, headers=kv)

             print(r.status_code)

             with open (path, 'wb') as f:

                 f.write(r.content)

                 print("文件已保存成功")

         else:

             print("文件已存在")

     except:

         ("爬虫失败")