python超链接抓取工具

python实现自动抓取某站点内所有超链接

（仅供学习使用）

代码部分

#!/usr/bin/python

import requests

import time

import re

import sys, getopt  #命令行选项

from bs4 import BeautifulSoup

localtime=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) #时间

z=[] #存取网站

x=[] #优化网站，去除冗杂部分

def main(argv):

   url = '' #输入的网址

   file_path = '' #保存路径

   try:

      opts, args = getopt.getopt(argv,"hu:f:",["url=","file="])

   except getopt.GetoptError:

      print ('allsite.py -u <url> -f <file>')

      sys.exit(2)

   for opt, arg in opts:

      if opt == '-h': #帮助

         print ('allsite.py -u <url> -f <file>')

         sys.exit()

      elif opt in ("-u", "--url"): #输入网址

         url = arg

         re1 = requests.get(url) #get网站内容

         re1.encoding = "utf-8"

         html = re1.text

         bt = BeautifulSoup(html, 'html.parser', )

         hh = bt.find_all('a') #查找<a>元素

         for site in hh:

                 z.append(site.get('href')) #进一步过滤得到超链接

         for i in z:

                 if (re.match('//www', str(i)) or re.match('www', str(i))):

                         xx = str(i).replace('//www', 'www', 1)

                         x.append(xx)

                 elif (re.match('http', str(i))): #过滤

                         x.append(str(i))

                 elif (re.match('/', str(i))): #过滤

                         xx = str(i).replace("/", "", 1)

                         if (re.match('/', xx)):

                                 xxx = str(xx).replace("/", "", 1)

                                 x.append(xxx)

                         else:

                                 x.append(url + xx)

                 else: #过滤

                         if (re.search('javascript', str(i)) == None):

                                 x.append(url + str(i))

         print(localtime + "  总共:" + str(len(x)) + "个网址") #输出超链接

         for i in x:

                 print(i)

      elif opt in ("-f", "--file"): #输入保存路径

         file_path = arg

         for i in x: #保存文件

                 with open(file_path, 'a') as file_object:

                         file_object.write(i)

                         file_object.write('\n')

if __name__ == "__main__":

   main(sys.argv[1:])