wooyun本地数据抓取

----

#-*-coding:utf-8-*-

import re

import urllib

import MySQLdb

import time

from urllib import unquote

def getHtml(url):

    page = urllib.urlopen(url)

    html = page.read()

    html = html.replace('\n', '')

    html = html.replace('       ', ' ')

    html = html.replace('   ', '')

    html = html.replace('   ', '')

    #html = html.replace(' ','')

    return html

    return mylist

def gettitle(mylist):

    reg = (r'<a href="/bugs/wooyun-.+">(.*?)</a></td>')

    listre = re.compile(reg)

    mytitle = re.findall(listre, mylist)

    return mytitle

def getoper(html):

    reg = (r'/whitehats/(.*?)">')

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist[0]

#-------------------------------------------------

def GetTitle(html):

    reg = (r"漏洞标题：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def BugNum(html):

    reg = (r'http://wooyun.org/bugs/(.*?)">查看原始来源')

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def JiaFang(html):

    reg = (r'http://www.wooyun.org/corps/(.*?)">')

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def SubmitTime(html):

    reg = (r"提交时间：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def OpenTime(html):

    reg = (r"公开时间：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def BugClass(html):

    reg = (r"漏洞类型：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def level(html):

    reg = (r"危害等级：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def BugState(html):

    reg = (r"漏洞状态：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    mylist = mylist[0].strip()

    return mylist

def BugSave(html):

    reg = (r'<a id="collection_num">(.*?)</a>人收藏')

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def OkTime(html):

    reg = (r"确认时间：(.*?)</p>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def Bugrank(html):

    reg = (r"漏洞Rank：(.*?)</p>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def BugMark(html):

    reg = (r"Tags标签：(.*?)</h3>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    return mylist

def ignoreTime(html):

    reg = (r"忽略时间：(.*?)</p>")

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    if len(mylist)!=0:

        mylist=mylist

    else:

        mylist='1900-01-01 00:00:00'

    return mylist

def Bugeye(html):

    reg = (r'<span id="attention_num">(.*)</span>')

    listre = re.compile(reg)

    mylist = re.findall(listre, html)

    mylist = mylist[0].strip()

    return mylist

conn= MySQLdb.connect(

        host='192.168.1.1',

        port = 3306,

        user='root',

        passwd='root',

        db ='wooyunTongji',

        charset='utf8'

        )

mark = 0

for i in range(53022, 89250, 1):

    try:

        Url = 'http://192.168.1.106/wooyun/select.php?id='+str(i)

        Html = getHtml(Url)

        Htmleye = getHtmleye(Url)

    except:

        print 'error'

    if len(Html)>100:

        if len(OkTime(Html)) == 0:

            whotime = ignoreTime(Html)[0].strip()

            whostyle = '忽略'

        else:

            whotime = OkTime(Html)[0].strip()

            whostyle = '确认'

        if len(Bugrank(Html)) < 1:

            BugrankFal='0'

        else:

            BugrankFal=Bugrank(Html)[0]

        print GetTitle(Html)[0].strip() \

            ,BugNum(Html)[0].strip() \

            ,unquote(JiaFang(Html)[0].strip()) \

            ,unquote(getoper(Html)) \

            ,SubmitTime(Html)[0].strip() \

            ,OpenTime(Html)[0].strip() \

            ,BugClass(Html)[0].strip() \

            ,level(Html)[0] \

            ,BugrankFal\

            ,BugState(Html) \

            ,whotime \

            ,whostyle \

            ,BugMark(Html)[0].strip()

        #f = open('wooyunlist.txt', 'a')

        mark += 1

        #f.close()

        cur = conn.cursor()

        mysql1='insert into alldata (id,title,BugNum,jiafang,oper,submittime,opentime,bugclass,level,bugrank,bugstate,oktime,okstyle,bugmark) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'

        cur.execute(mysql1,(mark,GetTitle(Html)[0].strip(),BugNum(Html)[0].strip(),unquote(JiaFang(Html)[0].strip()),unquote(getoper(Html)),SubmitTime(Html)[0].strip(),OpenTime(Html)[0].strip(),BugClass(Html)[0].strip(),level(Html)[0],BugrankFal,BugState(Html),whotime,whostyle,BugMark(Html)[0].strip()))

        cur.close()

        conn.commit()

        print mark

conn.close()

print('Over!')