Python爬虫病例信息

必须安装的第三方库

pip install urllib

pip install pypingin

# -*- coding:utf-8 -*-
import re
import urllib.request
import urllib
import xlwt
import time

import pypinyin
import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException
# 网址
buwei_url1 = "https://jbk.99.com.cn/"
buwei_url2 = ".html"

# 爬取部位信息https://jbk.99.com.cn/部位/
buwei_url = buwei_url1 + 'buwei/'
try:
    buwei_urlop = urllib.request.urlopen(buwei_url, timeout=100)
    # buwei_urlop = requests.get(buwei_url)
    # print(buwei_urlop)
except Exception as e:

    print("超时")
# HTTP异常
except HTTPError:
    print('httperror')
# 请求异常
except RequestException:
    print('reqerror')
data_buwei = buwei_urlop.read().decode('GBK')  # ！！！！！原网页编码方式为GBK
# 得到部位名称，保存在列表ls_buwei中
link_buwei = "<div class=\"part-txt\">\\s*<h3>\\s*<span>\\s*(.*?)</span>"
ls_buwei = []
patten = re.compile(link_buwei)
ls_buwei = patten.findall(data_buwei)
print(ls_buwei)

# 新建表格文本
book = xlwt.Workbook(encoding='utf-8')
heads = [u'疾病名称', u'链接', u'发病部位', u'就诊科室', u'体状特征', u'症状']


def hp(word):
    s = ''
    for i in pypinyin.pinyin(word, style=pypinyin.NORMAL):
        s += ''.join(i)
    return s

# 得到各部位疾病名称
for i, bw in enumerate(ls_buwei):
    catch = 1  # 更换部位开关
    ii = 31
    # 同一部位疾病放置在一个sheet中
    sheet = book.add_sheet(bw, cell_overwrite_ok=True)
    for j, head in enumerate(heads):
        sheet.write(0, j, head)

    while catch == 1:
        ii += 1

        if ii >60:
            break
        # 具体部位网址
        buwei_url_s = buwei_url + hp(bw) + '-' + str(ii) + buwei_url2
        # print(buwei_url_s)
        try:
            # 抓网页
            buwei_urlop_s = urllib.request.urlopen(buwei_url_s, timeout=100)
        except Exception:
            print('超时')
            continue
        try:
            data_buwei = buwei_urlop_s.read().decode('GBK')
            # 网页源代码抓取
        except:
            continue
        # 判断是否是最后一页，对比最后一页源代码可得
        link_is_msg = "<div class=\"part-cont3\">\\s*(.*?)<div class=\"digg\">"
        patten_is_msg = re.compile(link_is_msg)
        is_msg = patten_is_msg.findall(data_buwei)
        # 如果是最后一页切换到下一个部位
        if is_msg == ['']:
            catch = 0
            continue

        # 获取本页所有疾病名称及链接，结果保存在列表ls_des中
        link_buwei = "<dd>\\s*<h3>\\s*<span>\\s*</span>\\s*<a href=\"(.*?)\"\\s*title=\".+\" target=\"_blank\">(.*?)</a></h3>"
        patten_buwei = re.compile(link_buwei)
        ls_des = patten_buwei.findall(data_buwei)
        # 抓取ls_des中各疾病的具体信息
        iii = 1
        for x in ls_des:
            t1 = time.clock()

            row = (ii - 1) * 10 + iii
            print(row)
            # 将疾病名称和链接写入表格
            sheet.write(row, 0, x[1])
            sheet.write(row, 1, x[0])
            # 疾病链接
            jibing_url = 'https:' + x[0] + 'zhengzhuang.html'
            print(jibing_url)
            try:
                jibing_urlop = urllib.request.urlopen(jibing_url, timeout=100)
            except Exception:
                print('超时')
                continue
            try:
                # 源代码获取
                data_jibing = jibing_urlop.read().decode('GBK')
            except:
                continue
            # 发病部位信息
            link_fbbw = "<li><font>发病部位：(.*?)</a></li>"
            patten_fbbw = re.compile(link_fbbw)
            st_abbw = patten_fbbw.findall(data_jibing)
            if st_abbw == []:
                hz_fbbw = re.findall('[\u4e00-\u9fa5]+','')
            else:
                hz_fbbw = re.findall('[\u4e00-\u9fa5]+', st_abbw[0])  # 中文提取

            fbbw = ','.join(ff for ff in hz_fbbw)
            sheet.write(row, 2, fbbw)
            # 抓写就诊科室信息
            link_jzks = "<li><font>就诊科室：(.*?)</a></li>"
            patten_jzks = re.compile(link_jzks)
            st_jzks = patten_jzks.findall(data_jibing)
            if st_jzks == []:
                jzks = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+',''))  # 中文提取
            else:
                jzks = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+', st_jzks[0]))  # 中文提取

            sheet.write(row, 3, jzks)
            # 抓写体状特征
            link_tztz = "<dt><a>症状体征：</a></dt>\\s*<dd>\\s*<ul>(.*?)</a></li></ul>"
            patten_tztz = re.compile(link_tztz)
            st_tztz = patten_tztz.findall(data_jibing)
            if st_tztz == []:
                tztz = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+','')) if st_tztz != [] else 0
            else:
                tztz = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+', st_tztz[0])) if st_tztz != [] else 0

            sheet.write(row, 4, tztz)
            iii += 1
            t2 = time.clock()
            print(t2 - t1)
            book.save("F:/li结果/jibing11.xls")  # 保存表格