WOS论文详细数据页面信息selenium爬虫

最新推荐文章于 2026-04-13 09:53:14 发布

原创

最新推荐文章于 2026-04-13 09:53:14 发布 · 1.1k 阅读

标签

#python #selenium #爬虫

本文介绍了一个使用 Python 的 Selenium 库从 Web of Science 平台自动爬取学术论文详细信息的示例程序。该程序能够获取每篇论文的标题、作者、来源期刊、DOI、关键词等关键元数据，并将其保存为 CSV 文件。

该文章已生成可运行项目，

import random

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
import csv
import time
import re

def start_spider():
    browser.get(url)
    page = browser.page_source
    time.sleep(2)
    title=browser.find_element_by_class_name("title").text
    print(title)#标题
    list.append(title)

    author_list=[]
    authors=browser.find_elements_by_xpath('.//a[@title="查找此作者的更多记录"]')
    authors_num=browser.find_elements_by_tag_name("sup")
    # for author in authors:
    #     author_list.append(author.text)
    for author in range(0,len(authors)):
        try:
            author_list.append(authors[author].text + authors_num[author].text)
        # pattern3 = re.compile('alt="Find more records by this author">(.*?)</a>')
        # author = re.findall(pattern3, html)
        except:
            continue
    list.append(author_list)


    # ###作者详细信息
    # author_detail=browser.find_element_by_xpath("//*[@id='show_resc_blurb']/table")
    # tr_content=author_detail.find_elements_by_tag_name("tr")
    detailauthor_list = []
    # for tr in tr_content:
    #     try:
    #         tds=tr.find_elements_by_tag_name("td")
    #         for td in tds:
    #             detailauthor_list.append(td.get_attribute('textContent'))
    #             print(detailauthor_list)
    #             print(td.get_attribute('textContent'))
    #         # name=tr.find_element_by_tag_name("display_name").text
    #         # Web_of_Science_ResearcherID=tr.find_element_by_class_name("snowplow-RIDnumber").text
    #         # orcid=tr.find_element_by_class_name("snowplow-ORCIDnumber").text
    #         # detailauthor_list.append(name)
    #         # detailauth