python基础爬虫——BeautifulSoup查找元素-CSDN博客

本文介绍了Python爬虫中BeautifulSoup库的查找元素功能，包括find_all和find两个函数的使用。find_all返回元素列表，find返回第一个匹配的元素。文中通过示例代码展示了这两个函数的用法。

困于下衡于虑而后作
人生不能放弃
今天的学习目标是：beautiful soup 查找元素
总共有两个函数find_all和find
find_all返回列表，find返回查找的第一个值
1.python代码——find的使用

from bs4 import BeautifulSoup

# find_all or find
doc = '''
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>The Dormouse's story</title>
</head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">
    Once upon a time there were three little sister;and their names were
    <a href="http://example.com/else" class="sister" id="link1">Elsie</a>
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
    and they lived at the bottom of a well
</p>
<p class="story">...</p>
</body>
</html>
'''
soup = BeautifulSoup(doc, 'lxml')  # 解析html变成beautifulsoup对象
tag = soup.find("title")  # 查找第一个标题标签，title一般只有一个
print(type(tag), tag)  # 显示标签的类型和内容
tag = soup.find("a")  # 查找第一个a标签
print(type(tag), tag)  # 显示标签的类型和内容
tag = soup.find('p', attrs={'class': 'title'}) # 查找文本当中class为title的<p>元素
print(type(tag), tag)  # 显示标签的类型和内容
soup = BeautifulSoup(doc, 'lxml')
print(soup.name)  # 先打印文档名
tag = soup.find('b')
while tag:
    print(tag.name)  # 打印标签名，再进行查找父节点
    tag = tag.parent


soup = BeautifulSoup(doc, 'lxml')
print(soup.name)  # 先打印文档名
tag = soup.find('p')
for x in tag.children:  # 获取p元素的所有直接子节点
    print(x)

这里需要对比doc变量中的html文本
在这里插入图片描述

# 查找前后兄弟节点
soup = BeautifulSoup(doc, 'lxml')
print(soup.name)  # 先打印文档名
tag = soup.find('a')
print(tag.previous_siblings)
print(tag.previous)
print(tag.next)

运行后的结果
在这里插入图片描述

from bs4 import BeautifulSoup



doc='''
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>The Dormouse's story</title>
</head>
<body>
<p class="title"><b>The Dormouse's story<i>the</i></b></p>
<p class="story">
    Once upon a time there were three little sister;and their names were
    <a href="http://example.com/else" class="sister" id="link1">Elsie</a>
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
    and they lived at the bottom of a well
</p>
<p class="story">...</p>
</body>
</html>
'''
# 获取<p>元素的所有子孙节点
soup = BeautifulSoup(doc, 'lxml')
tag =soup.find('p')
for x in tag.descendants:
    print(x)


# 获取<b>元素的前后兄弟节点
soup = BeautifulSoup(doc, 'lxml')
tag =soup.find('b')
print(tag.previous_sibling)
print(tag.next_sibling)
tag=soup.find('i')
print(tag.previous_sibling)
print(tag.next_sibling)

2.python代码——find_all的使用

from bs4 import BeautifulSoup

# find_all or find
doc = '''
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>The Dormouse's story</title>
</head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">
    Once upon a time there were three little sister;and their names were
    <a href="http://example.com/else" class="sister" id="link1">Elsie</a>
    <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
    <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
    and they lived at the bottom of a well
</p>
<p class="story">...</p>
</body>
</html>
''' 
tags = soup.find_all("a")  # 找到所有的a标签
for t in tags:
    print(t)  # 遍历输出
tags = soup.find_all(name=None, attrs={'class': 'sister'})  # 查找所有的class为sister的元素
for tag in tags:
    print(tag)
tags = soup.find_all('a')  # 查找所有的标签a，返回为列表
for tag in tags:
    print(tag['href'])  # 打印href的链接，打印超链接
tags = soup.find_all('a')
for tag in tags:
    print(tag.text)  # 打印标签a的文本信息
tags = soup.find_all('p')
for tag in tags:
    print(tag.text)  # 打印标签p的文本信息

# 高级查找，查找文档中href='http://example.com/的节点元素<a>
def myFilter(tag):
    print(tag.name)
    return (tag.name == 'a' and tag.has_attr('href') and tag['href'] == 'http://example.com/lacie')


soup = BeautifulSoup(doc, 'lxml')
tag = soup.find_all(myFilter)
print(tag)


# 高级查找，可以查找到一些复杂的节点元素，查找文本值为‘cie’结尾的所有<a>节点
def endsWith(s, t):
    if len(s) >= len(t):
        return s[len(s) - len(t):] == t
    return False


def myFilter(tag):
    return (tag.name == 'a' and endsWith(tag.text, 'cie'))


soup = BeautifulSoup(doc, 'lxml')
tags = soup.find_all(myFilter)
for tag in tags:
    print(tag)