from lxml import etree
html = etree.parse(‘hello.html‘)

result = html.xpath(‘//li‘)

print(result)   # 打印
标签的元素集合

result_text = html.xpath(‘//li/text()‘)

2.获取

from lxml import etree

html = etree.parse(‘hello.html‘)
result = html.xpath(‘//li/@class‘)

print result

结果是
[‘item-0‘, ‘item-1‘, ‘item-inactive‘, ‘item-1‘, ‘item-0‘]

3.继续获取

from lxml import etree

html = etree.parse(‘hello.html‘)
result = html.xpath(‘//li/a[@href="http://www.mamicode.com/link1.html"]‘)

print result

运行结果

[]

4.获取最后一个

from lxml import etree

html = etree.parse(‘hello.html‘)

result = html.xpath(‘//li[last()]/a/@href‘)
# 谓语 [last()] 可以找到最后一个元素

print result

运行结果

[‘link5.html‘]

?

xpath 爬虫示例

?? 示例1

#爬取糗事百科中段子内容和作者名称
import requests
from lxml import etree

headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36‘
}
url = ‘https://www.qiushibaike.com/text/‘
page_text = requests.get(url=url, headers=headers).text

#解析内容
tree = etree.HTML(page_text)
div_list = tree.xpath(‘//div[@class="col1 old-style-col1"]/div‘)

for div in div_list:
    author = div.xpath(‘./div[1]/a[2]/h2/text()‘)[0] #实现局部解析
    content = div.xpath(‘./a[1]/div/span//text()‘)
    content = ‘‘.join(content)
    
    print(author, content)

?? 示例2

# http://pic.netbian.com/4kmeinv/  联系xpath和解决中文乱码问题

dirName = ‘./meinvLibs‘
if not os.path.exists(dirName):
    os.mkdir(dirName)
url = ‘http://pic.netbian.com/4kmeinv/index_%d.html‘
for page in range(1,11):
    if page == 1:
        new_url = ‘http://pic.netbian.com/4kmeinv/‘
    else:
        new_url = format(url%page)
    page_text = requests.get(url=new_url, headers=headers).text
    tree = etree.HTML(page_text)
    a_list = tree.xpath(‘//div[@class="slist"]/ul/li/a‘)
    for a in a_list:
        img_src = ‘http://pic.netbian.com‘ + a.xpath(‘./img/@src‘)[0]
        img_name = a.xpath(‘./b/text()‘)[0]
        img_name = img_name.encode(‘iso-8859-1‘).decode(‘gbk‘)
        img_data = requests.get(url=img_src, headers=headers).content
        imgPath = dirName+‘/‘+img_name+‘.jpg‘
        with open(imgPath, ‘wb‘) as fp:
            fp.write(img_data)
            print(img_name, "下载成功！！！")

?? 示例3

# http://sc.chinaz.com/jianli/free.html 爬取简历模板前五页  通过xpath解析
import requests, os
from lxml import etree

save_resume_dir = ‘./jianli‘
if not os.path.exists(save_resume_dir):
    os.mkdir(save_resume_dir)

headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36‘
}

url = ‘http://sc.chinaz.com/jianli/free_%d.html‘

for page in range(1,6):
    if page == 1:
        new_url = ‘http://sc.chinaz.com/jianli/free.html‘
    else:
        new_url = format(url%page)
    page_text = requests.get(url=new_url, headers=headers).text  #向每一页发起请求
    tree = etree.HTML(page_text)
    a_list = tree.xpath(‘//div[@id="container"]/div/a‘)   #获取每一页中所有的简历
    for a in a_list:
        resume_url = a.xpath(‘./@href‘)[0]  #得到每一份简历的url地址
        resume_anme = a.xpath(‘./img/@alt‘)[0]  #得到每一份简历的名字
        resume_anme = resume_anme.encode(‘iso-8859-1‘).decode(‘utf-8‘)  #解决简历名字乱码
        resume_text = requests.get(url=resume_url, headers=headers).text  #向每一份简历的页面发起请求
        tree = etree.HTML(resume_text)
        resume_down_url = tree.xpath(‘//div[@class="down_wrap"]/div[2]/ul/li[1]/a/@href‘)[0]  #对每一份简历页面请求数据进行解析，得到下载的url
        resume_down_text = requests.get(url=resume_down_url, headers=headers).content  #下载每一份简历
        resume_path = save_resume_dir + ‘/‘ + resume_anme + ‘.rar‘
        with open(resume_path, ‘wb‘) as fp:
            fp.write(resume_down_text)
            print(resume_anme+‘简历模板下载成功！！！‘)

?? 示例4

# 爬取梨视频任意板块最热视频  https://www.pearvideo.com/category_8
import requests
from lxml import etree
import re

headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36‘
}

main_url = ‘https://www.pearvideo.com/category_8‘

main_text = requests.get(url=main_url, headers=headers).text
tree = etree.HTML(main_text)

div_list = tree.xpath(‘//*[@id="listvideoListUl"]/li‘)

for li in div_list:
    detile_url = ‘https://www.pearvideo.com/‘ + li.xpath(‘./div/a/@href‘)[0]  #获取视频详情页
    title = li.xpath(‘./div/a/div[2]/text()‘)[0] + ‘.mp4‘  #获取视频名字

    detile_page_text = requests.get(url=detile_url, headers=headers).text  #向视频详情页发起请求

    ex = ‘srcUrl="(.*?)",vdoUrl‘   #通过正则获取js动态加载的数据
    video_url = re.findall(ex, detile_page_text, re.S)[0]  #解析出视频url的地址
    video_data = requests.get(video_url, headers=headers).content   #对视频url发起请求进行下载

    with open(f‘./video/{title}‘, ‘wb‘) as fp:
        fp.write(video_data)

Python--爬虫数据解析

标签：value   作者   agent   att   源码   乱码   属性   sele   rap   

原文地址：https://www.cnblogs.com/yanjieli/p/13353926.html