使用python+selenium对12306车票数据读取

2021-06-29 23:06

阅读:612

标签:size   cte   div   oms   elf   数据读取   tst   遇到的问题   遇到   

一、摘要

  突发奇想想读取12306的车票信息,最开始想用requests,但是突然又想试试selenium的无界面浏览器。有部分正则没调好,写好就懒得调了。

  套用我师傅的话就是:我凭本事写的bug,凭什么要改!

二、方案思路

  url = https://kyfw.12306.cn/otn/leftTicket/init

  1、模拟用户是怎么查车票信息、然后通过selenium去操作浏览器。

  2、最后输出字典。

三、源码

技术分享图片技术分享图片
#-*- coding:utf-8 -*-
#__anthor__:"Klay Zhu"
#date: 2018/9/7
import re
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from time import sleep

class TicketQuery:
    """
    查询12306车票
    """
    def __init__(self,**parame):
        try:
            self.obj_driver = parame["browser"]
        except:
            #无界面浏览器
            options = Options()
            options.add_argument(-headless)  # 无头参数
            self.obj_driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=options)
        self.str_from_station = parame["fromStation"] #出发地
        self.str_to_station = parame["toStation"]     #目的地
        self.str_train_date = parame["trainDate"]     #出发日

    def waitElement(self,value,way=By.ID):
        """等待忍耐"""
        if WebDriverWait(self.obj_driver, 10).until(expected_conditions.visibility_of_element_located((way,value))):
            return True
    def __jsOperation(self,element_id,value):
        """通过js操作界面元素"""
        if self.waitElement(element_id):
            js_value = document.getElementById("{}").value="{}".format(element_id,value)
            self.obj_driver.execute_script(js_value)
    def __getStation(self,element_id,string):
        """对input选择框进行选择"""
        self.obj_driver.find_element_by_id(element_id).clear()
        self.obj_driver.find_element_by_id(element_id).click()
        self.obj_driver.find_element_by_id(element_id).send_keys(string)
        __a = self.obj_driver.find_elements_by_class_name("ralign")
        for i in __a:
            if i.text == string:
                i.click()
                break
    #正则没调好 懒得调了
    pattern = r"]*>]*>\s*
]*>
]*> ]*>]*>[^]*]*>
]*>]*>(?P[^]*)]*>(?P[^]*)
]*>]*>(?P[^]*)]*>(?P[^]*)
]*>]*>(?P[^]*)]*>(?P[^]*)
]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>(?P([^]{1,4})|(
\d+
))
\s*]*?>.*?
" __url = r"https://kyfw.12306.cn/otn/leftTicket/init" def run(self): """执行操作""" sleep(2) # 不等这2秒会出现页面没刷新,数据获取不了的情况 self.obj_driver.get(self.__url) self.obj_driver.maximize_window() self.__getStation("fromStationText",self.str_from_station) self.__getStation("toStationText",self.str_to_station) self.__jsOperation("train_date",self.str_train_date) try: self.obj_driver.find_element_by_id("a_search_ticket").click() except: self.obj_driver.find_element_by_id("query_ticket").click() sleep(2) # 不等这2秒会出现页面没刷新,数据获取不了的情况 html = self.obj_driver.find_element_by_id("queryLeftTable").get_attribute("outerHTML") self.obj_driver.close() return html def outData(self,string = ""): if not string:string=self.run() obj_rr=re.finditer(self.pattern, string) list_data=[] if obj_rr: for obj_r in obj_rr: dict_ticket_info = { "train_num": "", # 车次 "from_station": "", # 出发站 "to_station": "", # 抵达站 "arrival_time": "", # 到站时间 "departure_time": "", # 离站时间 "over_time": "", # 历经时间 "intraday": True, # 是否当日到达 # 车票数据[0]特等座,[1]一等座,[2]二等座,[3]高级软卧,[4]软卧,[5]动卧,[6]硬卧,[7]软座, # [8]硬座,[9]无座,[10]其他,int数据代表剩余票数,Ture代表有票数量未知,False代表没票 "data": [], } dict_ticket_info["train_num"] = obj_r.group("train_num") dict_ticket_info["from_station"] = obj_r.group("from_station") dict_ticket_info["to_station"] = obj_r.group("to_station") dict_ticket_info["arrival_time"] = obj_r.group("arrival_time") dict_ticket_info["departure_time"] = obj_r.group("departure_time") dict_ticket_info["over_time"] = obj_r.group("over_time") if obj_r.group("intraday") == "当日到达":dict_ticket_info["intraday"] = True else:dict_ticket_info["intraday"] = False for i in range(1,11): str_info = obj_r.group("param" + str(i)) obj_r2=re.search(r"
(?P\d+)
",str_info) if obj_r2:dict_ticket_info["data"].append(obj_r2.group("num")) elif str_info=="":dict_ticket_info["data"].append(True) else:dict_ticket_info["data"].append(False) # print(dict_ticket_info) list_data.append(dict_ticket_info) return list_data if __name__ == __main__: import datetime m_time=(datetime.datetime.now()+datetime.timedelta(days=10)).strftime("%Y-%m-%d") testdata = { "fromStation":"北京", "toStation":"上海", "trainDate":m_time, #10天后 # "browser":webdriver.Chrome() #调试用 } data = TicketQuery(**testdata).run() print(data)
View Code

四、遇到的问题

1、日期输入,因为控件原因不能直接输入,使用JavaScript对元素进行操作。

from selenium import webdriver
driver = webdriver.Chrome()
element_id = "id"  #id
value = "value"    #给该标签设定的值
js_value = document.getElementById("{}").value="{}".format(element_id, value)#拼接成JavaScript代码 driver.execute_script(js_value)

 

2、input选择框,类似百度搜索这种使用ajax异步出来的数据。

from selenium import webdriver
element_id = "fromStationText"
string = "北京"
driver = webdriver.Chrome()

driver.find_element_by_id(element_id).clear()             # 清除原有的值
driver.find_element_by_id(element_id).click()             # 点击,使输入框获取焦点
driver.find_element_by_id(element_id).send_keys(string)   # 输入值
o_InputSelect=driver.find_elements_by_class_name("ralign")# 获取局部刷新的数据,然后循环比对文字
for i in o_InputSelect:                                   # 注意:如果不用这种方法,用输入回车来选择会出现  要选北京结果选到北京西  这类的
    if i.text == string:
        i.click()
        break

 

3、使用全局(嵌套)变量dict作为append到局部list里面,幸好出现问题就马上反应过来是深浅拷贝的问题,详情请百度!

class demo:
    dic={
        "name":"",
        "age":"",
        "data":[],
    }
    def f(self):
        lis = []
        for i in range(3):
            print("dic的地址:",id(f.dic))
            self.dic["name"]="name"+str(i)
            self.dic["age"]="age"+str(i)
            self.dic["data"].append(i)
            lis.append(self.dic)
        return lis
if __name__ == __main__:
    f = demo()
    print("最后结果:",f.f())
    # dic的地址: 1469349351640
    # dic的地址: 1469349351640
    # dic的地址: 1469349351640
    # 最后结果: [{‘name‘: ‘name2‘, ‘age‘: ‘age2‘, ‘data‘: [0, 1, 2]}, {‘name‘: ‘name2‘, ‘age‘: ‘age2‘, ‘data‘: [0, 1, 2]},
    #        {‘name‘: ‘name2‘, ‘age‘: ‘age2‘, ‘data‘: [0, 1, 2]}]

 

使用python+selenium对12306车票数据读取

标签:size   cte   div   oms   elf   数据读取   tst   遇到的问题   遇到   

原文地址:https://www.cnblogs.com/Klay/p/9643946.html


评论


亲,登录后才可以留言!