selenium模块

简单小例子

Author:songimport pyqueryfrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.common.by import Byfrom pyquery import PyQuery as pqfrom config import *import pymongoclient = pymongo.MongoClient(MONGO_URL)db = client[MONGO_DB]browser = webdriver.Chrome()wait =WebDriverWait(browser,10)def get_one_index():    try:        url = 'https://www.taobao.com/'        browser.get(url)        input = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#q')))        submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'#J_TSearchForm > div.search-button > button')))        input.send_keys('python')        submit.click()        get_informations()    except TimeoutException:        print('error')        return get_one_index()def get_more_indexs(page_number):    input = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input')))    submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit')))    input.clear()    input.send_keys(page_number)    submit.click()    get_informations()def get_informations():    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))    html = browser.page_source    doc = pq(html)    items = doc('#mainsrp-itemlist .items .item').items()    for item in items:        product = {            'title' : item.find('.row.row-2.title').text(),            'picture' : item.find('.J_ItemPic.img').attr('src'),            'price' : item.find('.price.g_price.g_price-highlight').text(),            'shop' : item.find('.shopname.J_MouseEneterLeave.J_ShopInfo').text(),            'location' : item.find('.location').text(),            'deal':item.find('.J_ClickStat').attr('href')        }        save_to_mongodb(product)def save_to_mongodb(result):        if db[MONGO_TABLE].insert(result):            print('存储成功',result)def main():    get_one_index()    for number in range(2,5):        get_more_indexs(number)if __name__ =='__main__':    main()