教你快速上手Selenium爬虫,万物皆可爬

 from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.by import By from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as ec browser = webdriver.Chrome() try: browser.get('https://www.jd.com') input_t = browser.find_element_by_id('key') input_t.send_keys('python爬虫') input_t.send_keys(Keys.ENTER)  # 模拟按下Enter键位 wait = WebDriverWait(browser, 4)  # 设置最长等待时间4秒 wait.until(ec.presence_of_all_elements_located((By.ID, 'J_goodsList'))) print(browser.title)  # 显示搜索页面的标题 print(browser.current_url) print(browser.page_source) browser.close() except Exception as e: print(e) browser.close() 

   <title>表单</title>  姓名： 年龄： 国家： 收入：<button>提交</button>

 from selenium import webdriver from selenium.webdriver.common.by import By # 不支持本地网页 browser = webdriver.Chrome() try: # 这里我是用flask自己搭建的一个服务 访问该页面即可打开demo.html browser.get('http://127.0.0.1:5000/') input_t = browser.find_element_by_id('name')  # 通过id属性查找姓名input节点 input_t.send_keys('Amo')  # 自动输入 input_t = browser.find_element_by_id('age') input_t.send_keys('18') input_t = browser.find_element_by_name('country')  # 通过name属性查找国家input节点 input_t.send_keys('中国') # 通过class属性查找收入input节点 input_t = browser.find_element_by_class_name('myclass') input_t.send_keys('1850') # 或下面的代码 input_t = browser.find_element(By.CLASS_NAME, 'myclass') # 要想覆盖前面的输入，需要清空input节点，否则会在input节点原来的内容后面追加新内容 input_t.clear() input_t.send_keys('3500') except Exception as e: print(e) browser.close() 

 from selenium import webdriver from selenium.webdriver.common.by import By # 不支持本地网页 browser = webdriver.Chrome() try: browser.get('https://www.jd.com')  # 打开京东 # 根据节点名查找所有名为li的节点 li_list = browser.find_elements_by_tag_name('li') # 输出节点本身 print(li_list) print(len(li_list)) print(li_list[0].text) ul = browser.find_elements(By.TAG_NAME, 'ul') print(ul) print(ul[0].text) browser.close() except Exception as e: print(e) browser.close() 

   <title>彩色按钮</title> <button class="mybutton" style="background: red">按钮1</button><button class="mybutton" style="background: blue">按钮2</button><button class="mybutton" style="background: yellow">按钮3</button><br><button class="mybutton" style="background: green">按钮4</button><button class="mybutton" style="background: blueviolet">按钮5</button><button class="mybutton" style="background: gold">按钮6</button><div id="bgcolor" style="width: 200px;height: 200px"> </div>

 from selenium import webdriver import time browser = webdriver.Chrome() try: browser.get('http://127.0.0.1:5000/') buttons = browser.find_elements_by_class_name('mybutton') i = 0 while True: buttons[i].click() time.sleep(1) i += 1 if i == len(buttons): i = 0 except Exception as e: print(e) browser.close() 

 from selenium import webdriver from selenium.webdriver import ActionChains import time browser = webdriver.Chrome() try: browser.get('https://www.jd.com') actions = ActionChains(browser) li_list = browser.find_elements_by_css_selector(".cate_menu_item") for li in li_list: actions.move_to_element(li).perform() time.sleep(1) except Exception as e: print(e) browser.close() 

 from selenium import webdriver from selenium.webdriver import ActionChains browser = webdriver.Chrome() try: browser.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable') browser.switch_to.frame('iframeResult') source = browser.find_element_by_css_selector('#draggable') target = browser.find_element_by_css_selector('#droppable') actions = ActionChains(browser) actions.drag_and_drop(source, target) actions.perform() except Exception as e: print(e) browser.close() 

 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.jd.com') browser.execute_script('window.scrollTo(0,document.body.scrollHeight)') browser.execute_async_script('alert("已经到达页面底端")') 

 from selenium import webdriver options = webdriver.ChromeOptions() options.add_argument('headless') browser = webdriver.Chrome(chrome_options=options) # browser = webdriver.PhantomJS('./webdriver/phantomjs') browser.get('https://www.jd.com') ul = browser.find_element_by_id("navitems-group1") print(ul.text) print('id', '=', ul.id)  # 内部id，不是节点id属性值 print('location', '=', ul.location) print('tag_name', '=', ul.tag_name) print('size', '=', ul.size) li_list = ul.find_elements_by_tag_name("li") for li in li_list: print(type(li)) # 属性没找到，返回None print('', 'class=', li.get_attribute('class')) a = li.find_element_by_tag_name('a') print('href', '=', a.get_attribute('href')) browser.close() 

 from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.jd.com') print(browser.get_cookies()) browser.add_cookie({'name': 'name', 'value': 'jd', 'domain': 'www.jd.com'}) print(browser.get_cookies()) browser.delete_all_cookies() print(browser.get_cookies())  # 大部分删除了，可能还剩下一些 

 from selenium import webdriver import time driver = webdriver.Chrome() driver.get("http://www.baidu.com") search_button = driver.find_element_by_id("su")  # 百度搜索按钮 # arguments[0]对应的是第一个参数，可以理解为python里的%s传参，与之类似 x_positions = [50, 90, 130, 170] y_positions = [100, 120, 160, 90] for i in range(len(x_positions)): js = ''' arguments[0].style.position = "absolute"; arguments[0].style.left="{}px"; arguments[0].style.top="{}px"; '''.format(x_positions[i], y_positions[i]) driver.execute_script(js, search_button) time.sleep(2) 

 from selenium import webdriver driver = webdriver.Chrome() driver.get("https://www.jd.com") ul = driver.find_element_by_id('navitems-group1') li_list = ul.find_elements_by_tag_name('li') a1 = li_list[0].find_element_by_tag_name('a') a2 = li_list[1].find_element_by_tag_name('a') js = ''' arguments[0].text = '3天极速掌握 Scala 语言：First Day' arguments[0].href = 'https://blog.gaodaimna/xw1680/article/details/118743183' arguments[1].text = '数据仓库 Hive 从入门到小牛(一)' arguments[1].href = 'https://blog.gaodaimna/xw1680/article/details/118675528' ''' driver.execute_script(js, a1, a2)