python爬虫实现获取下一页代码

 from time import sleep import faker import requests from lxml import etree fake = faker.Faker() base_url = "http://angelimg.spbeen.com" def get_next_link(url): content = downloadHtml(url) html = etree.HTML(content) next_url = html.xpath("//a[@class='ch next']/@href") if next_url: return base_url + next_url[0] else: return False def downloadHtml(ur): user_agent = fake.user_agent() headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"} response = requests.get(url, headers=headers) return response.text def getImgUrl(content): html = etree.HTML(content) img_url = html.xpath('//*[@id="content"]/a/img/@src') title = html.xpath(".//div['@class=article']/h2/text()") return img_url[0],title[0] def saveImg(title,img_url): if img_url is not None and title is not None: with open("txt/"+str(title)+".jpg-600",'wb') as f: user_agent = fake.user_agent() headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"} content = requests.get(img_url, headers=headers) #request_view(content) f.write(content.content) f.close() def request_view(response): import webbrowser request_url = response.url base_url = '' %(request_url) base_url = base_url.encode() content = response.content.replace(b"",base_url) tem_html = open('tmp.html','wb') tem_html.write(content) tem_html.close() webbrowser.open_new_tab('tmp.html') def crawl_img(url): content = downloadHtml(url) res<div style="color:transparent">来源gaodai.ma#com搞##代!^码网</div> = getImgUrl(content) title = res[1] img_url = res[0] saveImg(title,img_url) if __name__ == "__main__": url = "http://angelimg.spbeen.com/ang/4968/1" while url: print(url) crawl_img(url) url = get_next_link(url)

 from bs4 import BeautifulSoup import requests import time from lxml import etree import os # 该demo执行的为如何利用bs去爬一些文字 def start(): # 发起网络请求 html=requests.get('http://www.baidu.com') #编码 html.encoding=html.apparent_encoding #创建sp soup=BeautifulSoup(html.text,'html.parser') print(type(soup)) print('打印元素') print(soup.prettify()) #存储一下title 该方法没有提示直接展示 title=soup.head.title.string print(title) #   写入文本 with open(r'C:/Users/a/Desktop/a.txt','w') as f: f.write(title) print(time.localtime()) url_2 = 'http://news.gdzjdaily.com.cn/zjxw/politics/sz_4.shtml' def get_html_from_bs4(url): # response = requests.get(url,headers=data,proxies=ip).content.decode('utf-8') response = requests.get(url).content.decode('utf-8') soup = BeautifulSoup(response, 'html.parser') next_page = soup.select('#displaypagenum a:nth-of-type(9)')[0].get('href') # for i in nett print(next_page) next2='http://news.gdzjdaily.com.cn/zjxw/politics/'+next_page def get_html_from_etree(url): response = requests.get(url).content.decode('utf-8') html= etree.HTML(response) next_page = html.xpath('.//a[@class="PageNum"][8]/@href')[0] print(next_page) # next2='http://news.gdzjdaily.com.cn/zjxw/politics/'+next_page get_html_from_etree(url_2) if __name__ == '__main__': start()