Python基于分析Ajax请求实现抓取今日头条街拍图集功能示例

 import os import re import json import time from hashlib import md5 from multiprocessing import Pool import requests from requests.exceptions import RequestException from pymongo i<a style="color:transparent">来源gao($daima.com搞@代@#码网</a>mport MongoClient # 配置信息 OFFSET_START = 0  # 爬去页面的起始下标 OFFSET_END = 20  # 爬去页面的结束下标 KEYWORD = '街拍'  # 搜索的关键字 # mongodb相关配置 MONGO_URL = 'localhost' MONGO_DB = 'toutiao'  # 数据库名称 MONGO_TABLE = 'jiepai' # 集合名称 # 图片保存的文件夹名称 IMAGE_PATH = 'images' headers = { "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' } client = MongoClient(host=MONGO_URL) db = client[MONGO_DB] jiepai_table = db[MONGO_TABLE] if not os.path.exists(IMAGE_PATH): os.mkdir(IMAGE_PATH) def get_html(url, params=None): try: response = requests.get(url, params=params, headers=headers) if response.status_code == 200: return response.text return None except RequestException as e: print("请求%s失败: " % url, e) return None # 获取索引页内容 def get_index_page(offset, keyword): basic_url = 'http://www.toutiao.com/search_content/' params = { 'offset': offset, 'format': 'json', 'keyword': keyword, 'autoload': 'true', 'count': 20, 'cur_tab': 3 } return get_html(basic_url, params) def parse_index_page(html): ''' 解析索引页内容 返回: 索引页中包含的所有详情页url ''' if not html: return data = json.loads(html) if 'data' in data: for item in data['data']: article_url = item['article_url'] if 'toutiao.com/group' in article_url: yield article_url # 获取详情页 def get_detail_page(url): return get_html(url) # 解析详情页 def parse_detail_page(url, html): ''' 解析详情页 返回对应的标题，url和包含的图片url ''' title_reg = re.compile('<title>(.*?)</title>') title = title_reg.search(html).group(1) gallery_reg = re.compile('var gallery = (.*?);') gallery = gallery_reg.search(html) if gallery and 'sub_images' in gallery.group(1): images = json.loads(gallery.group(1))['sub_images'] image_list = [image['url'] for image in images] return { 'title': title, 'url': url, 'images': image_list } return None def save_to_mongodb(content): jiepai_table.insert(content) print("存储到mongdob成功", content) def download_images(image_list): for image_url in image_list: try: response = requests.get(image_url) if response.status_code == 200: save_image(response.content) except RequestException as e: print("下载图片失败: ", e) def save_image(content): ''' 对图片的二进制内容做hash，构造图片路径,以此保证图片不重复 ''' file_path = '{0}/{1}/{2}.{3}'.format(os.getcwd(), IMAGE_PATH, md5(content).hexdigest(), 'jpg') # 去除重复的图片 if not os.path.exists(file_path): with open(file_path, 'wb') as f: f.write(content) def jiepai(offset): html = get_index_page(offset, KEYWORD) if html is None: return page_urls = list(parse_index_page(html)) # print("详情页url列表:" ) # for page_url in page_urls: #   print(page_url) for page in page_urls: print('get detail page:', page) html = get_detail_page(page) if html is None: continue content = parse_detail_page(page, html) if content: save_to_mongodb(content) download_images(content['images']) time.sleep(1) print('-------------------------------------') if __name__ == '__main__': offset_list = range(OFFSET_START, OFFSET_END) pool = Pool() pool.map(jiepai, offset_list) 

 import requests basic_url = 'http://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=3' url = basic_url.format(0) html = requests.get(url).json() items = html['data'] for item in items: title = item['media_name'] image_list = [image_detail['url'] for image_detail in item['image_detail']] print(title, image_list)