05.编写spider

约 9 分钟 · 4719 字 · 178 views
 ›
05.编写spider

解析所有文章及提取下一页

# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.http import Request
from urllib import parse


# 提取字符串中的数字
def get_nums(str):
    match_re = re.match(".*(\d+).*", str)
    if match_re:
        nums = int(match_re.group(1))
    else:
        nums = 0
    return nums

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']



    def parse(self, response):
        """
        1.获取文章列表中的文章url并交给scrapy下载解析
        2.获取下一页的url并交给scrapy进行下载，下载后再交给parse
        """
        # 解析文章所有url并交给scrapy下载后解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            print(post_url)  # 文章url
            # parse.urljoin(response.url, post_url)表示有绝对域名就直接使用，相对域名就拼接主域名
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url": image_url}, callback=self.parse_detail)

        # 提取下一页并交给scrapy进行下载
        next_urls = response.css(".next.page-numbers::attr(href)").extract_first()
        if next_urls:
            yield Request(url=parse.urljoin(response.url, next_urls), callback=self.parse)

    # 提取文章的具体字段
    def parse_detail(self, response):
        front_image_url = response.meta.get("front_image_url", "")  # 获取response中的front_image_url 图片
        title = response.css('.entry-header h1::text').extract_first
        create_date = response.css('.entry-meta-hide-on-mobile::text').extract()[0].strip().replace("·", "")
        praise_nums = response.css('.vote-post-up h10::text').extract()[0]
        fav_str = response.css('.bookmark-btn::text').extract()[0]
        fav_nums = get_nums(fav_str)
        comment_str = response.css('a[href="#article-comment"] span::text').extract()[0]
        comment_nums = get_nums(comment_str)
        # content = response.css('div.entry').extract()[0]
        tags = response.css('p.entry-meta-hide-on-mobile a::text').extract()[0]
        pass

如果yield Request 不执行，需要加dont_filter=True

# parse.urljoin(response.url, post_url)表示有绝对域名就直接使用，相对域名就拼接主域名
yield Request(url=parse.urljoin(response.url, post_url), callback=self.parse_detail, dont_filter=True)

items设计

  1. 在setting中将以下代码的注释去掉
    ITEM_PIPELINES = {
'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
}
  2. 在item.py 中写下item项
# -*- coding: utf-8 -*-

import scrapy


class ArticlespiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


class JobBoleArticleItem(scrapy.Item):
    # 标题
    title = scrapy.Field()
    # 日期
    create_date = scrapy.Field()
    # url
    url = scrapy.Field()
    url_object_id = scrapy.Field()  # url是变长 可以md5定常加密保存
    # 标题图
    front_image_url = scrapy.Field()
    # 标题图地址
    front_image_path = scrapy.Field()
    # 点赞数
    praise_nums = scrapy.Field()
    # 喜爱数
    fav_nums = scrapy.Field()
    # 评论数
    comment_nums = scrapy.Field()
    # 文章内容
    content = scrapy.Field()
    # 标签
    tags = scrapy.Field()
  1. 在jobbloe.py解析函数中写入
from ArticleSpider.items import JobBoleArticleItem, ArticleItemLoader

        article_item["title"] = title
        article_item["create_date"] = create_date
        article_item["front_image_url"] = front_image_url
        article_item["praise_nums"] = praise_nums
        article_item["fav_nums"] = fav_nums
        article_item["comment_nums"] = comment_nums
        article_item["content"] = content
        article_item["tags"] = tags
        article_item["url"] = response.url

        yield article_item
  1. 设置图片存放路径

首先在项目目录下，与spider目录同级的地方新建images文件夹。再次在setting中配置如下代码

import os

ITEM_PIPELINES = {
   'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
    'scrapy.pipelines.images.ImagesPipeline': 1,
}
IMAGES_URLS_FIELD = "front_image_url"
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir, 'images')

如果报缺少"PIL"库，该库与图片相关请安装 pillow。 如果报valueError：Missing scheme in request url:h错,原因是将图片下载的item会被当作数组处理，所以修改jobbloe.py下的

article_item["front_image_url"] = front_image_url

        # 修改为
        article_item["front_image_url"] = [front_image_url]
  1. 重载imagespipeline

首先在pipelines.py中写入

# -*- coding: utf-8 -*-

from scrapy.pipelines.images import ImagesPipeline

class ArticlespiderPipeline(object):
    def process_item(self, item, spider):
        return item


class ArticleImagesPipeline(ImagesPipeline):
    # 重载文件获取实际下载地址
    def item_completed(self, result, item, info):
        for ok, value in result:
            image_file_path = value["path"]
        item["front_image_path"] = image_file_path
        return item

其次在setting.py中设置

ITEM_PIPELINES = {
   'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
   'ArticleSpider.pipelines.ArticleImagesPipeline': 1,
    # 'scrapy.pipelines.images.ImagesPipeline': 1,
}
# 将front_image_url字段图片保存到本地的配置
IMAGES_URLS_FIELD = "front_image_url"
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir, 'images')

新建utils文件夹新建common.py文件

# _*_ coding: utf-8 _*_

import hashlib


def get_md5(url):
    if isinstance(url, str):
        url= url.encode("utf-8")
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()


if __name__ == "__main__":
    print(get_md5("https://www.baidu.com"))

在jobbloe.py中添加

from ArticleSpider.utils.common import get_md5

article_item["front_image_path"] = get_md5(response.url)

📌 相关阅读