利用Scrapy爬取豆瓣排名前250部电影封面

一、爬虫代码

项目目录结构：

item.py文件

# -*- coding: utf-8 -*-
import scrapy

class DoubanmovieItem(scrapy.Item):
    # two items: url and name of image
    url = scrapy.Field()
    img_name = scrapy.Field()

pineline.py文件

# -*- coding: utf-8 -*-

from scrapy.pipelines.images import ImagesPipeline
from scrapy.http import Request 

class DoubanmoviePipeline(object):
    def process_item(self, item, spider):
        return item

class MyImagesPipeline(ImagesPipeline):
    # yield meta for file_path() function
    def get_media_requests(self, item, info): 
        for url in item['url']: 
            yield Request(url, meta={
   'item': item, 'index':item['url'].index(url)})

    # rename the image
    def file_path(self, request, response=None, info=None):
        item = request.meta['item']
        index = request.meta['index']

        image_name = item['img_name'][index]
        return 'full/%s.jpg' % (image_name)

setting.py文件

# -*- coding: utf-8 -*-
BOT_NAME = 'doubanMovie'
SPIDER_MODULES = ['doubanMovie.spiders']
NEWSPIDER_MODULE = 'doubanMovie.spiders'
ITEM_PIPELINES = {
   'doubanMovie.pipelines.DoubanmoviePipeline': 2,
                  'doubanMovie.pipelines.MyImagesPipeline':1 }
IMAGES_URLS_FIELD = 'url'
IMAGES_STORE = r'.'

middlewares.py文件

# -*- coding: utf-8 -*-
from scrapy import signals
class DoubanmovieSpiderMiddleware(object):

    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        pass

    def process_start_requests(self, start_requests, spider):
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DoubanmovieDownloaderMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        return None

    def process_response(self, request, response, spider):
        return response

    def process_exception(self, request, exception, spider):
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

doubanMovieSpider.py文件

import scrapy
from scrapy.spiders import Spider  
from scrapy.selector import Selector
from ..items import  DoubanmovieItem

class movieSpider(Spider):
    # name of Spider  
    name = "movie"
    #start urls
    start_urls = ["https://movie.douban.com/top250"] 
    for i in range(1,10):
        start_urls.append("https://movie.douban.com/top250?start=%d&filter="%(25*i))

    #parse function
    def parse(self, response):
        
        item = DoubanmovieItem()
        sel = Selector(response)
        images = sel.xpath('//*[@id="content"]/div/div[1]/ol/li')

        item['url'] = [] 
        item['img_name'] = []
        # append the url and name of the image in item
        for image in images:
            # extract url and name of the image   
            site = image.xpath('div/div[1]/a/img/@src').extract_first()
            img_name = image.xpath('div/div[1]/a/img/@alt').extract_first()
            
            item['url'].append(site)
            item['img_name'].append(img_name)
        yield item

二、爬取内容