如何将503导致的刮削失败标记为刮削错误?

RETRY_HTTP_CODES

https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#retry-http-codes

HTTPERROR_ALLOWED_CODES

https://doc.scrapy.org/en/latest/topics/spider-middleware.html#std:setting-HTTPERROR_ALLOWED_CODES

settings.py

from scrapy.downloadermiddlewares.retry import *

class Retry500Middleware(RetryMiddleware):

    def _retry(self, request, reason, spider):
        retries = request.meta.get('retry_times', 0) + 1

        if retries <= self.max_retry_times:
            logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
                         {'request': request, 'retries': retries, 'reason': reason},
                         extra={'spider': spider})
            retryreq = request.copy()
            retryreq.meta['retry_times'] = retries
            retryreq.dont_filter = True
            retryreq.priority = request.priority + self.priority_adjust
            return retryreq
        else:
            # This is the point where I update it. It used to be `logger.debug` instead of `logger.error`
            logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
                         {'request': request, 'retries': retries, 'reason': reason},
                         extra={'spider': spider})