{"id":6770,"date":"2022-07-21T15:45:33","date_gmt":"2022-07-21T06:45:33","guid":{"rendered":"https:\/\/blog.jansnap.com\/?p=6770"},"modified":"2022-07-21T15:45:34","modified_gmt":"2022-07-21T06:45:34","slug":"scrapy-2-6-1-%e3%81%a7%e3%82%a8%e3%83%a9%e3%83%bc%e3%83%8f%e3%83%b3%e3%83%89%e3%83%aa%e3%83%b3%e3%82%b0%e3%81%99%e3%82%8b","status":"publish","type":"post","link":"https:\/\/blog.jansnap.com\/?p=6770","title":{"rendered":"scrapy 2.6.1 \u3067\u30a8\u30e9\u30fc\u30cf\u30f3\u30c9\u30ea\u30f3\u30b0\u3059\u308b"},"content":{"rendered":"\n<p>Python\u3067\u4f7f\u3048\u308b\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u306e\u30e9\u30a4\u30d6\u30e9\u30ea\u306e\u300cscrapy\u300d\u3067\u3001spider\u5185\u3067\u30a8\u30e9\u30fc\u304c\u8d77\u304d\u305f\u6642\u306e\u30cf\u30f3\u30c9\u30ea\u30f3\u30b0\u65b9\u6cd5\u3092\u7d39\u4ecb\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%89%8B%E9%A0%861_middlewarespy_%E3%81%AE%E4%BF%AE%E6%AD%A3\"><\/span>\u624b\u98061: middlewares.py \u306e\u4fee\u6b63<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u4f5c\u6210\u6642\u306b\u3001\u3042\u3089\u304b\u3058\u3081<code>middlewares.py<\/code> \u304c\u81ea\u52d5\u4f5c\u6210\u3055\u308c\u308b\u306e\u3067\u3001\u3053\u308c\u3092\u4fee\u6b63\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<p><code>process_spider_exception<\/code>\u95a2\u6570\u3092\u4fee\u6b63\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code lang=\"python\" class=\"language-python\">    def process_spider_exception(self, response, exception, spider):\n        spider.logger.error(\"########\")\n        spider.logger.error(\"### PROCESS_SPIDER_EXCEPTION ###\")\n        spider.logger.error(exception)\n        spider.logger.error(\"########\")\n\n        return [{\n            \"url\": response.url,\n            \"status\": response.status,\n            \"error\": str(exception),\n        }]\n<\/code><\/pre>\n\n\n\n<p><\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E6%89%8B%E9%A0%862_settingspy_%E3%81%AE%E4%BF%AE%E6%AD%A3\"><\/span>\u624b\u98062: settings.py \u306e\u4fee\u6b63<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p><code>settings.py<\/code> \u3067\u3001<code>SPIDER_MIDDLEWARES<\/code> \u304c\u30b3\u30e1\u30f3\u30c8\u30a2\u30a6\u30c8\u306b\u306a\u3063\u3066\u3044\u308b\u306e\u3067\u3001\u30b3\u30e1\u30f3\u30c8\u3092\u5916\u3057\u3066\u6709\u52b9\u306b\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code lang=\"python\" class=\"language-python\"># Enable or disable spider middlewares\n# See https:\/\/docs.scrapy.org\/en\/latest\/topics\/spider-middleware.html\nSPIDER_MIDDLEWARES = {\n    'myproject_scrapy.middlewares.MyprojectScrapySpiderMiddleware': 543,\n}\n<\/code><\/pre>\n\n\n\n<p><\/p>\n\n\n\n<p>\u4ee5\u4e0a\u3067\u3001spider\u3067\u30a8\u30e9\u30fc\u304c\u51fa\u305f\u6642\u306b\u3001\u30ed\u30b0\u306b\u60c5\u5831\u3092\u51fa\u529b\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%E5%8F%82%E8%80%83URL\"><\/span>\u53c2\u8003URL<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ul class=\"wp-block-list\"><li><a href=\"https:\/\/blog.mudatobunka.org\/entry\/2016\/09\/24\/232456\">Scrapy \u3067\u30a8\u30e9\u30fc\u30cf\u30f3\u30c9\u30ea\u30f3\u30b0 for v1.1.3 (\u203b\u4e00\u90e8\u672a\u89e3\u6c7a)<\/a><\/li><\/ul>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Python\u3067\u4f7f\u3048\u308b\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u306e\u30e9\u30a4\u30d6\u30e9\u30ea\u306e\u300cscrapy\u300d\u3067\u3001spider\u5185\u3067\u30a8\u30e9\u30fc\u304c\u8d77\u304d\u305f\u6642\u306e\u30cf\u30f3\u30c9\u30ea\u30f3\u30b0\u65b9\u6cd5\u3092\u7d39\u4ecb\u3057\u307e\u3059\u3002 \u624b\u98061: middlewares.py \u306e\u4fee\u6b63 \u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u4f5c\u6210\u6642\u306b\u3001\u3042\u3089\u304b\u3058\u3081mid [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":6775,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[4070,4071],"tags":[4065,4066,4326],"class_list":["post-6770","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-python","category-python3","tag-python","tag-python3","tag-4326"],"_links":{"self":[{"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/posts\/6770","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=6770"}],"version-history":[{"count":5,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/posts\/6770\/revisions"}],"predecessor-version":[{"id":6776,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/posts\/6770\/revisions\/6776"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=\/wp\/v2\/media\/6775"}],"wp:attachment":[{"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=6770"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=6770"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.jansnap.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=6770"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}