提交代码

cxhuan · cxhuan · commit 160a7852b6db · 2021-09-14T17:04:05.000+08:00
diff --git a/xianhuan/README.md b/xianhuan/README.md
@@ -10,6 +10,8 @@ Python技术 公众号文章代码库
 
 ## 实例代码
 
+[神器！五分钟完成大型爬虫项目！](https://github.com/JustDoPython/python-examples/tree/master/xianhuan/airspider)：神器！五分钟完成大型爬虫项目！
+
 [卧槽！几行代码，干掉一个网站！](https://github.com/JustDoPython/python-examples/tree/master/xianhuan/gengif)：卧槽！几行代码，干掉一个网站！
 
 [牛逼！用Python为她设计专属签名软件](https://github.com/JustDoPython/python-examples/tree/master/xianhuan/artname)：牛逼！用Python为她设计专属签名软件
diff --git a/xianhuan/airspider/report_spider.py b/xianhuan/airspider/report_spider.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2021-09-14 15:07:49
+---------
+@summary:
+---------
+@author: 闲欢
+"""
+
+import feapder
+import json
+
+from feapder.db.mysqldb import MysqlDB
+
+
+class ReportSpider(feapder.AirSpider):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.db = MysqlDB()
+
+    def start_requests(self):
+        yield feapder.Request("http://reportapi.eastmoney.com/report/list?cb=datatable1351846&industryCode=*&pageSize=50&industry=*&rating=&ratingChange=&beginTime=2021-09-13&endTime=2021-09-14&pageNo=1&fields=&qType=0&orgCode=&code=*&rcode=&p=2&pageNum=2&_=1603724062679",
+                              callback=self.parse_report_info, pageNo=1)
+
+    def parse_report_info(self, request, response):
+        print(request.pageNo)
+        html = response.content.decode("utf-8")
+        if len(html):
+            content = html.replace('datatable1351846(', '')[:-1]
+            content_json = json.loads(content)
+            print(content_json)
+            self.save_data(content_json)
+
+    def save_data(self, items):
+        result_list = []
+        for i in items['data']:
+            result = {}
+            obj = i
+            result['title'] = obj['title'] #报告名称
+            result['stockName'] = obj['stockName'] #股票名称
+            result['stockCode'] = obj['stockCode'] #股票code
+            result['orgCode'] = obj['stockCode'] #机构code
+            result['orgName'] = obj['orgName'] #机构名称
+            result['orgSName'] = obj['orgSName'] #机构简称
+            result['publishDate'] = obj['publishDate'] #发布日期
+            result['predictNextTwoYearEps'] = obj['predictNextTwoYearEps'] #后年每股盈利
+            result['predictNextTwoYearPe'] = obj['predictNextTwoYearPe'] #后年市盈率
+            result['predictNextYearEps'] = obj['predictNextYearEps'] # 明年每股盈利
+            result['predictNextYearPe'] = obj['predictNextYearPe'] # 明年市盈率
+            result['predictThisYearEps'] = obj['predictThisYearEps'] #今年每股盈利
+            result['predictThisYearPe'] = obj['predictThisYearPe'] #今年市盈率
+            result['indvInduCode'] = obj['indvInduCode'] # 行业代码
+            result['indvInduName'] = obj['indvInduName'] # 行业名称
+            result['lastEmRatingName'] = obj['lastEmRatingName'] # 上次评级名称
+            result['lastEmRatingValue'] = obj['lastEmRatingValue'] # 上次评级代码
+            result['emRatingValue'] = obj['emRatingValue'] # 评级代码
+            result['emRatingName'] = obj['emRatingName'] # 评级名称
+            result['ratingChange'] = obj['ratingChange'] # 评级变动
+            result['researcher'] = obj['researcher'] # 研究员
+            result['encodeUrl'] = obj['encodeUrl'] # 链接
+            result['count'] = int(obj['count']) # 近一月个股研报数
+
+            result_list.append(result)
+
+            self.insertdb(result_list)
+
+        return result_list
+
+    def download_midware(self, request):
+        request.headers = {
+            "Connection": "keep-alive",
+            "Cookie": "qgqp_b_id=0f1ac887e1e3e484715bf0e3f148dbd8; intellpositionL=1182.07px; st_si=32385320684787; st_asi=delete; cowCookie=true; intellpositionT=741px; st_pvi=73966577539485; st_sp=2021-03-22%2009%3A25%3A40; st_inirUrl=https%3A%2F%2Fwww.baidu.com%2Flink; st_sn=4; st_psi=20210914160650551-113300303753-3491653988",
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
+            "Host": "reportapi.eastmoney.com"
+        }
+        return request
+
+    def validate(self, request, response):
+        if response.status_code != 200:
+            raise Exception("response code not 200") # 重试
+
+
+    def insertdb(self, data_list):
+        attrs = ['title', 'stockName', 'stockCode', 'orgCode', 'orgName', 'orgSName', 'publishDate', 'predictNextTwoYearEps',
+                 'predictNextTwoYearPe', 'predictNextYearEps', 'predictNextYearPe', 'predictThisYearEps', 'predictThisYearPe',
+                 'indvInduCode', 'indvInduName', 'lastEmRatingName', 'lastEmRatingValue', 'emRatingValue',
+                 'emRatingName', 'ratingChange', 'researcher', 'encodeUrl', 'count']
+        insert_tuple = []
+        for obj in data_list:
+            insert_tuple.append((obj['title'], obj['stockName'], obj['stockCode'], obj['orgCode'], obj['orgName'], obj['orgSName'], obj['publishDate'], obj['predictNextTwoYearEps'], obj['predictNextTwoYearPe'], obj['predictNextYearEps'], obj['predictNextYearPe'], obj['predictThisYearEps'], obj['predictThisYearPe'], obj['indvInduCode'], obj['indvInduName'], obj['lastEmRatingName'], obj['lastEmRatingValue'], obj['emRatingValue'],obj['emRatingName'], obj['ratingChange'], obj['researcher'], obj['encodeUrl'], obj['count']))
+        values_sql = ['%s' for v in attrs]
+        attrs_sql = '('+','.join(attrs)+')'
+        values_sql = ' values('+','.join(values_sql)+')'
+        sql = 'insert into %s' % 'report'
+        sql = sql + attrs_sql + values_sql
+
+        self.db.add_batch(sql, insert_tuple)
+
+
+
+if __name__ == "__main__":
+    ReportSpider().start()
diff --git a/xianhuan/airspider/setting.py b/xianhuan/airspider/setting.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+@author: 闲欢
+"""
+import os
+
+
+# MYSQL
+MYSQL_IP = "localhost"
+MYSQL_PORT = 3306
+MYSQL_DB = "xxx"
+MYSQL_USER_NAME = "root"
+MYSQL_USER_PASS = "xxx"
+
+# REDIS
+# IP:PORT
+REDISDB_IP_PORTS = "localhost:6379"
+REDISDB_USER_PASS = ""
+# 默认 0 到 15 共16个数据库
+REDISDB_DB = 0
+
+
+# 爬虫相关
+# COLLECTOR
+COLLECTOR_SLEEP_TIME = 1  # 从任务队列中获取任务到内存队列的间隔
+COLLECTOR_TASK_COUNT = 100  # 每次获取任务数量
+
+# SPIDER
+SPIDER_THREAD_COUNT = 10  # 爬虫并发数
+SPIDER_SLEEP_TIME = [1, 5]  # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5
+SPIDER_MAX_RETRY_TIMES = 50  # 每个请求最大重试次数
+
+# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败
+RETRY_FAILED_REQUESTS = False
+# request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒
+REQUEST_LOST_TIMEOUT = 600  # 10分钟
+# 保存失败的request
+SAVE_FAILED_REQUEST = True
+
+# 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用
+RESPONSE_CACHED_ENABLE = False  # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True
+RESPONSE_CACHED_EXPIRE_TIME = 3600  # 缓存时间 秒
+RESPONSE_CACHED_USED = False  # 是否使用缓存 补采数据时可设置为True
+
+WARNING_FAILED_COUNT = 1000  # 任务失败数 超过WARNING_FAILED_COUNT则报警
+
+# 爬虫是否常驻
+KEEP_ALIVE = False
+
+# 随机headers
+RANDOM_HEADERS = True
+# requests 使用session
+USE_SESSION = False
+
+# 去重
+ITEM_FILTER_ENABLE = False  # item 去重
+REQUEST_FILTER_ENABLE = False  # request 去重
+
+LOG_NAME = os.path.basename(os.getcwd())
+LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径
+LOG_LEVEL = "DEBUG"
+LOG_COLOR = True  # 是否带有颜色
+LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台
+LOG_IS_WRITE_TO_FILE = False  # 是否写文件
+LOG_MODE = "w"  # 写文件的模式
+LOG_MAX_BYTES = 10 * 1024 * 1024  # 每个日志文件的最大字节数
+LOG_BACKUP_COUNT = 20  # 日志文件保留数量
+LOG_ENCODING = "utf8"  # 日志文件编码
+OTHERS_LOG_LEVAL = "ERROR"  # 第三方库的log等级
