Python搭建代理池，为你的爬虫程序保驾护航( 二 ) 由于爬虫工作往往有大量数据

class Scheduler(object):"""Scheduler"""def run_tester(self, cycle=CYCLE_TESTER):"""定时启动检测器:param cycle::return:"""if not ENABLE_TESTER:app_logger.info('tester not enabled, exit')returntester = Tester()sch.add_job(tester.run, 'interval', seconds=cycle, id='run_tester')sch.start()def run_getter(self, cycle=CYCLE_GETTER):"""定时开启爬虫补充代理:param cycle::return:"""if not ENABLE_GETTER:app_logger.info('getter not enabled, exit')returngetter = Getproxies()sch.add_job(getter.run, 'interval', seconds=cycle, id='run_getter')sch.start()def run_server(self):"""web服务端开启"""if not ENABLE_SERVER:app_logger.info('server not enabled, exit')returnapp.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)def run(self):"""调度器启动"""global tester_process, getter_process, server_processtry:app_logger.info('starting proxypool...')if ENABLE_TESTER:tester_process = multiprocessing.Process(target=self.run_tester)app_logger.info(f'starting tester, pid {tester_process.pid}...')tester_process.start()if ENABLE_GETTER:getter_process = multiprocessing.Process(target=self.run_getter)# print(dir(getter_process),getter_process)app_logger.info(f'starting getter, pid{getter_process.pid}...')getter_process.start()if ENABLE_SERVER:server_process = multiprocessing.Process(target=self.run_server)app_logger.info(f'starting server, pid{server_process.pid}...')server_process.start()tester_process.join()getter_process.join()server_process.join()except KeyboardInterrupt:app_logger.info('received keyboard interrupt signal')tester_process.terminate()getter_process.terminate()server_process.terminate()finally:# must call join method before calling is_alivetester_process.join()getter_process.join()server_process.join()app_logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}')app_logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}')app_logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}')app_logger.info('proxy terminated')