添加清理不可靠的代理任务

Author: cwjokaka

setting.py

@@ -27,6 +27,11 @@
     'interval': 180                                 # 校验间隔(秒)
 }
 
+# 清除不可用代理配置
+EXPIRATION_VALIDATOR = {
+    'interval': 60 * 30
+}
+
 # 数据库配置
 DB = {
     'db_name': 'proxy.db',

src/database/abs_database.py

@@ -21,5 +21,8 @@ def update_anonymity(self, url, value):
     def remove(self, key):
         raise NotImplementedError
 
+    def remove_all_zero_reliability(self):
+        raise NotImplementedError
+
     def init_db(self):
         return

src/database/sqlite_opt.py

@@ -128,7 +128,6 @@ def init_db(self):
             """)
         except sqlite3.OperationalError as e:
             logger.warn(e)
-            # logger.exception(e)
         finally:
             cursor.close()
             conn.close()
@@ -163,6 +162,21 @@ def get_all_in_page(self):
             session.close()
         return None
 
+    def remove_all_zero_reliability(self):
+        conn = self._get_connect()
+        cursor = conn.cursor()
+        try:
+            cursor.execute(f"""
+            DELETE FROM {DB["table_name"]}
+            WHERE reliability = 0
+            """)
+            conn.commit()
+        except sqlite3.OperationalError as e:
+            logger.warn(e)
+        finally:
+            cursor.close()
+            conn.close()
+
     @staticmethod
     def _get_connect():
         return sqlite3.connect(DB['db_name'])

src/runner.py

@@ -7,7 +7,8 @@
 from src.entity.proxy_entity import ProxyEntity
 from src.log.logger import logger
 from src.spider.spiders import spider_collection
-from setting import WEB_SERVER, VALIDATOR, SPIDER, ANONYMITY_VALIDATOR
+from setting import WEB_SERVER, VALIDATOR, SPIDER, ANONYMITY_VALIDATOR, EXPIRATION_VALIDATOR
+from src.validator.expiration_validator import expiration_validator
 from src.validator.validator import validator
 from src.validator.anonymity_validator import anonymity_validator
 from src.web.web_flask import app
@@ -41,5 +42,6 @@ def run():
     scheduler.add_job(crawl, 'interval', seconds=SPIDER['crawl_interval'])
     scheduler.add_job(validator.run, 'interval', seconds=VALIDATOR['validate_interval'])
     scheduler.add_job(anonymity_validator.run, 'interval', seconds=ANONYMITY_VALIDATOR['interval'])
+    scheduler.add_job(expiration_validator.run, 'interval', seconds=EXPIRATION_VALIDATOR['interval'])
     scheduler.start()
     app.run(host=WEB_SERVER['host'], port=WEB_SERVER['port'])

src/spider/abs_spider.py

@@ -12,13 +12,12 @@ class AbsSpider(object):
 
     def __init__(self, name='unknown') -> None:
         self._name = name
-        self.urls = self.get_urls()
+        self._urls = self.get_urls()
 
     async def crawl(self):
         logger.info(f'{self._name}开始爬取...')
-        urls = self.get_urls()
         res = []
-        for url in urls:
+        for url in self._urls:
             try:
                 for page in self.get_page_range():
                     async with aiohttp.ClientSession() as session:

src/validator/expiration_validator.py

@@ -0,0 +1,13 @@
+from src.database.sqlite_opt import sqlite_opt
+from src.log.logger import logger
+
+
+class ExpirationValidator(object):
+
+    def run(self):
+        logger.info('开始删除不可用代理')
+        sqlite_opt.remove_all_zero_reliability()
+        logger.info('不可用代理删除完毕')
+
+
+expiration_validator = ExpirationValidator()

test/database/test_sqlite_opt.py

@@ -21,3 +21,6 @@ def test_add_proxy(self):
     def test_get_all_proxies(self):
         proxy_list = self._opt.get_all_proxies()
         assert len(proxy_list) > 0
+
+    def test_remove_all_zero_reliability(self):
+        self._opt.remove_all_zero_reliability()