first commit

Author: cwjokaka

.gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+.idea
+idea/

main.py

@@ -0,0 +1,4 @@
+from src.spider.spider_66_ip import Spider66Ip
+
+if __name__ == '__main__':
+    Spider66Ip().crawl()

setting.py

@@ -0,0 +1,5 @@
+from src.spider import Spider66Ip
+
+DB_TYPE = 'memory'      # memory/redis
+
+SPIDER_LIST = [Spider66Ip]

src/__init__.py

@@ -0,0 +1,10 @@
+class AbsDatabase(object):
+
+    def put(self, key, value):
+        raise RuntimeError('该put方法未实现!')
+
+    def get(self, key):
+        raise RuntimeError('该get方法未实现!')
+
+    def remove(self, key):
+        raise RuntimeError('该remove方法未实现!')

src/database/__init__.py

@@ -0,0 +1,18 @@
+from src.database.abs_database import AbsDatabase
+
+
+class Memory(AbsDatabase):
+    """
+    数据库:基于内存实现
+    """
+    def __init__(self) -> None:
+        self._box = {}
+
+    def put(self, key, value):
+        self._box[key] = value
+
+    def get(self, key):
+        return self._box[key]
+
+    def remove(self, key):
+        return self._box.pop(key, None)

src/database/abs_database.py

@@ -0,0 +1,38 @@
+class ProxyEntity(object):
+
+    def __init__(self, ip, port, source='', type='', check_count=0, region='', last_check_time=None):
+        self._ip = ip
+        self._port = port
+        self._source = source
+        self._type = type
+        self._check_count = check_count
+        self._region = region
+        self._last_check_time = last_check_time
+
+    @property
+    def ip(self):
+        return self._ip
+
+    @property
+    def port(self):
+        return self._port
+
+    @property
+    def source(self):
+        return self._source
+
+    @property
+    def type(self):
+        return self._type
+
+    @property
+    def check_count(self):
+        return self._check_count
+
+    @property
+    def region(self):
+        return self._region
+
+    @property
+    def last_check_time(self):
+        return self._last_check_time