Merge pull request #65 from RyouMon/refactor-config

Save files to FAVORS_HOME by default

Author: RyouMon

README.md

@@ -155,8 +155,8 @@ yandere:
 ```
 
 ## Download location
-By default, pictures will download to working directory.  
-If you want to change download location, you can add FILES_STORE option to config.  
+By default, pictures will download to `${FAVORS_HOME}/{site_name}`  
+If you want to change download location, you can update FILES_STORE.  
 For example, if you want save pixiv files to `pictures/a`, and want save yandere files to `pictures/b`, you can modify config file like this:
 ```yaml
 pixiv:

src/favorites_crawler/commands/crawl.py

@@ -72,7 +72,7 @@ def crawl(name, **kwargs):
     """
     spider = spider_loader.load(name)
     favors_home = os.getenv('FAVORS_HOME', DEFAULT_FAVORS_HOME)
-    overwrite_spider_settings(spider, scrapy_settings, load_config(favors_home))
+    overwrite_spider_settings(spider, favors_home, load_config(favors_home))
     process = CrawlerProcess(scrapy_settings)
     process.crawl(spider, **kwargs)
     for crawler in process.crawlers:

src/favorites_crawler/constants/path.py

@@ -1 +1,3 @@
-DEFAULT_FAVORS_HOME = '~/.favorites_crawler'
+import os
+
+DEFAULT_FAVORS_HOME = os.path.join('~', '.favorites_crawler')

src/favorites_crawler/utils/config.py

@@ -1,7 +1,8 @@
 import os
-import yaml
 from copy import deepcopy
 
+import yaml
+
 
 DEFAULT_CONFIG = {
     'global': {
@@ -10,28 +11,28 @@
         'EXIF_TOOL_EXECUTABLE': None,
     },
     'pixiv': {
-        'FILES_STORE': 'favorites_crawler_files/pixiv',
+        'FILES_STORE': os.path.join('$FAVORS_HOME', 'pixiv'),
         'USER_ID': '',
         'ACCESS_TOKEN': '',
         'REFRESH_TOKEN': '',
     },
     'yandere': {
-        'FILES_STORE': 'favorites_crawler_files/yandere',
+        'FILES_STORE': os.path.join('$FAVORS_HOME', 'yandere'),
         'USERNAME': '',
     },
     'twitter': {
-        'FILES_STORE': 'favorites_crawler_files/twitter',
+        'FILES_STORE': os.path.join('$FAVORS_HOME', 'twitter'),
         'USER_ID': '',
         'AUTHORIZATION': '',
         'LIKES_ID': '',
         'X_CSRF_TOKEN': '',
     },
     'lemon': {
-        'FILES_STORE': 'favorites_crawler_files/lemon',
+        'FILES_STORE': os.path.join('$FAVORS_HOME', 'lemon'),
     },
     'nhentai': {
         'USER_AGENT': '',
-        'FILES_STORE': 'favorites_crawler_files/nhentai',
+        'FILES_STORE': os.path.join('$FAVORS_HOME', 'nhentai'),
     }
 }
 
@@ -63,22 +64,26 @@ def create_favors_home(path: str):
         os.makedirs(path, exist_ok=True)
 
 
-def overwrite_spider_settings(spider, default_settings, user_config):
+def overwrite_spider_settings(spider, home, user_config):
     """
     Overwrite spider settings by user config
     Priority: favors spider config > favors global config > spider custom settings > scrapy settings
 
     :param spider: Spider class
-    :param default_settings: :class:`scrapy.settings.Settings`
+    :param home: favors home
     :param user_config: favorites crawler config
     """
     global_config = user_config.get('global')
     if global_config:
         spider.custom_settings.update(global_config)
 
-    spider_config = user_config.get(spider.name)
+    spider_config = user_config.get(spider.name, {})
     if spider_config:
         spider.custom_settings.update(spider_config)
 
-    default_files_store = os.path.join(default_settings.get('FILES_STORE', ''), spider.name)
-    spider.custom_settings.setdefault('FILES_STORE', default_files_store)
+    home = os.path.expanduser(home)
+    files_store = spider_config.get('FILES_STORE')
+    if files_store:
+        spider.custom_settings['FILES_STORE'] = files_store.replace('$FAVORS_HOME', home)
+    else:
+        spider.custom_settings['FILES_STORE'] = os.path.join(home, spider.name)

tests/test_utils/test_config.py

@@ -84,7 +84,7 @@ def test_overwrite_spider_settings(self):
         }
         spider = spider_loader.load('pixiv')
 
-        overwrite_spider_settings(spider, scrapy_settings, user_config)
+        overwrite_spider_settings(spider, '~', user_config)
 
         assert spider.custom_settings['FILES_STORE'] == user_config['pixiv']['FILES_STORE']
         assert spider.custom_settings['ENABLE_ORGANIZE_BY_ARTIST'] == user_config['global']['ENABLE_ORGANIZE_BY_ARTIST']
@@ -100,14 +100,22 @@ def test_spider_config_priority_should_gt_global_config(self):
         }
         spider = spider_loader.load('yandere')
 
-        overwrite_spider_settings(spider, scrapy_settings, user_config)
+        overwrite_spider_settings(spider, '~', user_config)
 
         assert spider.custom_settings['ENABLE_ORGANIZE_BY_ARTIST'] == user_config['yandere']['ENABLE_ORGANIZE_BY_ARTIST']
 
     def test_should_set_default_file_store_when_user_doesnt_config_it(self):
         user_config = {}
         spider = spider_loader.load('nhentai')
 
-        overwrite_spider_settings(spider, scrapy_settings, user_config)
+        overwrite_spider_settings(spider, '~', user_config)
 
-        assert spider.custom_settings['FILES_STORE'] == os.path.join(scrapy_settings.get('FILES_STORE', ''), 'nhentai')
+        assert spider.custom_settings['FILES_STORE'] == os.path.expanduser(os.path.join('~', 'nhentai'))
+
+    def test_should_replace_favors_home_in_files_store(self):
+        user_config = {'twitter': {'FILES_STORE': os.path.join('$FAVORS_HOME', 'twitter')}}
+        spider = spider_loader.load('twitter')
+
+        overwrite_spider_settings(spider, '~', user_config)
+
+        assert spider.custom_settings['FILES_STORE'] == os.path.expanduser(os.path.join('~', 'twitter'))