RedisLite -> FakeRedis (Netflix#9327)

* Upgrade dependencies to support Python 3.10

* Upgrade dependencies to support Python 3.10

* Python 3.10 in Pre-commit

* Python 3.10 in Pre-commit try 2

* Arm support: Switch Redislite with Fakeredis

* Arm support: Switch Redislite with Fakeredis

* fix lint

Author: castrapel

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party=consoleme,consoleme_ecs_cdk
-known_third_party = aiozipkin,asgiref,atlassian,aws_cdk,bcrypt,billiard,bleach,boto3,botocore,celery,cfnresponse,click,click_log,cloudaux,cryptography,dateutil,deepdiff,distutils,elasticsearch,email_validator,furl,git,google,googleapiclient,jsonschema,jwt,logmatic,marshmallow,mock,mockredis,moto,nacl,nested_stacks,okta_jwt,onelogin,pandas,parliament,password_strength,pkg_resources,policy_sentry,policyuniverse,pydantic,pytest,pytz,questionary,redis,redislite,requests,retrying,ruamel,sentry_sdk,setuptools,simplejson,tenacity,tornado,ujson,uvloop,validate_email,yaml
+known_third_party = aiozipkin,asgiref,atlassian,aws_cdk,bcrypt,billiard,bleach,boto3,botocore,celery,cfnresponse,click,click_log,cloudaux,cryptography,dateutil,deepdiff,distutils,elasticsearch,email_validator,fakeredis,furl,git,google,googleapiclient,jsonschema,jwt,logmatic,marshmallow,mock,mockredis,moto,nacl,nested_stacks,okta_jwt,onelogin,pandas,parliament,password_strength,pkg_resources,policy_sentry,policyuniverse,pydantic,pytest,pytz,questionary,redis,requests,retrying,ruamel,sentry_sdk,setuptools,simplejson,tenacity,tornado,ujson,uvloop,validate_email,yaml
 multi_line_output=3
 include_trailing_comma=True
 balanced_wrapping=True

Dockerfile

@@ -1,5 +1,5 @@
 # Dockerfile should instantiate AWS Project with configurable plugins
-FROM python:3.8
+FROM python:3.10
 MAINTAINER Netflix Security
 WORKDIR /apps/consoleme
 # NODE_OPTIONS meeded to increase memory size of Node for the `yarn build` step. The Monaco Editor

README.md

@@ -1,4 +1,4 @@
-[![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-386/)
+[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3105/)
 [![Discord](https://img.shields.io/discord/730908778299523072?label=Discord&logo=discord&style=flat-square)](https://discord.gg/nQVpNGGkYu)
 
 # ConsoleMe

consoleme/celery_tasks/celery_tasks.py

@@ -10,7 +10,6 @@
 from __future__ import absolute_import
 
 import json  # We use a separate SetEncoder here so we cannot use ujson
-import os
 import sys
 import time
 from datetime import datetime, timedelta
@@ -117,25 +116,13 @@ def on_configure(self) -> None:
     ),
     backend=config.get(
         f"celery.backend.{config.region}",
-        config.get("celery.broker.global", "redis://127.0.0.1:6379/2"),
+        config.get("celery.backend.global"),
     ),
 )
 
-if config.get("redis.use_redislite"):
-    import tempfile
-
-    import redislite
-
-    redislite_db_path = os.path.join(
-        config.get("redis.redislite.db_path", tempfile.NamedTemporaryFile().name)
-    )
-    redislite_client = redislite.Redis(redislite_db_path)
-    redislite_socket_path = f"redis+socket://{redislite_client.socket_file}"
-    app = Celery(
-        "tasks",
-        broker=f"{redislite_socket_path}?virtual_host=1",
-        backend=f"{redislite_socket_path}?virtual_host=2",
-    )
+broker_transport_options = config.get("celery.broker_transport_options")
+if broker_transport_options:
+    app.conf.update({"broker_transport_options": dict(broker_transport_options)})
 
 app.conf.result_expires = config.get("celery.result_expires", 60)
 app.conf.worker_prefetch_multiplier = config.get("celery.worker_prefetch_multiplier", 4)

consoleme/config/config.py

@@ -156,24 +156,6 @@ def __set_flag_on_main_exit(self):
         # Main thread exited, signal to other threads
         main_exit_flag.set()
 
-    def purge_redislite_cache(self):
-        """
-        Purges redislite cache in primary DB periodically. This will force a cache refresh, and it is
-        convenient for cases where you cannot securely run shared Redis (ie: AWS AppRunner)
-        """
-        if not self.get("redis.use_redislite"):
-            return
-        from consoleme.lib.redis import RedisHandler
-
-        red = RedisHandler().redis_sync()
-        while threading.main_thread().is_alive():
-            red.flushdb()
-            # Wait till main exit flag is set OR a fixed timeout
-            if main_exit_flag.wait(
-                timeout=self.get("redis.purge_redislite_cache_interval", 1800)
-            ):
-                break
-
     async def merge_extended_paths(self, extends, dir_path):
         for s in extends:
             extend_config = {}
@@ -243,10 +225,6 @@ async def load_config(
         if allow_start_background_threads:
             Timer(0, self.__set_flag_on_main_exit, ()).start()
 
-        if allow_start_background_threads and self.get("redis.use_redislite"):
-            t = Timer(1, self.purge_redislite_cache, ())
-            t.start()
-
         if allow_start_background_threads and self.get("config.load_from_dynamo", True):
             t = Timer(2, self.load_config_from_dynamo_bg_thread, ())
             t.start()
@@ -361,8 +339,6 @@ def set_logging_levels(self):
             "spectator.HttpClient": "WARNING",
             "spectator.Registry": "WARNING",
             "urllib3": "ERROR",
-            "redislite.client": "WARNING",
-            "redislite.configuration": "WARNING",
         }
         for logger, level in self.get("logging_levels", default_logging_levels).items():
             logging.getLogger(logger).setLevel(level)

consoleme/default_plugins/plugins/celery_tasks/celery_tasks.py

@@ -4,7 +4,6 @@
 
 """
 import json
-import os
 from datetime import timedelta
 
 from asgiref.sync import async_to_sync
@@ -26,31 +25,15 @@
     ),
     backend=config.get(
         f"celery.backend.{config.region}",
-        config.get("celery.broker.global", "redis://127.0.0.1:6379/2"),
+        config.get("celery.backend.global"),
     ),
 )
 
-if config.get("redis.use_redislite"):
-    import tempfile
-
-    import redislite
-
-    redislite_db_path = os.path.join(
-        config.get("redis.redislite.db_path", tempfile.NamedTemporaryFile().name)
-    )
-    redislite_client = redislite.Redis(redislite_db_path)
-    redislite_socket_path = f"redis+socket://{redislite_client.socket_file}"
-    app = Celery(
-        "tasks",
-        broker=f"{redislite_socket_path}?virtual_host=1",
-        backend=f"{redislite_socket_path}?virtual_host=2",
-    )
-
 app.conf.result_expires = config.get("celery.result_expires", 60)
 app.conf.worker_prefetch_multiplier = config.get("celery.worker_prefetch_multiplier", 4)
 app.conf.task_acks_late = config.get("celery.task_acks_late", True)
 
-if config.get("celery.purge") and not config.get("redis.use_redislite"):
+if config.get("celery.purge"):
     # Useful to clear celery queue in development
     with Timeout(seconds=5, error_message="Timeout: Are you sure Redis is running?"):
         app.control.purge()

consoleme/lib/plugins.py

@@ -27,8 +27,8 @@ def get_plugin_by_name(plugin_name: str) -> Any:
     if plugin_name == "default_config":
         initial_exception_message = (
             f"Could not find the specified plugin: {plugin_name}. "
-            "Please install it with `pip install -e default_plugins` "
-            "from the ConsoleMe directory. "
+            "Please install it with `pip install -e consoleme/default_plugins` "
+            "from the ConsoleMe base directory. "
         )
 
     exception_message = (

consoleme/lib/redis.py

@@ -1,4 +1,3 @@
-import os
 import sys
 import threading
 import time
@@ -13,14 +12,6 @@
 from consoleme.config import config
 from consoleme.lib.plugins import get_plugin_by_name
 
-if config.get("redis.use_redislite"):
-    import tempfile
-
-    import redislite
-
-    if not config.get("redis.redis_lite.db_path"):
-        default_redislite_db_path = tempfile.NamedTemporaryFile().name
-
 region = config.region
 log = config.get_logger()
 stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
@@ -371,11 +362,6 @@ def __init__(
             self.enabled = False
 
     async def redis(self, db: int = 0) -> Redis:
-        if config.get("redis.use_redislite"):
-            REDIS_DB_PATH = os.path.join(
-                config.get("redis.redislite.db_path", default_redislite_db_path)
-            )
-            return redislite.StrictRedis(REDIS_DB_PATH, decode_responses=True)
         self.red = await sync_to_async(ConsoleMeRedis)(
             host=self.host,
             port=self.port,
@@ -386,11 +372,6 @@ async def redis(self, db: int = 0) -> Redis:
         return self.red
 
     def redis_sync(self, db: int = 0) -> Redis:
-        if config.get("redis.use_redislite"):
-            REDIS_DB_PATH = os.path.join(
-                config.get("redis.redislite.db_path", default_redislite_db_path)
-            )
-            return redislite.StrictRedis(REDIS_DB_PATH, decode_responses=True)
         self.red = ConsoleMeRedis(
             host=self.host,
             port=self.port,

example_config/example_config_test.yaml

@@ -41,6 +41,12 @@ celery:
     enabled: true
   cache_cloudtrail_denies:
     enabled: true
+  broker:
+    global: filesystem://.pytest_cache/0
+  broker_transport_options:
+    data_folder_in: .pytest_cache/celery_data_folder_in
+    data_folder_out: .pytest_cache/celery_data_folder_out
+    data_folder_processed: .pytest_cache/celery_data_folder_processed
 event_bridge:
   detect_role_changes_and_update_cache:
     queue_arn: arn:aws:sqs:{region}:123456789012:consoleme-cloudtrail-role-events-test

requirements-test.in

@@ -4,10 +4,11 @@ bandit
 black
 coverage
 docker # Required by moto
+fakeredis
 flake8
 flake8-docstrings
 flake8-import-order
-mock<4 # Mock 4 breaks a bunch of unit tests
+mock
 mockredispy
 moto
 mypy
@@ -20,7 +21,6 @@ pytest-mock
 pytest-timeout
 pytest-tornado
 pytest-xdist
-redislite
 readme_renderer
 tornado
 tox