Cleanup hash conditions

Author: agahkarakuzu

README.md

@@ -1,4 +1,3 @@
-
 # MyST Libre
 
 ![PyPI - Version](https://img.shields.io/pypi/v/myst-libre?style=flat&logo=python&logoColor=white&logoSize=8&labelColor=rgb(255%2C0%2C0)&color=white)
@@ -45,21 +44,44 @@ DOCKER_PRIVATE_REGISTRY_PASSWORD=your_password
 
 **Import libraries and define REES resources**
 
+Minimal example to create a rees object:
+
 ```python
 from myst_libre.tools import JupyterHubLocalSpawner, MystMD
 from myst_libre.rees import REES
 from myst_libre.builders import MystBuilder
 
-rees_resources = REES(dict(
+rees = REES(dict(
                   registry_url="https://your-registry.io",
-                  gh_user_repo_name = "owner/repository",
-                  gh_repo_commit_hash = "full_SHA_commit_A",
-                  binder_image_tag = "full_SHA_commit_A_or_B",
-                  dotenv = '/path/to/dotenv'))
+                  gh_user_repo_name = "owner/repository"
+                  ))
 ```
 
-> [!NOTE]
-> Currently, the assumption is that the Docker image was built by binderhub from a REES-compliant repository that also includes the MyST content. Therefore, `binder_image_tag` and `gh_repo_commit_hash` are simply two different commits in the same (`gh_repo_user_name`) repository. However, `binder_image_tag` is not allowed to be ahead of `gh_repo_commit_hash`.
+Other optional parameters that can be passed to the REES constructor:
+
+
+- `gh_repo_commit_hash`: Full SHA commit hash of the `gh_user_repo_name` repository (optional, default: latest commit)
+- `binder_image_tag`: Full SHA commit hash at which a binder tag is available for the "found image name" (optional, default: latest)
+- `binder_image_name_override`: Override the "found image name" whose container will be used to build the MyST article (optional, default: None)
+- `dotenv`: Path to a directory containing the .env file for authentication credentials to pull images from `registry_url` (optional, default: None)
+- `bh_image_prefix`: Binderhub names the images with a prefix, e.g., `<prefix>agahkarakuzu-2dmriscope-7a73fb`, typically set as `binder-`. This will be used in the regex pattern to find the "binderhub built image name" in the `registry_url`. See [reference docs](https://binderhub.readthedocs.io/en/latest/zero-to-binderhub/setup-binderhub.html) for more details. 
+- `bh_private_project_name`: See [this issue ](https://github.com/jupyterhub/binderhub/issues/800) (optional, default: [`registry_url` without `http://` or `https://`])
+
+
+Note that in this context what is meant by "prefix" is not the same as in the reference docs. (optional, default: `binder-`)
+
+**Image Selection Order**
+
+1. If the `myst.yml` file in the `gh_user_repo_name` repository contains `project/thebe/binder/repo`, this image is prioritized.
+2. If `project/thebe/binder/repo` is not specified, the `gh_user_repo_name` is used as the image name.
+
+Note that if (2) is the case, your build command probably should not be `myst build`, but you can still use other builders, e.g., `jupyter-book build`.
+
+If you specify `binder_image_name_override`, it will be used as the repository name to locate the image.
+
+This allows you to build the MyST article using a runtime from a different repository than the one specified in `gh_user_repo_name`, as defined in `myst.yml` or overridden by `binder_image_name_override`.
+
+The `binder_image_tag` set to `latest` refers to the most recent successful build of an image that meets the specified conditions. The repository content might be more recent than the `binder_image_tag` (e.g., `gh_repo_commit_hash`), but the same binder image can be reused.
 
 **Fetch resources and spawn JupyterHub in the respective container**
 

example.py

@@ -4,19 +4,32 @@
 
 
 
-resources = REES(dict(
-            registry_url="https://binder-registry.conp.cloud",
-            gh_user_repo_name = "agahkarakuzu/mriscope",
-            gh_repo_commit_hash = "ae64d9ed17e6ce66ecf94d585d7b68a19a435d70",
-            binder_image_tag = "489ae0eb0d08fe30e45bc31201524a6570b9b7dd"))
+rees = REES(dict(
+    registry_url="https://binder-registry.conp.cloud",
+    gh_user_repo_name = "agahkarakuzu/mriscope",
+    dotenv = "/Users/agah/Desktop/neurolibre/myst_libre",
+    bh_project_name = "binder-registry.conp.cloud"
+    ))     
 
 
-hub = JupyterHubLocalSpawner(resources,
-                             host_data_parent_dir = "/Users/agah/Desktop/tmp/DATA",
-                             host_build_source_parent_dir = '/Users/agah/Desktop/tmp',
-                             container_data_mount_dir = '/home/jovyan/data',
-                             container_build_source_mount_dir = '/home/jovyan')
+# Under the hood it looks like this:
+# if rees.search_img_by_repo_name():
+#     print("🐳 Image name:",rees.found_image_name)
+#     print("🏷️ Unsorted tags:",rees.found_image_tags)
 
-hub.spawn_jupyter_hub()
+# if rees.get_tags_sorted_by_date():
+#     print("🏷️ Sorted image tags:",rees.found_image_tags_sorted)
 
-MystBuilder(hub).build_site()
+hub = JupyterHubLocalSpawner(rees,
+                        host_build_source_parent_dir = "/Users/agah/Desktop/tmp",
+                        container_build_source_mount_dir = '/home/jovyan',
+                        host_data_parent_dir = "/Users/agah/Desktop/tmp/DATA",
+                        container_data_mount_dir = '/home/jovyan/data')
+
+# # This has to be called
+hub_logs = hub.spawn_jupyter_hub()
+
+builder = MystBuilder(hub=hub)
+myst_logs = builder.build('--execute','--html')
+
+print(myst_logs)

myst_libre/abstract_class.py

@@ -36,12 +36,13 @@ def set_log_level(self, level):
         self.logging_level = level
         self.logger = logging.basicConfig(level=self.logging_level, format='%(asctime)s - %(levelname)s - %(message)s')
 
-    def cprint(self, message, color):
+    def cprint(self, message, color,highlight=None):
         """
         Print a message in a specified color using termcolor.
         
         Args:
             message (str): The message to print.
             color (str): The color to use for printing the message.
+            highlight (str): The highlight to use for printing the message.
         """
-        print(colored(message, color))
+        print(colored(message, color, highlight))

myst_libre/rees/rees.py

@@ -10,17 +10,40 @@ def __init__(self, rees_dict):
         # These are needed in the scope of the base classes
         self.registry_url = rees_dict['registry_url']
         self.gh_user_repo_name = rees_dict['gh_user_repo_name']
-        self.gh_repo_commit_hash = rees_dict['gh_repo_commit_hash']
-        self.binder_image_tag = rees_dict['binder_image_tag']
-        self.binder_image_name = rees_dict.get('binder_image_name', None)
-        
+        self.gh_repo_commit_hash = rees_dict.get('gh_repo_commit_hash', "latest")
+        self.binder_image_name_override = rees_dict.get('binder_image_name_override', None)
+        self.binder_image_tag = rees_dict.get('binder_image_tag', "latest")
+        self.bh_private_project_name = rees_dict.get('bh_private_project_name', None)
+        self.bh_image_prefix = rees_dict.get('bh_image_prefix', "binder-")
+        self.bh_project_name = rees_dict.get('bh_project_name', None)
+
         if 'dotenv' in rees_dict.keys():
             self.dotenvloc = rees_dict['dotenv']
 
         # Initialize as base to rees
         BuildSourceManager.__init__(self)
         DockerRegistryClient.__init__(self)
 
+        # Get the latest commit hash from the GitHub API
+        if self.gh_repo_commit_hash == "latest":
+            self.cprint("🔎 Searching for latest commit hash in GitHub.","light_blue")
+            api_url = f'https://api.github.com/repos/{self.gh_user_repo_name}/commits/{self.branch}'
+            response = self.rest_client.get(api_url)
+            if response.status_code == 200:
+                latest_commit_hash = response.json()['sha']
+                self.gh_repo_commit_hash = latest_commit_hash
+                self.cprint(f"🎉 Found latest commit hash: {self.gh_repo_commit_hash}","white","on_blue")
+
+        if self.search_img_by_repo_name():
+            if self.binder_image_tag == "latest":
+                self.cprint("🔎 Searching for latest image tag in registry.","light_blue")
+                if self.get_tags_sorted_by_date():
+                    self.cprint(f"🎉 Found latest tag in {self.registry_url}","light_blue")
+                    self.cprint(f"🏷️  Latest runtime tag {self.found_image_tags_sorted[0]} for {self.found_image_name}","white","on_blue")
+                    self.binder_image_tag = self.found_image_tags_sorted[0]
+        else:
+            raise Exception(f"[ERROR] A docker image has not been found for {self.gh_user_repo_name} at {self.binder_image_tag}.")
+
         self.cprint(f"␤[Preflight checks]","light_grey")
         self.check_docker_installed()
 
@@ -58,10 +81,10 @@ def pull_image(self):
         """
         if bool(self._auth) or not self.use_public_registry:
             self.login_to_registry()
-            self.logger.info(f"Logging into {self.registry_url_bare}")
+            self.logger.info(f"Logging into {self.registry_url}")
 
         try:
-            self.pull_image_name = f'{self.registry_url_bare}/{self.found_image_name}'
+            self.pull_image_name = f'{self.bh_private_project_name}/{self.found_image_name}'
             self.logger.info(f'Pulling image {self.pull_image_name}:{self.binder_image_tag} from {self.registry_url}.')
             self.docker_image = self.docker_client.images.pull(self.pull_image_name, tag=self.binder_image_tag)
         except:

myst_libre/tools/build_source_manager.py

@@ -54,15 +54,17 @@ def git_clone_repo(self,clone_parent_directory):
         self.build_dir = os.path.join(self.host_build_source_parent_dir, self.username, self.repo_name, self.gh_repo_commit_hash)
 
         if os.path.exists(self.build_dir):
-            self.cprint(f'Source {self.build_dir} already exists.', "yellow")
+            self.cprint(f'Source {self.build_dir} already exists.', "black","on_yellow")
             self.repo_object = Repo(self.build_dir)
+            if os.path.exists(os.path.join(self.build_dir, '_build/html')):
+                self.cprint(f'⛔️ A build already exists at this commit, terminating...', "white","on_light_red")
+                raise Exception("A build already exists at this commit")
         else:
             os.makedirs(os.path.dirname(self.build_dir), exist_ok=True)
             self.cprint(f'Cloning into {self.build_dir}', "green")
             self.repo_object = Repo.clone_from(f'{self.provider}/{self.gh_user_repo_name}', self.build_dir)
 
         self.set_commit_info()
-        self.validate_commits()
 
     def git_checkout_commit(self):
         """
@@ -103,20 +105,14 @@ def repo2data_download(self,target_directory):
             repo2data.install()
 
     def set_commit_info(self):
-        if self.binder_image_name:
+        if self.binder_image_name_override:
             self.binder_commit_info['datetime'] = "20 November 2024"
             self.binder_commit_info['message'] =  "Base runtime from myst-libre"
         else:
             self.binder_commit_info['datetime'] = self.repo_object.commit(self.binder_image_tag).committed_datetime
             self.binder_commit_info['message'] = self.repo_object.commit(self.binder_image_tag).message
         self.repo_commit_info['datetime'] = self.repo_object.commit(self.gh_repo_commit_hash).committed_datetime
         self.repo_commit_info['message'] = self.repo_object.commit(self.gh_repo_commit_hash).message
-        self.validate_commits()
-
-    def validate_commits(self):
-        if not self.binder_image_name:
-            if self.repo_commit_info['datetime'] < self.binder_commit_info['datetime']:
-                raise ValueError("The repo commit datetime cannot be older than the binder commit datetime.")
 
     def create_latest_symlink(self):
         """

myst_libre/tools/docker_registry_client.py

@@ -5,6 +5,9 @@
 """
 
 import re
+import datetime
+import yaml
+
 from .rest_client import RestClient
 from .authenticator import Authenticator
 from .decorators import request_set_decorator
@@ -30,10 +33,11 @@ def __init__(self, **kwargs):
         else:
             super().__init__()
             self.rest_client = RestClient()
-
-        self.registry_url_bare = self.registry_url.replace("http://", "").replace("https://", "")
+        
         self.found_image_name = None
         self.found_image_tags = None
+        self.found_image_tags_sorted = None
+        self.myst_yml_dict = None
         self.docker_images = []
 
 
@@ -60,16 +64,36 @@ def search_img_by_repo_name(self):
             bool: True if image found, else False.
         
         Assuming use in rees, TODO refactored for clarity:
-            binder_image_name
+            binder_image_name_override
             gh_user_repo_name 
         """
         self.get_image_list()
-        if self.binder_image_name:
-            src_name = self.binder_image_name
+        if self.binder_image_name_override:
+            src_name = self.binder_image_name_override
         else:
-            src_name = self.gh_user_repo_name
+            # If not overridden, the (thebe/binder)repo in myst.yml takes precedence
+            if self.get_myst_yml_as_dict():
+                src_name = self.myst_yml_dict['project']['thebe']['binder']['repo']
+                if src_name:
+                    self.cprint(f"🥳 Using project::thebe::binder::repo from myst config to look for 🐳 in {self.registry_url}: {src_name}","light_blue")
+                else:
+                    # Fallback to the GitHub user/repo name
+                    src_name = self.gh_user_repo_name
+                    self.cprint(f"ℹ️ project::thebe::binder::repo not found in myst.yml, using GitHub {self.gh_user_repo_name} name to look for 🐳 in {self.registry_url}.","light_blue")
+                    self.cprint("‼️ IMPORTANT WARNING: Myst build may succeed, but binder-specific config must be added to myst.yml to attach the built page to a proper runtime.","light_red")
+                    return False
+            else:
+                # Well, build will most likely fail, but we can try
+                self.cprint("⚠️ myst.yml not found, using GitHub user/repo name for 🐳.","light_red")
+                src_name = self.gh_user_repo_name
+
         user_repo_formatted = src_name.replace('-', '-2d').replace('_', '-5f').replace('/', '-2d')
-        pattern = f'{self.registry_url_bare}/binder-{user_repo_formatted}.*'
+        if self.bh_project_name:
+            pattern = f'{self.bh_project_name}/{self.bh_image_prefix}{user_repo_formatted}.*'
+        else:
+            pattern = f'{self.bh_image_prefix}{user_repo_formatted}.*'
+
+        print(pattern)
         for image in self.docker_images:
             if re.match(pattern, image):
                 self.found_image_name = image
@@ -98,3 +122,52 @@ def list_tags(self):
         """
         tags_url = f"{self.registry_url}/v2/{self.found_image_name}/tags/list"
         return self.rest_client.get(tags_url)
+
+    #@request_set_decorator(success_status_code=200, set_attribute="found_image_tags_sorted", json_key="tags")
+    def get_tags_sorted_by_date(self):
+        tags = self.list_tags()
+        tag_dates = []
+        for tag in tags:
+            manifest_url = f"{self.registry_url}/v2/{self.found_image_name}/manifests/{tag}"
+            headers = {'Accept': 'application/vnd.docker.distribution.manifest.v2+json'}
+            manifest = self.rest_client.get(manifest_url, headers=headers).json()
+
+            config_digest = manifest['config']['digest']
+            config_url = f"{self.registry_url}/v2/{self.found_image_name}/blobs/{config_digest}"
+            config = self.rest_client.get(config_url).json()
+
+            created = config.get('created', None)
+            if created:
+                # Truncate the fractional seconds to 6 digits
+                created = re.sub(r'(\.\d{6})\d+', r'\1', created)
+                dt = datetime.datetime.fromisoformat(created.rstrip('Z'))
+                tag_dates.append((tag, dt))
+        sorted_tags = sorted(tag_dates, key=lambda x: x[1], reverse=True)
+        self.found_image_tags_sorted = [tag for tag, _ in sorted_tags]
+        return bool(tag_dates)
+
+    def get_myst_yml_as_dict(self):
+        """
+        Get the myst.yml file as a dictionary.
+        
+        Returns:
+            dict: The parsed myst.yml file as a dictionary.
+        """
+        # This is a helper function to get the myst.yml file as a dictionary
+        # Maybe move it to a helper file, but this whole project needs a refactor anyway.
+        # I know it makes your eyes bleed, sorry.
+        user, repo = self.gh_user_repo_name.split('/')
+        url = f"https://raw.githubusercontent.com/{user}/{repo}/main/myst.yml"
+        response = self.rest_client.get(url)
+        
+        if response.status_code == 200:
+            try:
+                # Parse the YAML content into a dictionary
+                self.myst_yml_dict = yaml.safe_load(response.text)
+                return True
+            except yaml.YAMLError as e:
+                self.logger.error(f"Error parsing YAML: {e}")
+                return False
+        else:
+            self.logger.error(f"Failed to fetch myst.yml: {response.status_code}")
+            return False

myst_libre/tools/jupyter_hub_local_spawner.py

@@ -82,11 +82,6 @@ def spawn_jupyter_hub(self, jb_build_command=None):
             this_entrypoint = f"/bin/sh -c 'jupyter-book build --all --verbose --path-output {self.container_build_source_mount_dir} content 2>&1 | tee -a jupyter_book_build.log'"
         else:
             this_entrypoint = f'jupyter server --allow-root --ip 0.0.0.0 --log-level=DEBUG --IdentityProvider.token="{self.jh_token}" --ServerApp.port="{self.port}"'
-
-        if not self.rees.search_img_by_repo_name():
-            raise Exception(f"[ERROR] A docker image has not been found for {self.rees.gh_user_repo_name} at {self.rees.binder_image_tag}.")
-        if self.rees.binder_image_tag not in self.rees.found_image_tags:
-            raise Exception(f"[ERROR] A docker image exists for {self.rees.gh_user_repo_name}, yet the tag {self.rees.binder_image_tag} is missing.")
 
         # self.rees.found_image_name is assigned if above not fails
 
@@ -139,8 +134,8 @@ def log_and_print(message, color=None):
             log_and_print(f'     ├───────── ✸ {self.rees.pull_image_name}', 'light_blue')
             log_and_print(f'     ├───────── ⎌ {self.rees.binder_image_tag}', 'light_blue')
             log_and_print(f"     ├───────── ⏲ {self.rees.binder_commit_info['datetime']}: {self.rees.binder_commit_info['message']}".replace('\n', ''), 'light_blue')
-            if self.rees.binder_image_name:
-                log_and_print(f'     └───────── ℹ Using NeuroLibre base image {self.rees.binder_image_name}', 'yellow')
+            if self.rees.binder_image_name_override:
+                log_and_print(f'     └───────── ℹ Using NeuroLibre base image {self.rees.binder_image_name_override}', 'yellow')
             else:    
                 log_and_print(f'     └───────── ℹ This image was built from REES-compliant {self.rees.gh_user_repo_name} repository at the commit above', 'yellow')
         except Exception as e:

myst_libre/tools/rest_client.py

@@ -23,7 +23,7 @@ def __init__(self,dotenvloc = '.'):
         self.session = requests.Session()
         self.session.auth = HTTPBasicAuth(self._auth['username'], self._auth['password'])
 
-    def get(self, url):
+    def get(self, url,headers=None):
         """
         Perform a GET request.
         
@@ -33,7 +33,7 @@ def get(self, url):
         Returns:
             Response: HTTP response object.
         """
-        response = self.session.get(url)
+        response = self.session.get(url,headers=headers)
         return response
 
     def post(self, url, data=None, json=None):

pyproject.toml

@@ -25,11 +25,12 @@ dependencies = [
     "PyGithub",
     "termcolor",
     "mystmd",
-    "repo2data"
+    "repo2data",
+    "pyyaml"
 ]
 
 [project.urls]
-Homepage = "https://github.com/agahkarakuzu/myst_libre"
+Homepage = "https://github.com/neurolibre/myst_libre"
 
 [tool.setuptools_scm]
 version_scheme = "post-release"