Proper repo2data handling

Author: agahkarakuzu

example.py

@@ -2,16 +2,17 @@
 from myst_libre.rees import REES
 from myst_libre.builders import MystBuilder
 
+
+
 resources = REES(dict(
             registry_url="https://binder-registry.conp.cloud",
             gh_user_repo_name = "agahkarakuzu/mriscope",
             gh_repo_commit_hash = "ae64d9ed17e6ce66ecf94d585d7b68a19a435d70",
-            binder_image_tag = "489ae0eb0d08fe30e45bc31201524a6570b9b7dd",
-            dotenv = "/where/dotenv/is/located"))
+            binder_image_tag = "489ae0eb0d08fe30e45bc31201524a6570b9b7dd"))
 
 
 hub = JupyterHubLocalSpawner(resources,
-                             host_data_parent_dir = "/Users/agah/Desktop/neurolibre/mriscope/data",
+                             host_data_parent_dir = "/Users/agah/Desktop/tmp/DATA",
                              host_build_source_parent_dir = '/Users/agah/Desktop/tmp',
                              container_data_mount_dir = '/home/jovyan/data',
                              container_build_source_mount_dir = '/home/jovyan')

myst_libre/builders/myst_builder.py

@@ -6,7 +6,6 @@ def __init__(self, hub):
         if not isinstance(hub, JupyterHubLocalSpawner):
             raise TypeError(f"Expected 'hub' to be an instance of JupyterHubLocalSpawner, got {type(hub).__name__} instead")
         super().__init__()
-        #aa = MystMD('/Users/agah/Desktop', {"anan":5})
         self.env_vars = {}
         self.build_dir = ""
         self.hub = hub

myst_libre/rees/rees.py

@@ -26,6 +26,9 @@ def __init__(self, rees_dict):
         self.use_public_registry = False
         self.repo_commit_info = {}
         self.binder_commit_info = {}
+        # CHECK: This may not work properly without
+        # logging in to the registry on the host machine 
+        # which keeps that auth info on the config file. 
         self.docker_client = docker.from_env()
 
     def check_docker_installed(self):

myst_libre/tools/build_source_manager.py

@@ -10,6 +10,7 @@
 from git import Repo
 from datetime import datetime
 from myst_libre.abstract_class import AbstractClass
+from repo2data.repo2data import Repo2Data
 
 class BuildSourceManager(AbstractClass):
     """
@@ -84,6 +85,16 @@ def get_project_name(self):
             data = json.load(file)
         self.dataset_name = data['projectName']
 
+    def repo2data_download(self,target_directory):
+        data_req_path = os.path.join(self.build_dir, 'binder', 'data_requirement.json')
+        if not os.path.isfile(data_req_path):
+            self.logger.info(f'Skipping repo2data download')
+        else:
+            self.logger.info(f'Starting repo2data download')
+            repo2data = Repo2Data(data_req_path, server=True)
+            repo2data.set_server_dst_folder(target_directory)
+            repo2data.install()
+
     def set_commit_info(self):
         self.binder_commit_info['datetime'] = self.repo_object.commit(self.binder_image_tag).committed_datetime
         self.binder_commit_info['message'] = self.repo_object.commit(self.binder_image_tag).message

myst_libre/tools/jupyter_hub_local_spawner.py

@@ -90,8 +90,7 @@ def spawn_jupyter_hub(self):
             self.rees.get_project_name()
 
         if self.rees.dataset_name:
-            print(self.container_data_mount_dir)
-            print(self.rees.dataset_name)
+            self.rees.repo2data_download(self.host_data_parent_dir)
             mnt_vol = {f'{os.path.join(self.host_data_parent_dir,self.rees.dataset_name)}': {'bind': os.path.join(self.container_data_mount_dir,self.rees.dataset_name), 'mode': 'ro'},
                     self.rees.build_dir: {'bind': f'{self.container_build_source_mount_dir}', 'mode': 'rw'}}
         else:

pyproject.toml

@@ -24,7 +24,8 @@ dependencies = [
     "python-dotenv",
     "PyGithub",
     "termcolor",
-    "mystmd"
+    "mystmd",
+    "repo2data"
 ]
 
 [project.urls]

requirements.txt

@@ -5,4 +5,5 @@ docker
 hashlib
 termcolor
 mystmd
-requests
+requests
+repo2data==2.9.2