diff --git a/setup.py b/setup.py index 81deeca..6067b2c 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,10 @@ def prerelease_local_scheme(version): # perform the install setup( name='girder-slicer-cli-web', - use_scm_version={ - 'local_scheme': prerelease_local_scheme, - 'fallback_version': '0.0.0'}, + # use_scm_version={ + # 'local_scheme': prerelease_local_scheme, + # 'fallback_version': '0.0.0'}, + version='1.5.5', setup_requires=[ 'setuptools-scm', ], @@ -72,6 +73,9 @@ def prerelease_local_scheme(version): 'worker': [ 'docker>=2.6.0', 'girder-worker[worker]>=0.6.0', + ], + 'singularity': [ + 'slicer-cli-web-singularity', ] }, entry_points={ diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 8d62f2f..503dfb3 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -33,9 +33,14 @@ from girder_jobs.models.job import Job from .config import PluginSettings -from .models import CLIItem, DockerImageItem, DockerImageNotFoundError +from .models import CLIItem, DockerImageNotFoundError from .rest_slicer_cli import genRESTEndPointsForSlicerCLIsForItem +try: + from slicer_cli_web_singularity.singularity_image import SingularityImageItem as ImageItem +except ImportError: + from .models import DockerImageItem as ImageItem + class DockerResource(Resource): """ @@ -72,7 +77,7 @@ def __init__(self, name): def getDockerImages(self, params): data = {} if self.getCurrentUser(): - for image in DockerImageItem.findAllImages(self.getCurrentUser()): + for image in ImageItem.findAllImages(self.getCurrentUser()): imgData = {} for cli in image.getCLIs(): basePath = '/%s/cli/%s' % (self.resourceName, cli._id) @@ -119,7 +124,7 @@ def _deleteImage(self, names, deleteImage): docker rmi -f ) :type name: boolean """ - removed = DockerImageItem.removeImages(names, self.getCurrentUser()) + removed = ImageItem.removeImages(names, self.getCurrentUser()) if removed != names: rest = [name for name in names if name not in removed] raise RestException('Some docker images could not be removed. %s' % (rest)) diff --git a/slicer_cli_web/image_job.py b/slicer_cli_web/image_job.py index 8aa4539..520a17b 100644 --- a/slicer_cli_web/image_job.py +++ b/slicer_cli_web/image_job.py @@ -17,6 +17,7 @@ ############################################################################### import json +import os import time import docker @@ -29,6 +30,19 @@ from .models import DockerImageError, DockerImageItem, DockerImageNotFoundError +try: + from slicer_cli_web_singularity.job import (find_and_remove_local_sif_files, + is_singularity_installed, + load_meta_data_for_singularity, + pull_image_and_convert_to_sif) + from slicer_cli_web_singularity.singularity_image import SingularityImage, SingularityImageItem + from slicer_cli_web_singularity.utils import (generate_image_name_for_singularity, + switch_to_sif_image_folder) + USE_SINGULARITY = True # TODO: do this better +except ImportError as e: + USE_SINGULARITY = False + logger.info(f'Failed to import singularity modules: {e}') + def deleteImage(job): """ @@ -40,32 +54,42 @@ def deleteImage(job): """ job = Job().updateJob( job, - log='Started to Delete Docker images\n', + log=f'Started to Delete {"Singularity" if USE_SINGULARITY else "Docker"} images\n', status=JobStatus.RUNNING, ) + docker_client = None + try: deleteList = job['kwargs']['deleteList'] error = False - try: - docker_client = docker.from_env(version='auto') + if USE_SINGULARITY: + sif_folder = os.getenv('SIF_IMAGE_PATH') + else: + try: + docker_client = docker.from_env(version='auto') - except docker.errors.DockerException as err: - logger.exception('Could not create the docker client') - job = Job().updateJob( - job, - log='Failed to create the Docker Client\n' + str(err) + '\n', - status=JobStatus.ERROR, - ) - raise DockerImageError('Could not create the docker client') + except docker.errors.DockerException as err: + logger.exception('Could not create the docker client') + job = Job().updateJob( + job, + log='Failed to create the Docker Client\n' + str(err) + '\n', + status=JobStatus.ERROR, + ) + raise DockerImageError('Could not create the docker client') for name in deleteList: try: - docker_client.images.remove(name, force=True) + if USE_SINGULARITY: + name = generate_image_name_for_singularity(name) + filename = os.path.join(sif_folder, name) + os.remove(filename) + else: + docker_client.images.remove(name, force=True) except Exception as err: - logger.exception('Failed to remove image') + logger.exception('Failed to remove image ', name) job = Job().updateJob( job, log='Failed to remove image \n' + str(err) + '\n', @@ -129,38 +153,68 @@ def jobPullAndLoad(job): try: job = Job().updateJob( job, - log='Started to Load Docker images\n', + log=f'Started to Load {"Singularity" if USE_SINGULARITY else "Docker"} images\n', status=JobStatus.RUNNING, ) user = User().load(job['userId'], level=AccessType.READ) baseFolder = Folder().load( job['kwargs']['folder'], user=user, level=AccessType.WRITE, exc=True) - + logger.info(f"names = {job['kwargs']['nameList']}") loadList = job['kwargs']['nameList'] + logger.info(f'loadList = {loadList}') + job = Job().updateJob( + job, + log=f'loadList = {loadList}\n', + ) + + job = Job().updateJob( + job, + log=f'singularity = {USE_SINGULARITY}\n', + ) errorState = False notExistSet = set() - try: - docker_client = docker.from_env(version='auto') + if USE_SINGULARITY: + is_singularity_installed() - except docker.errors.DockerException as err: - logger.exception('Could not create the docker client') - job = Job().updateJob( - job, - log='Failed to create the Docker Client\n' + str(err) + '\n', - ) - raise DockerImageError('Could not create the docker client') + # Singularity doesn't use layers and uses caching so if we have a new version of the + # image with the same tag, it will not be pulled but instead the same is used from + # singularity cache. Therefore, we need to remove local images if new version with + # same tag has to be pulled. + # MAKE SURE YOU'RE' NOT CONSTANTLY PULLING THE SAME VERSION OF THE IMAGE + # UNTIL THE ACTUAL IMAGE IS UPDATED + for name in loadList: + find_and_remove_local_sif_files(name) + + pullList = loadList + loadList = [] + + else: + try: + docker_client = docker.from_env(version='auto') + + except docker.errors.DockerException as err: + logger.exception('Could not create the docker client') + job = Job().updateJob( + job, + log='Failed to create the Docker Client\n' + str(err) + '\n', + ) + raise DockerImageError('Could not create the docker client') - pullList = [ - name for name in loadList - if not findLocalImage(docker_client, name) or - str(job['kwargs'].get('pull')).lower() == 'true'] - loadList = [name for name in loadList if name not in pullList] + pullList = [ + name for name in loadList + if not findLocalImage(docker_client, name) or + str(job['kwargs'].get('pull')).lower() == 'true'] + loadList = [name for name in loadList if name not in pullList] try: stage = 'pulling' - pullDockerImage(docker_client, pullList, job) + if USE_SINGULARITY: + switch_to_sif_image_folder() + pull_image_and_convert_to_sif(pullList) + else: + pullDockerImage(docker_client, pullList, job) except DockerImageNotFoundError as err: errorState = True notExistSet = set(err.imageName) @@ -170,12 +224,23 @@ def jobPullAndLoad(job): notExistSet) + '\n', ) stage = 'metadata' - images, loadingError = loadMetadata(job, docker_client, pullList, - loadList, notExistSet) - for name, cli_dict in images: - docker_image = docker_client.images.get(name) - stage = 'parsing' - DockerImageItem.saveImage(name, cli_dict, docker_image, user, baseFolder) + + if USE_SINGULARITY: + images, loadingError = load_meta_data_for_singularity(job, pullList, + loadList, notExistSet) + for name, cli_dict in images: + singularity_image_object = SingularityImage(name) + stage = 'parsing' + SingularityImageItem.saveImage( + name, cli_dict, singularity_image_object, user, baseFolder) + else: + images, loadingError = loadMetadata(job, docker_client, pullList, + loadList, notExistSet) + for name, cli_dict in images: + docker_image = docker_client.images.get(name) + stage = 'parsing' + DockerImageItem.saveImage(name, cli_dict, docker_image, user, baseFolder) + if errorState is False and loadingError is False: newStatus = JobStatus.SUCCESS else: @@ -222,7 +287,6 @@ def loadMetadata(job, docker_client, pullList, loadList, notExistSet): job = Job().updateJob( job, log='Image %s was pulled successfully \n' % name, - ) try: diff --git a/slicer_cli_web/rest_slicer_cli.py b/slicer_cli_web/rest_slicer_cli.py index 48d8c81..c37e183 100644 --- a/slicer_cli_web/rest_slicer_cli.py +++ b/slicer_cli_web/rest_slicer_cli.py @@ -396,7 +396,10 @@ def cliSubHandler(currentItem, params, user, token, datalist=None): :param datalist: if not None, an object with keys that override parameters. No outputs are used. """ - from .girder_worker_plugin.direct_docker_run import run + try: + from slicer_cli_web_singularity.girder_worker_plugin.direct_singularity_run import run + except ImportError: + from .girder_worker_plugin.direct_docker_run import run original_params = copy.deepcopy(params) if hasattr(getCurrentToken, 'set'): @@ -461,7 +464,7 @@ def cliSubHandler(currentItem, params, user, token, datalist=None): girder_job_type=jobType, girder_job_title=jobTitle, girder_result_hooks=result_hooks, - image=cliItem.digest, + image=cliItem.image, # TODO: check this shouldnt be .digest for Docker pull_image='if-not-present', container_args=container_args, **job_kwargs diff --git a/slicer_cli_web/singularity/pyproject.toml b/slicer_cli_web/singularity/pyproject.toml new file mode 100644 index 0000000..8fd8d67 --- /dev/null +++ b/slicer_cli_web/singularity/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" diff --git a/slicer_cli_web/singularity/setup.py b/slicer_cli_web/singularity/setup.py new file mode 100644 index 0000000..0f804f5 --- /dev/null +++ b/slicer_cli_web/singularity/setup.py @@ -0,0 +1,30 @@ +from setuptools import find_packages, setup + +setup( + name='slicer-cli-web-singularity', + version='0.0.0', + description='A girder plugin adding singularity support to slicer-cli-web', + author='Kitware, Inc.', + author_email='kitware@kitware.com', + license='Apache Software License 2.0', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + ], + install_requires=['girder-slicer-cli-web', 'girder-worker-singularity'], + entry_points={ + 'girder_worker_plugins': [ + 'slicer_cli_web_singularity = slicer_cli_web_singularity:SlicerCLISingularityWebWorkerPlugin', + ] + }, + packages=find_packages(), + zip_safe=False +) diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/__init__.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/__init__.py new file mode 100644 index 0000000..a8b6769 --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/__init__.py @@ -0,0 +1,25 @@ +############################################################################### +# Copyright Kitware Inc. +# +# Licensed under the Apache License, Version 2.0 ( the "License" ); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +from girder_worker import GirderWorkerPluginABC + + +class SlicerCLISingularityWebWorkerPlugin(GirderWorkerPluginABC): + def __init__(self, app, *args, **kwargs): + self.app = app + + def task_imports(self): + return ['slicer_cli_web_singularity.girder_worker_plugin.direct_singularity_run'] diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/commands.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/commands.py new file mode 100644 index 0000000..5c9feab --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/commands.py @@ -0,0 +1,90 @@ +import json +import subprocess + +from girder import logger + +from .utils import generate_image_name_for_singularity, switch_to_sif_image_folder + + +class SingularityCommands: + @staticmethod + def singularity_version(): + """ + This method is used to check whether apptainer is currently installed on the system. + """ + return ['apptainer', '--version'] + + @staticmethod + def singularity_pull(name: str, uri: str = 'docker'): + """ + This method is used to generate the command for the singualrity pull function for pulling + images from online. + Args: + name(str.required) - image name and tag as a single string ':' + uri(str, optional) - image uri (necessary for Dockerhub) + + Returns: + List of strings for singularity subprocess command. + """ + sif_name = generate_image_name_for_singularity(name) + return ['apptainer', 'pull', '--force', sif_name, f'{uri}://{name}'] + + @staticmethod + def get_work_dir(imageName: str): + switch_to_sif_image_folder() + sif_name = generate_image_name_for_singularity(imageName) + cmd = ['apptainer', 'sif', 'dump', '3', sif_name] + label_json = run_command(cmd) + labels = json.loads(label_json) + return labels.get('WorkingDir') + + @staticmethod + def singualrity_run(imageName: str, run_parameters=None, container_args=None): + sif_name = generate_image_name_for_singularity(imageName) + cmd = ['apptainer', 'run', '--no-mount', '/cmsuf'] + if run_parameters: + run_parameters = run_parameters.split(' ') + cmd.extend(run_parameters) + cmd.append(sif_name) + if container_args: + container_args = container_args.split(' ') + cmd.extend(container_args) + return cmd + + @staticmethod + def singularity_get_env(image: str, run_parameters=None): + sif_name = generate_image_name_for_singularity(image) + cmd = ['apptainer', 'exec', '--cleanenv'] + if run_parameters: + run_parameters = run_parameters.split(' ') + cmd.extend(run_parameters) + cmd.append(sif_name) + cmd.append('env') + return cmd + + @staticmethod + def singularity_inspect(imageName, option='-l', json_format=True): + """ + This function is used to get the apptainer command for inspecting the sif file. By default, + it inspects the labels in a json format, but you can you any option allowed by apptainer + by setting the option flag appropriately and also the json flag is set to True by default. + """ + sif_name = generate_image_name_for_singularity(imageName) + cmd = ['apptainer', 'inspect'] + if json_format: + cmd.append('--json') + cmd.append(option) + cmd.append(sif_name) + return cmd + + +def run_command(cmd): + try: + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + if isinstance(res.stdout, bytes): + res = res.stdout.decode('utf-8') + res = res.strip() + return res + except Exception as e: + logger.exception(f'Error occured when running command {cmd} - {e}') + raise Exception(f'Error occured when running command {cmd} - {e}') diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/girder_worker_plugin/direct_singularity_run.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/girder_worker_plugin/direct_singularity_run.py new file mode 100644 index 0000000..5c2ca53 --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/girder_worker_plugin/direct_singularity_run.py @@ -0,0 +1,59 @@ +import os +from uuid import uuid4 + +from girder import logger +from girder_worker.app import app +from girder_worker.docker.io import FDReadStreamConnector +from girder_worker_singularity.tasks import SingularityTask, singularity_run + +from slicer_cli_web.girder_worker_plugin.cli_progress import CLIProgressCLIWriter +from slicer_cli_web.girder_worker_plugin.direct_docker_run import _resolve_direct_file_paths + +from ..commands import SingularityCommands +from ..job import _is_nvidia_img, generate_image_name_for_singularity + + +class DirectSingularityTask(SingularityTask): + def __call__(self, *args, **kwargs): + extra_volumes = _resolve_direct_file_paths(args, kwargs) + if extra_volumes: + volumes = kwargs.setdefault('volumes', []) + if isinstance(volumes, list): + # list mode use + volumes.extend(extra_volumes) + else: + for extra_volume in extra_volumes: + volumes.update(extra_volume._repr_json_()) + super().__call__(*args, **kwargs) + + +@app.task(base=DirectSingularityTask, bind=True) +def run(task, **kwargs): + """Wraps singularity_run to support running singularity containers""" + image = kwargs['image'] + kwargs['image'] = generate_image_name_for_singularity(image) + + pwd = SingularityCommands.get_work_dir(image) + kwargs['pwd'] = pwd + + logs_dir = os.getenv('LOGS') + kwargs['nvidia'] = _is_nvidia_img(image) + + # Cahnge to reflect JOBID for logs later + random_file_name = str(uuid4()) + 'logs.log' + log_file_name = os.path.join(logs_dir, random_file_name) + kwargs['log_file'] = log_file_name + + # Create file since it doesn't exist + if not os.path.exists(log_file_name): + with open(log_file_name, 'x'): + pass + + file_obj = open(log_file_name, 'rb') + if hasattr(task, 'job_manager'): + stream_connectors = kwargs.setdefault('stream_connectors', []) + stream_connectors.append(FDReadStreamConnector( + input=file_obj, + output=CLIProgressCLIWriter(task.job_manager) + )) + return singularity_run(task, **kwargs) diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/job.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/job.py new file mode 100644 index 0000000..34d7441 --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/job.py @@ -0,0 +1,237 @@ +import json +import os +import subprocess + +from girder import logger +from girder_jobs.models.job import Job + +from slicer_cli_web.models import DockerImageError, DockerImageNotFoundError + +from .commands import SingularityCommands, run_command +from .utils import (generate_image_name_for_singularity, sanitize_and_return_json, + switch_to_sif_image_folder) + + +def is_valid_path(path): + """ + Check if the provided path is a valid and accessible path in the file system. + + Parameters: + path (str): The path to check. + + Returns: + bool: True if the path is valid and accessible, False otherwise. + """ + return os.path.exists(path) and os.access(path, os.R_OK) + + +def is_singularity_installed(path=None): + """ + This function is used to check whether singularity is availble on the target system. + This function is useful to make sure that singularity is accessible from a SLURM job submitted + to HiperGator + + Args: + path (str, optional): If the user wants to provide a specific path where singularity is + installed, you can provide that path. Defaults to None. + + Returns: + bool: True if singualrity is successfully accessible on the target system. False otherwise + """ + try: + logger.info('Checking path') + if path and is_valid_path(path): + os.chdir(path) + except Exception: + logger.exception(f'{path} is not a valid path') + raise Exception( + f'{path} is not a valid path' + ) + try: + subprocess.run(SingularityCommands.singularity_version(), check=True) + logger.info('Singularity env available') + except Exception as e: + logger.info(f'Exception {e} occured') + raise e + + +def find_local_singularity_image(name: str, path=''): + """ + Check if the image is present locally on the system in a specified path. For our usecase, we + insall the images to a specific path on /blue directory, which can be modified via the argument + to the function + + Args: + name(str, required) - The name of the docker image with the tag :. + path(str, optional) - This path refers to the path on the local file system designated for + placing singularity images after they are pulled from the interweb. + Returns: + bool: True if singularity image is avaialble on the given path on host system. False otherwise. + + """ + try: + sif_name = generate_image_name_for_singularity(name) + except Exception: + logger.exception("There's an error with the image name. Please check again and try") + raise Exception("There's an error with the image name. Please check again and try") + if not path: + path = os.getenv('SIF_IMAGE_PATH', '') + if not is_valid_path(path): + logger.exception( + 'Please provide a valid path or set the environment variable "SIF_IMAGE_PATH" and' + 'ensure the path has appropriate access') + raise Exception( + 'Please provide a valid path or set the environment variable "SIF_IMAGE_PATH" and' + 'ensure the path has appropriate access') + return os.path.exists(path + sif_name) + + +def pull_image_and_convert_to_sif(names): + """ + This is the function similar to the pullDockerImage function that pulls the image from + Dockerhub or other instances if it's supported in the future + Args: + names(List(str), required) -> The list of image names of the format : + + Raises: + If pulling of any of the images fails, the function raises an error with the list of images + that failed. + """ + failedImageList = [] + for name in names: + try: + logger.info(f'Starting to pull image {name}') + pull_cmd = SingularityCommands.singularity_pull(name) + subprocess.run(pull_cmd, check=True) + except Exception as e: + logger.info(f'Failed to pull image {name}: {e}') + failedImageList.append(name) + if len(failedImageList) != 0: + raise DockerImageNotFoundError('Could not find multiple images ', + image_name=failedImageList) + + +def load_meta_data_for_singularity(job, pullList, loadList, notExistSet): + # flag to indicate an error occurred + errorState = False + images = [] + for name in pullList: + if name not in notExistSet: + job = Job().updateJob( + job, + log=f'Image {name} was pulled successfully \n', + ) + + try: + cli_dict = get_cli_data_for_singularity(name, job) + images.append((name, cli_dict)) + job = Job().updateJob( + job, + log=f'Got pulled image {name} metadata \n' + + ) + except DockerImageError as err: + job = Job().updateJob( + job, + log=f'FAILURE: Error with recently pulled image {name}\n{err}\n', + ) + errorState = True + + for name in loadList: + # create dictionary and load to database + try: + cli_dict = get_cli_data_for_singularity(name, job) + images.append((name, cli_dict)) + job = Job().updateJob( + job, + log=f'Loaded metadata from pre-existing local image {name}\n' + ) + except DockerImageError as err: + job = Job().updateJob( + job, + log=f'FAILURE: Error with recently loading pre-existing image {name}\n{err}\n', + ) + errorState = True + return images, errorState + + +def get_cli_data_for_singularity(name, job): + try: + # We want to mimic the behaviour of docker run : --list_cli in singularity + cli_dict = get_local_singularity_output(name, '--list_cli') + # contains nested dict + # {:{type:}} + if isinstance(cli_dict, bytes): + cli_dict = cli_dict.decode('utf8') + cli_dict = json.loads(cli_dict) + + for key, info in cli_dict.items(): + desc_type = info.get('desc-type', 'xml') + cli_desc = get_local_singularity_output(name, f'{key} --{desc_type}') + + if isinstance(cli_desc, bytes): + cli_desc = cli_desc.decode('utf8') + + cli_dict[key][desc_type] = cli_desc + job = Job().updateJob( + job, + log=f'Got image {name}, cli {key} metadata\n', + ) + return cli_dict + except Exception as err: + logger.exception('Error getting %s cli data from image', name) + raise DockerImageError('Error getting %s cli data from image ' % (name) + str(err)) + + +def _is_nvidia_img(imageName): + switch_to_sif_image_folder() + inspect_labels_cmd = SingularityCommands.singularity_inspect(imageName) + try: + res = run_command(inspect_labels_cmd) + res = sanitize_and_return_json(res) + nvidia = res.get('com.nvidia.volumes.needed', None) + if not nvidia: + return False + return True + except Exception as e: + raise Exception(f'Error occured {e.stderr.decode()}') + + +def get_local_singularity_output(imgName, cmdArg: str): + """ + This function is used to run the singularity command locally for non-resource intensive tasks + such as getting schema, environment variables and so on and return that output to the calling + function + """ + try: + cwd = SingularityCommands.get_work_dir(imgName) + if not cwd: + logger.exception('Please set the entry_path env variable on the Docker Image') + raise Exception('Please set the entry_path env variable on the Docker Image') + run_parameters = f'--cwd {cwd}' + cmd = SingularityCommands.singualrity_run( + imgName, run_parameters=run_parameters, container_args=cmdArg) + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return res.stdout + except Exception as e: + raise Exception(f'error occured {e}') + + +def find_and_remove_local_sif_files(name: str, path=None): + try: + sif_name = generate_image_name_for_singularity(name) + except Exception: + logger.exception("There's an error with the image name. Please check again and try") + raise Exception("There's an error with the image name. Please check again and try") + if not path: + path = os.getenv('SIF_IMAGE_PATH', '') + if not is_valid_path(path): + logger.exception( + 'Please provide a valid path or set the environment variable "SIF_IMAGE_PATH" and' + 'ensure the path has appropriate access') + raise Exception( + 'Please provide a valid path or set the environment variable "SIF_IMAGE_PATH" and' + 'ensure the path has appropriate access') + filename = os.path.join(path, sif_name) + if os.path.exists(filename): + os.remove(filename) diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/singularity_image.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/singularity_image.py new file mode 100644 index 0000000..f96877c --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/singularity_image.py @@ -0,0 +1,253 @@ +############################################################################### +# Copyright Kitware Inc. +# +# Licensed under the Apache License, Version 2.0 ( the "License" ); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +from girder.constants import AccessType +from girder.models.file import File +from girder.models.folder import Folder +from girder.models.item import Item + +from slicer_cli_web.models.docker_image import CLIItem +from slicer_cli_web.models.parser import parse_json_desc, parse_xml_desc, parse_yaml_desc + +from .commands import SingularityCommands, run_command +from .utils import sanitize_and_return_json + + +def _split(name): + """ + :param name: image name + :type name: string + """ + if ':' in name: + return name.split(':') + return name.split('@') + + +class SingularityImage: + """ + This class is used to produce the Singularity equivalent of the Docker Image object as part + of the Python SDK. This helps us to reuse all the functions where Docker is not not directly + involved rather the snapshort of the Docker Image object should suffice to perform the + necessaray operations + """ + + def __init__(self, local_sif_file: str): + self.id = None + self.labels = None + self.short_id = None + self.tags = None + self._set_all_fields(local_sif_file) + + def _set_all_fields(self, local_sif_file: str): + inspect_labels_cmd = SingularityCommands.singularity_inspect(local_sif_file) + try: + res = run_command(inspect_labels_cmd) + # Convert the string labels into json format and only get the labels part from the code + res = sanitize_and_return_json(res) + self.id = res.get('id', '') + self.labels = res + self.tags = res.get('tags', '') + self.short_id = res.get('short_id', '') + except Exception as e: + raise Exception(f'Failed to add metadata from Singularity Image \n {e} \n') + + # A get method to retrieve any label necessary or None, in order to avoid + # errors in certain portions of the code and better emulate Docker Image + # Object behavior + def get(self, label: str): + return self.labels.get(label, None) + + +class SingularityImageItem: + def __init__(self, imageFolder, tagFolder, user): + self.image = imageFolder['name'] + self.tag = tagFolder['name'] + self.name = '%s:%s' % (self.image, self.tag) + self.imageFolder = imageFolder + self.tagFolder = tagFolder + self._id = self.tagFolder['_id'] + self.user = user + self.name = '%s:%s' % (self.image, self.tag) + self.digest = tagFolder['meta'].get('digest', self.name) + + def getCLIs(self): + itemModel = Item() + q = { + 'meta.slicerCLIType': 'task', + 'folderId': self.tagFolder['_id'] + } + if self.user: + items = itemModel.findWithPermissions(q, user=self.user, level=AccessType.READ) + else: + items = itemModel.find(q) + + return [CLIItem(item) for item in items] + + @staticmethod + def find(tagFolderId, user): + folderModel = Folder() + tagFolder = folderModel.load(tagFolderId, user=user, level=AccessType.READ) + if not tagFolder: + return None + imageFolder = folderModel.load(tagFolder['parentId'], user=user, level=AccessType.READ) + return SingularityImageItem(imageFolder, tagFolder, user) + + @staticmethod + def findAllImages(user=None, baseFolder=None): + folderModel = Folder() + q = {'meta.slicerCLIType': 'image'} + if baseFolder: + q['parentId'] = baseFolder['_id'] + + if user: + imageFolders = folderModel.findWithPermissions(q, user=user, level=AccessType.READ) + else: + imageFolders = folderModel.find(q) + + images = [] + + for imageFolder in imageFolders: + qt = { + 'meta.slicerCLIType': 'tag', + 'parentId': imageFolder['_id'] + } + if user: + tagFolders = folderModel.findWithPermissions(qt, user=user, level=AccessType.READ) + else: + tagFolders = folderModel.find(qt) + for tagFolder in tagFolders: + images.append(SingularityImageItem(imageFolder, tagFolder, user)) + return images + + @staticmethod + def removeImages(names, user): + folderModel = Folder() + removed = [] + for name in names: + image, tag = _split(name) + q = { + 'meta.slicerCLIType': 'image', + 'name': image + } + imageFolder = folderModel.findOne(q, user=user, level=AccessType.READ) + if not imageFolder: + continue + qt = { + 'meta.slicerCLIType': 'tag', + 'parentId': imageFolder['_id'], + 'name': tag + } + tagFolder = folderModel.findOne(qt, user=user, level=AccessType.WRITE) + if not tagFolder: + continue + folderModel.remove(tagFolder) + removed.append(name) + + if folderModel.hasAccess(imageFolder, user, AccessType.WRITE) and \ + folderModel.countFolders(imageFolder) == 0: + # clean also empty image folders + folderModel.remove(imageFolder) + + return removed + + @staticmethod + def _create(name, singularity_image_object, user, baseFolder): + folderModel = Folder() + fileModel = File() + + imageName, tagName = _split(name) + + image = folderModel.createFolder(baseFolder, imageName, + 'Slicer CLI generated docker image folder', + creator=user, reuseExisting=True) + folderModel.setMetadata(image, dict(slicerCLIType='image')) + + fileModel.createLinkFile('Docker Hub', image, 'folder', + 'https://hub.docker.com/r/%s' % imageName, + user, reuseExisting=True) + + tag = folderModel.createFolder(image, tagName, + 'Slicer CLI generated docker image tag folder', + creator=user, reuseExisting=True) + + # add docker image labels as meta data + labels = {} + if singularity_image_object.labels: + labels = singularity_image_object.labels.copy() + + if 'description' in labels: + tag['description'] = labels['description'] + del labels['description'] + labels = {k.replace('.', '_'): v for k, v in labels.items()} + labels['digest'] = singularity_image_object.get('digest') + folderModel.setMetadata(tag, labels) + + folderModel.setMetadata(tag, dict(slicerCLIType='tag')) + + return SingularityImageItem(image, tag, user) + + @staticmethod + def _syncItems(image, cli_dict, user): + folderModel = Folder() + itemModel = Item() + + children = folderModel.childItems(image.tagFolder, filters={'meta.slicerCLIType': 'task'}) + existingItems = {item['name']: item for item in children} + + for cli, desc in cli_dict.items(): + item = itemModel.createItem(cli, user, image.tagFolder, + 'Slicer CLI generated CLI command item', + reuseExisting=True) + meta_data = dict(slicerCLIType='task', type=desc.get('type', 'Unknown')) + + # copy some things from the image to be independent + meta_data['image'] = image.name + meta_data['digest'] = image.digest if image.name != image.digest else None + if desc.get('docker-params'): + meta_data['docker-params'] = desc['docker-params'] + + desc_type = desc.get('desc-type', 'xml') + + if desc_type == 'xml': + meta_data.update(parse_xml_desc(item, desc, user)) + elif desc_type == 'yaml': + meta_data.update(parse_yaml_desc(item, desc, user)) + elif desc_type == 'json': + meta_data.update(parse_json_desc(item, desc, user)) + + itemModel.setMetadata(item, meta_data) + + if cli in existingItems: + del existingItems[cli] + + # delete superfluous items + for item in existingItems.values(): + itemModel.remove(item) + + @staticmethod + def saveImage(name, cli_dict, singularity_image_object, user, baseFolder): + """ + :param baseFolder + :type Folder + """ + image = SingularityImageItem._create(name, singularity_image_object, user, baseFolder) + SingularityImageItem._syncItems(image, cli_dict, user) + + return image + + @staticmethod + def prepare(): + Item().ensureIndex(['meta.slicerCLIType', {'sparse': True}]) diff --git a/slicer_cli_web/singularity/slicer_cli_web_singularity/utils.py b/slicer_cli_web/singularity/slicer_cli_web_singularity/utils.py new file mode 100644 index 0000000..bd42f5d --- /dev/null +++ b/slicer_cli_web/singularity/slicer_cli_web_singularity/utils.py @@ -0,0 +1,85 @@ +import json +import os + + +def sanitize_and_return_json(res: str): + """ + This function tries to parse the given str as json in couple different ways. If the output is + still not json or a python dictionary, it raises an error. + """ + try: + res = json.loads(res) + # This is the form in which we get data back if we use the --json label in + # singularity inspect + return res['data']['attributes']['labels'] + except json.decoder.JSONDecodeError: + # If the json label was excluded, we can still parse the labels manually + # and create a dictionary + labels = [line for line in res.split('\n')] + res_dict = {} + for label in labels: + key, value = label.split(': ') + res_dict[key] = value + return res_dict + except Exception as e: + raise Exception(f'Error occured when parsing labels as json {e}') + + +def is_valid_image_name_format(image_str: str): + """ + This function is used to validate whether the user supplied a valid string : as an + argument for functions like singularity pull + + Args: + image_str(str, required) - The string that needs to be validated. + + Returns: + bool - True if the image is in a valid format, False otherwise. + """ + if not image_str: + return False + return True if len(image_str.split(':')) == 2 else False + + +def generate_image_name_for_singularity(image_str: str): + """ + We need to generate the image name for storing the .sif files on the filesystem so that it is + standardized, so it can be referenced in future calls. + + Args: + image_str (str,required) - the image_name in the format : + + Return: + str - A string that is to be used for the .sif filename + """ + if not is_valid_image_name_format(image_str): + raise Exception( + f'Not a valid image name. Please pass the image name in the format {image_str}') + image_str = image_str.replace('/', '_').replace(':', '_') + return f'{image_str}.sif' + + +def switch_to_sif_image_folder(image_path: str = ''): + """ + This function is used to handle Issues that is occuring when Singularity switches directory + when running a plugin and not having the context of where the SIF IMAGES are located for + subsequent image pulls. + This function ensures that Singularity always looks for the plugins in the proper location + + Args: + image_path (str, optional) - This parameter is highly optional and is not recommended unless a + specific use-case arises in the future + + Returns: + None + + Raises: + This function raises an Exception if the SIF_IMAGE_PATH env variable is not set. + + """ + try: + if not image_path: + image_path = os.getenv('SIF_IMAGE_PATH') + os.chdir(image_path + '/') + except Exception: + raise Exception('Please set the SIF_IMAGE_PATH environment variable to locate SIF images')