Merge branch 'master' into feature/mlflow-experiment-tracking

Author: Valentin Thorey

dataikuapi/dss/admin.py

@@ -695,13 +695,18 @@ def set_definition(self, env):
 
         * env.permissions, env.usableByAll, env.desc.owner
         * env.specCondaEnvironment, env.specPackageList, env.externalCondaEnvName, env.desc.installCorePackages,
-          env.desc.installJupyterSupport, env.desc.yarnPythonBin
+          env.desc.corePackagesSet, env.desc.installJupyterSupport, env.desc.yarnPythonBin, env.desc.yarnRBin
+          env.desc.envSettings, env.desc.allContainerConfs, env.desc.containerConfs, 
+          env.desc.allSparkKubernetesConfs, env.desc.sparkKubernetesConfs
 
         Fields that can be updated in automation node (where {version} is the updated version):
 
-        * env.permissions, env.usableByAll, env.owner
+        * env.permissions, env.usableByAll, env.owner, env.envSettings
         * env.{version}.specCondaEnvironment, env.{version}.specPackageList, env.{version}.externalCondaEnvName, 
-          env.{version}.desc.installCorePackages, env.{version}.desc.installJupyterSupport, env.{version}.desc.yarnPythonBin
+          env.{version}.desc.installCorePackages, env.{version}.corePackagesSet, env.{version}.desc.installJupyterSupport
+          env.{version}.desc.yarnPythonBin, env.{version}.desc.yarnRBin, env.{version}.desc.allContainerConfs, 
+          env.{version}.desc.containerConfs, env.{version}.desc.allSparkKubernetesConfs, 
+          env.{version}.{version}.desc.sparkKubernetesConfs
 
         Note: this call requires an API key with admin rights
         
@@ -722,6 +727,39 @@ def get_version_for_project(self, project_key):
         return self.client._perform_json(
             "GET", "/admin/code-envs/%s/%s/%s/version" % (self.env_lang, self.env_name, project_key))
 
+
+    def get_settings(self):
+        """
+        Returns the settings of this code env as a :class:`DSSCodeEnvSettings`, or one of its subclasses.
+
+        Known subclasses of :class:`DSSCodeEnvSettings` include :class:`DSSDesignCodeEnvSettings` 
+        and :class:`DSSAutomationCodeEnvSettings`
+
+        You must use :meth:`~DSSCodeEnvSettings.save()` on the returned object to make your changes effective
+        on the code env.
+
+        .. code-block:: python
+
+            # Example: setting the required packagd
+            codeenv = client.get_code_env("PYTHON", "code_env_name")
+            settings = codeenv.get_settings()
+            settings.set_required_packages("dash==2.0.0", "bokeh<2.0")
+            settings.save()
+            # then proceed to update_packages()
+
+        :rtype: :class:`DSSCodeEnvSettings`
+        """
+        data = self.client._perform_json(
+            "GET", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name))
+
+        # you can't just use deploymentMode to check if it's an automation code
+        # env, because some modes are common to both types of nodes. So we rely 
+        # on a non-null field that only the automation code envs have
+        if data.get("versions", None) is not None:
+            return DSSAutomationCodeEnvSettings(self, data)
+        else:
+            return DSSDesignCodeEnvSettings(self, data)
+
 
     ########################################################
     # Code env actions
@@ -806,6 +844,171 @@ def get_log(self, log_name):
             "GET", "/admin/code-envs/%s/%s/logs/%s" % (self.env_lang, self.env_name, log_name))
 
 
+class DSSCodeEnvSettings(object):
+    """
+    Base settings class for a DSS code env.
+    Do not instantiate this class directly, use :meth:`DSSCodeEnv.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
+
+    def __init__(self, codeenv, settings):
+        self.codeenv = codeenv
+        self.settings = settings
+
+    def get_raw(self):
+        """Get the raw code env settings as a dict"""
+        return self.settings
+
+    @property
+    def env_lang(self):
+        return self.codeenv.env_lang
+
+    @property
+    def env_name(self):
+        return self.codeenv.env_name
+
+    def save(self):
+        self.codeenv.client._perform_json(
+            "PUT", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name), body=self.settings)
+
+class DSSCodeEnvPackageListBearer(object):
+    def get_required_packages(self, as_list=False):
+        """
+        Return the list of required packages, as a single string
+
+        :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string
+        """
+        x = self.settings.get("specPackageList", "")
+        return x.split('\n') if as_list else x
+    def set_required_packages(self, *packages):
+        """
+        Set the list of required packages
+        """
+        self.settings["specPackageList"] = '\n'.join(packages)
+
+    def get_required_conda_spec(self, as_list=False):
+        """
+        Return the list of required conda packages, as a single string
+
+        :param boolean as_list: if True, return the spec as a list of lines; if False, return as a single multiline string
+        """
+        x = self.settings.get("specCondaEnvironment", "")
+        return x.split('\n') if as_list else x
+    def set_required_conda_spec(self, *spec):
+        """
+        Set the list of required conda packages
+        """
+        self.settings["specCondaEnvironment"] = '\n'.join(packages)
+
+class DSSCodeEnvContainerConfsBearer(object):
+    def get_built_for_all_container_confs(self):
+        """
+        Return whether the code env creates an image for each container config
+        """
+        return self.settings.get("allContainerConfs", False)
+    def get_built_container_confs(self):
+        """
+        Return the list of container configs for which the code env builds an image (if not all)
+        """
+        return self.settings.get("containerConfs", [])
+    def set_built_container_confs(self, *configs, **kwargs):
+        """
+        Set the list of container configs for which the code env builds an image
+
+        :param boolean all: if True, an image is built for each config
+        :param list configs: list of configuration names to build images for
+        """
+        all = kwargs.get("all", False)
+        self.settings['allContainerConfs'] = all
+        if not all:
+            self.settings['containerConfs'] = configs
+    def built_for_all_spark_kubernetes_confs(self):
+        """
+        Return whether the code env creates an image for each managed Spark over Kubernetes config
+        """
+        return self.settings.get("allSparkKubernetesConfs", False)
+    def get_built_spark_kubernetes_confs(self):
+        """
+        Return the list of managed Spark over Kubernetes configs for which the code env builds an image (if not all)
+        """
+        return self.settings.get("sparkKubernetesConfs", [])
+    def set_built_spark_kubernetes_confs(self, *configs, **kwargs):
+        """
+        Set the list of managed Spark over Kubernetes configs for which the code env builds an image
+
+        :param boolean all: if True, an image is built for each config
+        :param list configs: list of configuration names to build images for
+        """
+        all = kwargs.get("all", False)
+        self.settings['allSparkKubernetesConfs'] = all
+        if not all:
+            self.settings['sparkKubernetesConfs'] = configs
+
+
+class DSSDesignCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvPackageListBearer, DSSCodeEnvContainerConfsBearer):
+    """
+    Base settings class for a DSS code env on a design node.
+    Do not instantiate this class directly, use :meth:`DSSCodeEnv.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
+
+    def __init__(self, codeenv, settings):
+        super(DSSDesignCodeEnvSettings, self).__init__(codeenv, settings)
+
+
+class DSSAutomationCodeEnvSettings(DSSCodeEnvSettings, DSSCodeEnvContainerConfsBearer):
+    """
+    Base settings class for a DSS code env on an automation node.
+    Do not instantiate this class directly, use :meth:`DSSCodeEnv.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
+
+    def __init__(self, codeenv, settings):
+        super(DSSAutomationCodeEnvSettings, self).__init__(codeenv, settings)
+
+
+    def get_version(self, version_id=None):
+        """
+        Get a specific code env version (for versioned envs) or the single
+        version
+
+        :param string version_id: for versioned code env, identifier of the desired version 
+
+        :rtype: :class:`DSSAutomationCodeEnvVersionSettings`
+        """
+        deployment_mode = self.settings.get("deploymentMode", None)
+        if deployment_mode in ['AUTOMATION_SINGLE']:
+            return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('currentVersion', {}))
+        elif deployment_mode in ['AUTOMATION_VERSIONED']:
+            versions = self.settings.get("versions", [])
+            version_ids = [v.get('versionId') for v in versions]
+            if version_id is None:
+                raise Exception("A version id is required in a versioned code env. Existing ids: %s" % ', '.join(version_ids))
+            for version in versions:
+                if version_id == version.get("versionId"):
+                    return DSSAutomationCodeEnvVersionSettings(self.codeenv, version)
+            raise Exception("Version %s not found in : %s" % (version_id, ', '.join(version_ids)))
+        elif deployment_mode in ['PLUGIN_NON_MANAGED', 'PLUGIN_MANAGED', 'AUTOMATION_NON_MANAGED_PATH', 'EXTERNAL_CONDA_NAMED']:
+            return DSSAutomationCodeEnvVersionSettings(self.codeenv, self.settings.get('noVersion', {}))
+        else:
+            raise Exception("Unexpected deployment mode %s for an automation node code env. Alter the settings directly with get_raw()", deployment_mode)
+
+class DSSAutomationCodeEnvVersionSettings(DSSCodeEnvPackageListBearer):
+    """
+    Base settings class for a DSS code env version on an automation node.
+    Do not instantiate this class directly, use :meth:`DSSAutomationCodeEnvSettings.get_version`
+
+    Use :meth:`save` on the :class:`DSSAutomationCodeEnvSettings` to save your changes
+    """
+
+    def __init__(self, codeenv_settings, version_settings):
+        self.codeenv_settings = codeenv_settings
+        self.settings = version_settings
+
+
 class DSSGlobalApiKey(object):
     """
     A global API key on the DSS instance
@@ -1004,3 +1207,65 @@ def get_raw(self):
         Gets the whole status as a raw dictionary.
         """
         return self.status
+
+
+class DSSInstanceVariables(dict):
+    """
+    Dict containing the instance variables. The variables can be modified directly in the dict and persisted using its :meth:`save` method.
+
+    Do not create this directly, use :meth:`dataikuapi.DSSClient.get_global_variables`
+    """
+
+    def __init__(self, client, variables):
+        super(dict, self).__init__()
+        self.update(variables)
+        self.client = client
+
+    def save(self):
+        """
+        Save the changes made to the instance variables.
+
+        Note: this call requires an API key with admin rights.
+        """
+        return self.client._perform_empty("PUT", "/admin/variables/", body=self)
+
+
+class DSSGlobalUsageSummary(object):
+    """
+    The summary of the usage of the DSS instance.
+    Do not create this directly, use :meth:`dataikuapi.dss.DSSClient.get_global_usage_summary`
+    """
+    def __init__(self, data):
+        self.data = data
+
+    @property
+    def raw(self):
+        return self.data
+
+    @property
+    def projects_count(self):
+        return self.data["projects"]
+
+    @property
+    def total_datasets_count(self):
+        return self.data["datasets"]["all"]
+
+    @property
+    def total_recipes_count(self):
+        return self.data["recipes"]["all"]
+
+    @property
+    def total_jupyter_notebooks_count(self):
+        return self.data["notebooks"]["nbJupyterNotebooks"]
+
+    @property
+    def total_sql_notebooks_count(self):
+        return self.data["notebooks"]["nbSqlNotebooks"]
+
+    @property
+    def total_scenarios_count(self):
+        return self.data["scenarios"]["all"]
+
+    @property
+    def total_active_with_trigger_scenarios_count(self):
+        return self.data["scenarios"]["activeWithTriggers"]

dataikuapi/dss/app.py

@@ -135,7 +135,7 @@ def __init__(self, client, project_key):
         DSSAppInstance.__init__(self, client,project_key)
 
     def close(self):
-        self.get_as_project().delete(drop_data=True)
+        self.get_as_project().delete(clear_managed_datasets=True)
 
     def __enter__(self,):
         return self

dataikuapi/dss/flow.py

@@ -338,6 +338,22 @@ def add_item(self, obj):
         self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/items" % (self.flow.project.project_key, self.id),
                                   body=self.flow._to_smart_ref(obj))
 
+    def add_items(self, items):
+        """
+        Adds items to this zone.
+
+        The items will automatically be moved from their existing zones. Additional items may be moved to this zone
+        as a result of the operations (notably the recipe generating the `items`).
+
+        :param list items: A list of objects, either :class:`dataikuapi.dss.dataset.DSSDataset`, :class:`dataikuapi.dss.managedfolder.DSSManagedFolder`,
+                           or :class:`dataikuapi.dss.savedmodel.DSSSavedModel` to add to the zone
+        """
+        smart_refs = []
+        for item in items:
+            smart_refs.append(self.flow._to_smart_ref(item))
+        self._raw = self.client._perform_json("POST", "/projects/%s/flow/zones/%s/add-items" % (self.flow.project.project_key, self.id),
+                                  body=smart_refs)
+
     @property
     def items(self):
         """

dataikuapi/dss/managedfolder.py

@@ -111,18 +111,19 @@ def put_file(self, path, f):
 
     def upload_folder(self, path, folder):
         """
-        Upload folder and its content as path in the managed folder.
+        Upload the content of a folder at path in the managed folder.
+
+        Note: upload_folder("target", "source") will result in "target" containing the content
+        of "source", not in "target" containing "source".
 
         :param str path: the destination path of the folder in the managed folder
         :param str folder: path  (absolute or relative) of the source folder to upload
         """
-        real_root = os.path.realpath(folder)
-        for root, _, files in os.walk(real_root):
+        for root, _, files in os.walk(folder):
             for file in files:
                 filename = os.path.join(root, file)
-                relpath = os.path.relpath(filename, real_root)
                 with open(filename, "rb") as f:
-                    self.put_file(os.path.join(path, relpath), f)
+                    self.put_file(os.path.join(path, os.path.relpath(filename, folder)), f)
 
     ########################################################
     # Managed folder actions

dataikuapi/dss/project.py

@@ -83,17 +83,25 @@ def move_to_folder(self, folder):
     # Project deletion
     ########################################################
 
-    def delete(self, drop_data=False):
+    def delete(self, clear_managed_datasets=False, clear_output_managed_folders=False, clear_job_and_scenario_logs=True, **kwargs):
         """
         Delete the project
 
         This call requires an API key with admin rights
 
-        :param bool drop_data: Should the data of managed datasets be dropped
+        :param bool clear_managed_datasets: Should the data of managed datasets be cleared
+        :param bool clear_output_managed_folders: Should the data of managed folders used as outputs of recipes be cleared
+        :param bool clear_job_and_scenario_logs: Should the job and scenario logs be cleared
         """
+        # For backwards compatibility
+        if 'drop_data' in kwargs and kwargs['drop_data']:
+            clear_managed_datasets = True
+
         return self.client._perform_empty(
-            "DELETE", "/projects/%s" % self.project_key, params = {
-                "dropData": drop_data
+            "DELETE", "/projects/%s" % self.project_key, params={
+                "clearManagedDatasets": clear_managed_datasets,
+                "clearOutputManagedFolders": clear_output_managed_folders,
+                "clearJobAndScenarioLogs": clear_job_and_scenario_logs
             })
 
     ########################################################
@@ -721,8 +729,6 @@ def create_mlflow_pyfunc_model(self, name, prediction_type = None):
         :param string name: Human readable name for the new saved model in the flow
         :param string prediction_type: Optional (but needed for most operations). One of BINARY_CLASSIFICATION, MULTICLASS or REGRESSION
         """
-        if not name:
-            raise ValueError("name can not be empty")
         model = {
             "savedModelType" : "MLFLOW_PYFUNC",
             "predictionType" : prediction_type,