Merge remote-tracking branch 'origin/release/5.1' into release/5.2

Author: cstenac

HISTORY.txt

@@ -1,6 +1,11 @@
 Changelog
 ==========
 
+5.1.0 (2019-03-01)
+------------------
+
+* Initial release for DSS 5.1
+
 5.0.0 (2018-07-26)
 ------------------
 

dataikuapi/dss/admin.py

@@ -728,7 +728,7 @@ def save(self):
             "PUT", "/admin/clusters/%s" % (self.cluster_id), body=self.settings)
 
 class DSSClusterStatus(object):
-    def __init__(self, client, cluster_id, settings):
+    def __init__(self, client, cluster_id, status):
         """Do not call directly, use :meth:`DSSCluster.get_Status`"""
         self.client = client
         self.cluster_id = cluster_id

dataikuapi/dss/apideployer.py

@@ -2,7 +2,8 @@
 from .future import DSSFuture
 
 class DSSAPIDeployer(object):
-    """Handle to interact with the API Deployer.
+    """
+    Handle to interact with the API Deployer.
 
     Do not create this directly, use :meth:`dataikuapi.dss.DSSClient.get_apideployer`
     """
@@ -70,12 +71,29 @@ def list_infras(self, as_objects = True):
         else:
             return l
 
+    def create_infra(self, infra_id, stage, type):
+        """
+        Creates a new infrastructure on the API Deployer and returns the handle to interact with it.
+
+        :param str infra_id: Unique Identifier of the infra to create
+        :param str stage: Infrastructure stage. Stages are configurable on each API Deployer
+        :param str type: STATIC or KUBERNETES
+        :rtype: :class:`DSSAPIDeployerInfra`
+        """
+        settings = {
+            "id": infra_id,
+            "stage": stage,
+            "type": type,
+        }
+        self.client._perform_json("POST", "/api-deployer/infras", body=settings)
+        return self.get_infra(infra_id)
+
     def get_infra(self, infra_id):
         """
         Returns a handle to interact with a single deployment infra, as a :class:`DSSAPIDeployerInfra` 
 
         :param str infra_id: Identifier of the infra to get
-        :rtype: :class:`DSSAPIDeployerDeployment`
+        :rtype: :class:`DSSAPIDeployerInfra`
         """
         return DSSAPIDeployerInfra(self.client, infra_id)
 
@@ -235,7 +253,7 @@ def start_update(self):
 
         :returns: a :class:`dataikuapi.dss.future.DSSFuture` tracking the progress of the update. Call 
                    :meth:`~dataikuapi.dss.future.DSSFuture.wait_for_result` on the returned object
-                to wait for completion (or failure)
+                   to wait for completion (or failure)
         """
         future_response = self.client._perform_json(
             "POST", "/api-deployer/deployments/%s/actions/update" % (self.deployment_id))
@@ -453,4 +471,4 @@ def get_raw(self):
         Gets the raw status information. This returns a dictionary with various information about the service,
         :rtype: dict
         """
-        return self.light_status
+        return self.light_status

dataikuapi/dss/dataset.py

@@ -254,3 +254,51 @@ def get_object_discussions(self):
         :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions`
         """
         return DSSObjectDiscussions(self.client, self.project_key, "DATASET", self.dataset_name)
+
+class DSSManagedDatasetCreationHelper(object):
+
+    def __init__(self, project, dataset_name):
+        self.project = project
+        self.dataset_name = dataset_name
+        self.creation_settings = { "specificSettings" : {} }
+
+    def get_creation_settings(self):
+        return self.creation_settings
+
+    def with_store_into(self, connection, type_option_id = None, format_option_id = None):
+        """
+        Sets the connection into which to store the new managed dataset
+        :param str connection: Name of the connection to store into
+        :param str type_option_id: If the connection accepts several types of datasets, the type
+        :param str format_option_id: Optional identifier of a file format option
+        :return: self
+        """
+        self.creation_settings["connectionId"] = connection
+        if type_option_id is not None:
+            self.creation_settings["typeOptionId"] = type_option_id
+        if format_option_id is not None:
+            self.creation_settings["specificSettings"]["formatOptionId"] = format_option_id 
+        return self
+
+    def with_copy_partitioning_from(self, dataset_ref):
+        """
+        Sets the new managed dataset to use the same partitioning as an existing dataset_name
+
+        :param str dataset_ref: Name of the dataset to copy partitioning from
+        :return: self
+        """
+        self.creation_settings["partitioningOptionId"] = "copy:%s" % dataset_ref
+        return self
+
+    def create(self):
+        """
+        Executes the creation of the managed dataset according to the selected options
+
+        :return: The :class:`DSSDataset` corresponding to the newly created dataset
+        """
+        self.project.client._perform_json("POST", "/projects/%s/datasets/managed" % self.project.project_key,
+            body = {
+                "name": self.dataset_name,
+                "creationSettings":  self.creation_settings
+        })
+        return DSSDataset(self.project.client, self.project.project_key, self.dataset_name)

dataikuapi/dss/plugin.py

@@ -7,6 +7,29 @@
 from .apiservice import DSSAPIService
 import sys
 
+class DSSPluginSettings(object):
+    """
+    The settings of a plugin.
+    """
+
+    def __init__(self, client, plugin_id, settings):
+        """Do not call this directly, use :meth:`DSSPlugin.get_settings`"""
+        self.client = client
+        self.plugin_id = plugin_id
+        self.settings = settings
+
+    def get_raw(self):
+        """Returns the raw settings object"""
+        return self.settings
+
+    def set_code_env(self, code_env_name):
+        """Sets the name of the code env to use for this plugin"""
+        self.settings["codeEnvName"] = code_env_name
+
+    def save(self):
+        """Saves the settings to DSS"""
+        self.client._perform_empty("POST", "/plugins/%s/settings" % (self.plugin_id), body=self.settings)
+
 class DSSPlugin(object):
     """
     A plugin on the DSS instance
@@ -16,65 +39,110 @@ def __init__(self, client, plugin_id):
        self.plugin_id = plugin_id
 
     ########################################################
-    # plugin upload/update as zip
+    # Settings
     ########################################################
 
-    def upload(self, file_path):
+    def get_settings(self):
+        """Return the plugin-level settings
+
+        :return: a :class:`DSSPluginSettings`
         """
-        Upload the given file as a plugin
+        settings = self.client._perform_json("GET", "/plugins/%s/settings" % (self.plugin_id))
+        return DSSPluginSettings(self.client, self.plugin_id, settings)
 
-        Note: this call requires an API key with admin rights
-        
-        :param: file_path : the path to the zip file of the plugin
+    ########################################################
+    # Code env
+    ########################################################
+
+    def create_code_env(self, python_interpreter=None, conda=False):
         """
-        with open(file_path, 'rb') as f:
-            return self.client._perform_json_upload("POST", "/plugins/%s/upload" % (self.plugin_id), 'plugin.zip', f).text
+        Starts the creation of the code env of the plugin
 
-    def update(self, file_path):
+        :return: a :class:`dataikuapi.dssfuture.DSSFuture` 
         """
-        Update the plugin with the given file
+        ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/create" % (self.plugin_id), body={
+            "deploymentMode" : "PLUGIN_MANAGED",
+            "conda": conda,
+            "pythonInterpreter": python_interpreter
+        })
+        return self.client.get_future(ret["jobId"])
 
-        Note: this call requires an API key with admin rights
-        
-        :param: file_path : the path to the zip file of the plugin
+
+    def update_code_env(self):
+        """
+        Starts an update of the code env of the plugin
+
+        :return: a :class:`dataikuapi.dss.future.DSSFuture` 
+        """
+        ret = self.client._perform_json("POST", "/plugins/%s/code-env/actions/update" % (self.plugin_id))
+        return self.client.get_future(ret["jobId"])
+
+
+    ########################################################
+    # Plugin update
+    ########################################################
+
+    def update_from_zip(self, fp):
+        """
+        Updates the plugin from a plugin archive (as a file object)
+
+        :param object fp: A file-like object pointing to a plugin archive zip
+        """
+        files = {'file': fp }
+        self.client._perform_json("POST", "/plugins/%s/actions/updateFromZip" % (self.plugin_id), files=files)
+
+    def update_from_store(self):
+        """
+        Updates the plugin from the Dataiku plugin store
+
+        :return: a :class:`~dataikuapi.dss.future.DSSFuture`
+        """
+        ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromStore" % (self.plugin_id))
+        return self.client.get_future(ret["jobId"])
+
+    def update_from_git(self, repository_url, checkout = "master", subpath=None):
+        """
+        Updates the plugin from a Git repository. DSS must be setup to allow access to the repository.
+
+        :param str repository_url: URL of a Git remote
+        :param str checkout: branch/tag/SHA1 to commit. For example "master"
+        :param str subpath: Optional, path within the repository to use as plugin. Should contain a 'plugin.json' file
+        :return: a :class:`~dataikuapi.dss.future.DSSFuture`
         """
-        with open(file_path, 'rb') as f:
-            return self.client._perform_json_upload("POST", "/plugins/%s/update" % (self.plugin_id), 'plugin.zip', f).text
+        ret = self.client._perform_json("POST", "/plugins/%s/actions/updateFromGit" % (self.plugin_id), body={
+            "gitRepositoryUrl": repository_url,
+            "gitCheckout" : checkout,
+            "gitSubpath": subpath
+        })
+        return self.client.get_future(ret["jobId"])
 
     ########################################################
     # Managing the dev plugin's contents
     ########################################################
 
     def list_files(self):
         """
-        Get the hierarchy of files in the plugin
-        
-        Returns:
-            the plugins's contents
+        Get the hierarchy of files in the plugin (dev plugins only)
         """
         return self.client._perform_json("GET", "/plugins/%s/contents" % (self.plugin_id))
 
     def get_file(self, path):
         """
-        Get a file from the plugin folder
-        
-        Args:
-            path: the name of the file, from the root of the plugin
+        Get a file from the plugin folder (dev plugins only)
 
-        Returns:
-            the file's content, as a stream
+        :param str path: the path of the file, relative to the root of the plugin
+
+        :return: a file-like object containing the file's content
         """
         return self.client._perform_raw("GET", "/plugins/%s/contents/%s" % (self.plugin_id, path)).raw
 
     def put_file(self, path, f):
         """
-        Update a file in the plugin folder
+        Update a file in the plugin folder (dev plugins only)
         
-        Args:
-            f: the file contents, as a stream
-            path: the name of the file, from the root of the plugin
+        :param file-like f: the file contents, as a file-like object
+        :param str path: the path of the file, relative ot the root of the plugin
         """
-
         file_name = path.split('/')[-1]
         data = f.read() # eat it all, because making it work with a path variable and a MultifilePart in swing looks complicated
         return self.client._perform_empty("POST", "/plugins/%s/contents/%s" % (self.plugin_id, path), raw_body=data)

dataikuapi/dss/project.py

@@ -1,5 +1,5 @@
 import time
-from .dataset import DSSDataset
+from .dataset import DSSDataset, DSSManagedDatasetCreationHelper
 from .recipe import DSSRecipe
 from .managedfolder import DSSManagedFolder
 from .savedmodel import DSSSavedModel
@@ -54,6 +54,19 @@ def get_export_stream(self, options = {}):
         Return a stream of the exported project
         You need to close the stream after download. Failure to do so will result in the DSSClient becoming unusable.
 
+        :param dict options: Dictionary of export options. The following options are available:
+
+            * exportUploads (boolean): Exports the data of Uploaded datasets - default False
+            * exportManagedFS (boolean): Exports the data of managed Filesystem datasets - default False
+            * exportAnalysisModels (boolean): Exports the models trained in analysis - default False
+            * exportSavedModels (boolean): Exports the models trained in saved models - default False
+            * exportManagedFolders (boolean): Exports the data of managed folders - default False
+            * exportAllInputDatasets (boolean): Exports the data of all input datasets - default False
+            * exportAllDatasets (boolean): Exports the data of all datasets - default False
+            * exportAllInputManagedFolders (boolean): Exports the data of all input managed folders - default False
+            * exportGitRepositoy (boolean): Exports the Git repository history - default False
+            * exportInsightsData (boolean): Exports the data of static insights - default False
+
         :returns: a file-like obbject that is a stream of the export archive
         :rtype: file-like
         """
@@ -231,6 +244,15 @@ def create_dataset(self, dataset_name, type,
                        body = obj)
         return DSSDataset(self.client, self.project_key, dataset_name)
 
+    def new_managed_dataset_creation_helper(self, dataset_name):
+        """
+        Creates a helper class to create a managed dataset in the project
+
+        :param string dataset_name: Name of the new dataset - must be unique in the project
+        :return: A :class:`dataikuapi.dss.dataset.DSSManagedDatasetCreationHelper` object to create the managed dataset
+        """
+        return DSSManagedDatasetCreationHelper(self, dataset_name)
+
     ########################################################
     # ML
     ########################################################