Merge remote-tracking branch 'origin/release/8.0' into feature/dss80-flow-zones

Author: cstenac

dataikuapi/dss/admin.py

@@ -11,7 +11,7 @@
 from . import recipe
 
 class DSSDatasetListItem(DSSTaggableObjectListItem):
-    """An item in a list of datasets. Do not instantiate this class"""
+    """An item in a list of datasets. Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.list_datasets`"""
     def __init__(self, client, data):
         super(DSSDatasetListItem, self).__init__(data)
         self.client = client
@@ -51,7 +51,7 @@ def get_column(self, column):
 
 class DSSDataset(object):
     """
-    A dataset on the DSS instance
+    A dataset on the DSS instance. Do not instantiate this class, use :meth:`dataikuapi.dss.project.DSSProject.get_dataset`
     """
     def __init__(self, client, project_key, dataset_name):
         self.client = client
@@ -109,9 +109,9 @@ def get_settings(self):
         """
         data = self.client._perform_json("GET", "/projects/%s/datasets/%s" % (self.project_key, self.dataset_name))
 
-        if data["type"] in self.__class__.FS_TYPES:
+        if data["type"] in self.__class__._FS_TYPES:
             return FSLikeDatasetSettings(self, data)
-        elif data["type"] in self.__class__.SQL_TYPES:
+        elif data["type"] in self.__class__._SQL_TYPES:
             return SQLDatasetSettings(self, data)
         else:
             return DSSDatasetSettings(self, data)
@@ -558,33 +558,41 @@ def get_object_discussions(self):
     # Test / Autofill
     ########################################################
 
-    FS_TYPES = ["Filesystem", "UploadedFiles", "FilesInFolder",
+    _FS_TYPES = ["Filesystem", "UploadedFiles", "FilesInFolder",
                 "HDFS", "S3", "Azure", "GCS", "FTP", "SCP", "SFTP"]
     # HTTP is FSLike but not FS                
 
-    SQL_TYPES = ["JDBC", "PostgreSQL", "MySQL", "Vertica", "Snowflake", "Redshift",
+    _SQL_TYPES = ["JDBC", "PostgreSQL", "MySQL", "Vertica", "Snowflake", "Redshift",
                 "Greenplum", "Teradata", "Oracle", "SQLServer", "SAPHANA", "Netezza",
                 "BigQuery", "Athena", "hiveserver2"]
 
     def test_and_detect(self, infer_storage_types=False):
+        """Used internally by autodetect_settings. It is not usually required to call this method"""
         settings = self.get_settings()
 
-        if settings.type in self.__class__.FS_TYPES:
+        if settings.type in self.__class__._FS_TYPES:
             future_resp = self.client._perform_json("POST",
                 "/projects/%s/datasets/%s/actions/testAndDetectSettings/fsLike"% (self.project_key, self.dataset_name),
                 body = {"detectPossibleFormats" : True, "inferStorageTypes" : infer_storage_types })
 
             return DSSFuture(self.client, future_resp.get('jobId', None), future_resp)
-        elif settings.type in self.__class__.SQL_TYPES:
+        elif settings.type in self.__class__._SQL_TYPES:
             return self.client._perform_json("POST",
                 "/projects/%s/datasets/%s/actions/testAndDetectSettings/externalSQL"% (self.project_key, self.dataset_name))
         else:
             raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type)
 
     def autodetect_settings(self, infer_storage_types=False):
+        """
+        Detects appropriate settings for this dataset using Dataiku detection engine
+
+        Returns new suggested settings that you can :meth:`DSSDatasetSettings.save`
+
+        :rtype: :class:`DSSDatasetSettings` or a subclass
+        """
         settings = self.get_settings()
 
-        if settings.type in self.__class__.FS_TYPES:
+        if settings.type in self.__class__._FS_TYPES:
             future = self.test_and_detect(infer_storage_types)
             result = future.wait_for_result()
 
@@ -597,7 +605,7 @@ def autodetect_settings(self, infer_storage_types=False):
 
             return settings
 
-        elif settings.type in self.__class__.SQL_TYPES:
+        elif settings.type in self.__class__._SQL_TYPES:
             result = self.test_and_detect()
 
             if not "schemaDetection" in result:
@@ -610,6 +618,7 @@ def autodetect_settings(self, infer_storage_types=False):
             raise ValueError("don't know how to test/detect on dataset type:%s" % settings.type)
 
     def get_as_core_dataset(self):
+        """Returns the :class:`dataiku.Dataset` object corresponding to this dataset"""
         import dataiku
         return dataiku.Dataset("%s.%s" % (self.project_key, self.dataset_name))
 
@@ -645,6 +654,13 @@ def new_recipe(self, type, recipe_name=None):
         return builder
 
 class DSSDatasetSettings(DSSTaggableObjectSettings):
+    """
+    Base settings class for a DSS dataset.
+    Do not instantiate this class directly, use :meth:`DSSDataset.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
+
     def __init__(self, dataset, settings):
         super(DSSDatasetSettings, self).__init__(settings)
         self.dataset = dataset
@@ -662,6 +678,10 @@ def get_raw_params(self):
     def type(self):
         return self.settings["type"]
 
+    @property
+    def schema_columns(self):
+        return self.settings["schema"]["columns"]
+
     def remove_partitioning(self):
         self.settings["partitioning"] = {"dimensions" : []}
 
@@ -680,6 +700,13 @@ def save(self):
                 body=self.settings)
 
 class FSLikeDatasetSettings(DSSDatasetSettings):
+    """
+    Settings for a files-based dataset. This class inherits from :class:`DSSDatasetSettings`.
+    Do not instantiate this class directly, use :meth:`DSSDataset.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
+
     def __init__(self, dataset, settings):
         super(FSLikeDatasetSettings, self).__init__(dataset, settings)
 
@@ -711,6 +738,12 @@ def set_partitioning_file_pattern(self, pattern):
         self.settings["partitioning"]["filePathPattern"] = pattern
 
 class SQLDatasetSettings(DSSDatasetSettings):
+    """
+    Settings for a SQL dataset. This class inherits from :class:`DSSDatasetSettings`.
+    Do not instantiate this class directly, use :meth:`DSSDataset.get_settings`
+
+    Use :meth:`save` to save your changes
+    """
     def __init__(self, dataset, settings):
         super(SQLDatasetSettings, self).__init__(dataset, settings)
 

dataikuapi/dss/dataset.py

@@ -2,6 +2,7 @@
 from ..utils import DataikuUTF8CSVReader
 from ..utils import DataikuStreamedHttpUTF8CSVReader
 import json
+from requests import utils
 from .metrics import ComputedMetrics
 from .future import DSSFuture
 from .discussion import DSSObjectDiscussions
@@ -82,27 +83,26 @@ def get_file(self, path):
             the file's content, as a stream
         """
         return self.client._perform_raw(
-                "GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path))
+                "GET", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)))
 
     def delete_file(self, path):
         """
         Delete a file from the managed folder
         """
         return self.client._perform_empty(
-                "DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, path))
+                "DELETE", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)))
 
-    def put_file(self, name, f):
+    def put_file(self, path, f):
         """
         Upload the file to the managed folder
         
         Args:
             f: the file contents, as a stream
-            name: the name of the file
+            path: the path of the file
         """
-
         return self.client._perform_json_upload(
-                "POST", "/projects/%s/managedfolders/%s/contents/" % (self.project_key, self.odb_id),
-                name, f)
+                "POST", "/projects/%s/managedfolders/%s/contents/%s" % (self.project_key, self.odb_id, utils.quote(path)),
+                "", f)
 
     ########################################################
     # Managed folder actions

dataikuapi/dss/managedfolder.py

@@ -1,3 +1,5 @@
+import re
+
 from ..utils import DataikuException
 from ..utils import DataikuUTF8CSVReader
 from ..utils import DataikuStreamedHttpUTF8CSVReader
@@ -524,6 +526,22 @@ def save_user_meta(self):
                 "PUT", "/projects/%s/savedmodels/%s/versions/%s/user-meta" % (self.saved_model.project_key,
                     self.saved_model.sm_id, self.saved_model_version), body = um)
 
+    def get_origin_analysis_trained_model(self):
+        """
+        Fetch details about the model in an analysis, this model has been exported from. Returns None if the
+        deployed trained model does not have an origin analysis trained model.
+
+        :rtype: DSSTrainedModelDetails | None
+        """
+        if self.saved_model is None:
+            return self
+        else:
+            fmi = self.get_raw().get("smOrigin", {}).get("fullModelId")
+            if fmi is not None:
+                origin_ml_task = DSSMLTask.from_full_model_id(self.saved_model.client, fmi,
+                                                              project_key=self.saved_model.project_key)
+                return origin_ml_task.get_trained_model_details(fmi)
+
 class DSSTreeNode(object):
     def __init__(self, tree, i):
         self.tree = tree
@@ -1425,6 +1443,17 @@ def get_scatter_plots(self):
 
 
 class DSSMLTask(object):
+
+    @staticmethod
+    def from_full_model_id(client, fmi, project_key=None):
+        match = re.match("^A-(\w+)-(\w+)-(\w+)-(s[0-9]+)-(pp[0-9]+(-part-(\w+)|-base)?)-(m[0-9]+)$", fmi)
+        if match is None:
+            return DataikuException("Invalid model id: {}".format(fmi))
+        else:
+            if project_key is None:
+                project_key = match.group(1)
+            return DSSMLTask(client, project_key, match.group(2), match.group(3))
+
     """A handle to interact with a MLTask for prediction or clustering in a DSS visual analysis"""
     def __init__(self, client, project_key, analysis_id, mltask_id):
         self.client = client
@@ -1686,15 +1715,19 @@ def redeploy_to_flow(self, model_id, recipe_name=None, saved_model_id=None, acti
             "POST", "/projects/%s/models/lab/%s/%s/models/%s/actions/redeployToFlow" % (self.project_key, self.analysis_id, self.mltask_id, model_id),
             body = obj)
 
-    def guess(self, prediction_type=None):
+    def guess(self, prediction_type=None, reguess_level=None):
         """
         Guess the feature handling and the algorithms.
         :param string prediction_type: In case of a prediction problem the prediction type can be specify. Valid values are BINARY_CLASSIFICATION, REGRESSION, MULTICLASS.
+        :param bool reguess_level: One of the following values: TARGET_CHANGE, TARGET_REGUESS and FULL_REGUESS. Only valid for prediction ML Tasks, cannot be specified if prediction_type is also set.
         """
         obj = {}
         if prediction_type is not None:
             obj["predictionType"] = prediction_type
 
+        if reguess_level is not None:
+            obj["reguessLevel"] = reguess_level
+
         self.client._perform_empty(
             "PUT",
             "/projects/%s/models/lab/%s/%s/guess" % (self.project_key, self.analysis_id, self.mltask_id),