Merge branch 'release/8.0' into task/streaming-endpoints-public-api

Author: FChataigner

.gitignore

@@ -1,3 +1,3 @@
 *.pyc
 .idea
-.iml
+*.iml

dataikuapi/apinode_client.py

@@ -16,7 +16,8 @@ def __init__(self, uri, service_id, api_key=None):
         """
         DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key)
 
-    def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None):
+    def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None,
+                       with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None):
         """
         Predicts a single record on a DSS API node endpoint (standard or custom prediction)
 
@@ -25,12 +26,24 @@ def predict_record(self, endpoint_id, features, forced_generation=None, dispatch
         :param forced_generation: See documentation about multi-version prediction
         :param dispatch_key: See documentation about multi-version prediction
         :param context: Optional, Python dictionary of additional context information. The context information is logged, but not directly used.
+        :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint.
+        :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint.
+        :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint.
+        :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000).
+         If None, will use the value configured in the endpoint.
 
         :return: a Python dict of the API answer. The answer contains a "result" key (itself a dict)
         """
-        obj =  {
-            "features" :features
+        obj = {
+            "features": features,
+            "explanations": {
+                "enabled": with_explanations,
+                "method": explanation_method,
+                "nExplanations": n_explanations,
+                "nMonteCarloSteps": n_explanations_mc_steps
+            }
         }
+
         if context is not None:
             obj["context"] = context
         if forced_generation is not None:
@@ -40,14 +53,20 @@ def predict_record(self, endpoint_id, features, forced_generation=None, dispatch
 
         return self._perform_json("POST", "%s/predict" % endpoint_id, body = obj)
 
-    def predict_records(self, endpoint_id, records, forced_generation=None, dispatch_key=None):
+    def predict_records(self, endpoint_id, records, forced_generation=None, dispatch_key=None, with_explanations=None,
+                        explanation_method=None, n_explanations=None, n_explanations_mc_steps=None):
         """
         Predicts a batch of records on a DSS API node endpoint (standard or custom prediction)
 
         :param str endpoint_id: Identifier of the endpoint to query
         :param records: Python list of records. Each record must be a Python dict. Each record must contain a "features" dict (see predict_record) and optionally a "context" dict.
         :param forced_generation: See documentation about multi-version prediction
         :param dispatch_key: See documentation about multi-version prediction
+        :param with_explanations: Optional, whether individual explanations should be computed for each record. The prediction endpoint must be compatible. If None, will use the value configured in the endpoint.
+        :param explanation_method: Optional, method to compute explanations. Valid values are 'SHAPLEY' or 'ICE'. If None, will use the value configured in the endpoint.
+        :param n_explanations: Optional, number of explanations to output per prediction. If None, will use the value configured in the endpoint.
+        :param n_explanations_mc_steps: Optional, precision parameter for SHAPLEY method, higher means more precise but slower (between 25 and 1000).
+         If None, will use the value configured in the endpoint.
 
         :return: a Python dict of the API answer. The answer contains a "results" key (which is an array of result objects)
         """
@@ -57,7 +76,13 @@ def predict_records(self, endpoint_id, records, forced_generation=None, dispatch
                 raise ValueError("Each record must contain a 'features' dict")
 
         obj = {
-            "items" : records
+            "items": records,
+            "explanations": {
+                "enabled": with_explanations,
+                "method": explanation_method,
+                "nExplanations": n_explanations,
+                "nMonteCarloSteps": n_explanations_mc_steps
+            }
         }
 
         if forced_generation is not None:

dataikuapi/dss/app.py

@@ -1,4 +1,5 @@
 import sys
+import re
 import os.path as osp
 from .future import DSSFuture
 from dataikuapi.utils import DataikuException
@@ -40,7 +41,8 @@ def create_instance(self, instance_key, instance_name, wait=True):
             return future
 
     def make_random_project_key(self):
-        return "%s_tmp_%s" % (self.app_id, random_string(10))
+        slugified_app_id = re.sub(r'[^A-Za-z_0-9]+', '_', self.app_id)
+        return "%s_tmp_%s" % (slugified_app_id, random_string(10))
 
     def create_temporary_instance(self):
         """
@@ -77,15 +79,17 @@ def get_instance(self, instance_key):
 
     def get_manifest(self):
         raw_data = self.client._perform_json("GET", "/apps/%s/" % self.app_id)
-        return DSSAppManifest(self.client, raw_data)
+        project_key = self.app_id[8:] if self.app_id.startswith('PROJECT_') else None
+        return DSSAppManifest(self.client, raw_data, project_key)
 
 
 class DSSAppManifest(object):
 
-    def __init__(self, client, raw_data):
+    def __init__(self, client, raw_data, project_key=None):
         """The manifest for an app. Do not create this class directly"""
         self.client = client
         self.raw_data = raw_data
+        self.project_key = project_key
 
     def get_raw(self):
         return self.raw_data
@@ -97,6 +101,13 @@ def get_runnable_scenarios(self):
         """Return the scenario identifiers that are declared as actions for this app"""
         return [x["scenarioId"] for x in self.get_all_actions() if x["type"] == "SCENARIO_RUN"]
 
+    def save(self):
+        """Saves the changes to this manifest object back to the template project"""
+        if self.project_key is None:
+            raise Exception("This manifest object wasn't created from a project, cannot be saved back")
+        self.client._perform_empty("PUT", "/projects/%s/app-manifest" % self.project_key, body=self.raw_data)
+
+
 class DSSAppInstance(object):
 
     def __init__(self, client, project_key):
@@ -124,7 +135,7 @@ def __init__(self, client, project_key):
 
 
     def close(self):
-        self.get_as_project().delete()
+        self.get_as_project().delete(drop_data=True)
 
     def __enter__(self,):
         return self

dataikuapi/dss/dataset.py

@@ -3,7 +3,7 @@
 from ..utils import DataikuStreamedHttpUTF8CSVReader
 from .future import DSSFuture
 import json, warnings
-from .utils import DSSTaggableObjectListItem
+from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings
 from .future import DSSFuture
 from .metrics import ComputedMetrics
 from .discussion import DSSObjectDiscussions
@@ -59,6 +59,10 @@ def __init__(self, client, project_key, dataset_name):
         self.project_key = project_key
         self.dataset_name = dataset_name
 
+    @property
+    def name(self):
+        return self.dataset_name
+    
     ########################################################
     # Dataset deletion
     ########################################################
@@ -398,13 +402,31 @@ def create_analysis(self):
         """
         return self.project_create_analysis(self.dataset_name)
 
-    def list_analyses(self):
+    def list_analyses(self, as_type="listitems"):
         """
         List the visual analyses on this dataset
-        :return list of dicts
+        :param str as_type: How to return the list. Supported values are "listitems" and "objects".
+        :returns: The list of the analyses. If "as_type" is "listitems", each one as a dict,
+                  If "as_type" is "objects", each one as a :class:`dataikuapi.dss.analysis.DSSAnalysis` 
+        :rtype: list
         """
-        analysis_list = self.project.list_analyses()
-        return [desc for desc in analysis_list if self.dataset_name == desc.get('inputDataset')]
+        analysis_list = [al for al in self.project.list_analyses() if self.dataset_name == al.get('inputDataset')]
+
+        if as_type == "listitems" or as_type == "listitem":
+            return analysis_list
+        elif as_type == "objects" or as_type == "object":
+            return [self.project.get_analysis(item["analysisId"])for item in analysis_list]
+        else:
+            raise ValueError("Unknown as_type")
+
+    def delete_analyses(self, drop_data=False):
+        """
+        Deletes all analyses that have this dataset as input dataset. Also deletes
+        ML tasks that are part of the analysis
+
+        :param: bool drop_data: whether to drop data for all ML tasks in the analysis
+        """
+        [analysis.delete(drop_data=drop_data) for analysis in self.list_analyses(as_type="objects")]
 
     ########################################################
     # Statistics worksheets
@@ -603,8 +625,9 @@ def new_recipe(self, type, recipe_name=None):
         builder.with_input(self.dataset_name)
         return builder
 
-class DSSDatasetSettings(object):
+class DSSDatasetSettings(DSSTaggableObjectSettings):
     def __init__(self, dataset, settings):
+        super(DSSDatasetSettings, self).__init__(settings)
         self.dataset = dataset
         self.settings = settings
 
@@ -740,4 +763,4 @@ def already_exists(self):
             dataset.get_metadata()
             return True
         except Exception as e:
-            return False
+            return False

dataikuapi/dss/flow.py

@@ -41,7 +41,8 @@ def replace_input_computable(self, current_ref, new_ref, type="DATASET"):
             settings.save()
 
         for recipe in self.project.list_recipes():
-            fake_rap = DSSRecipeDefinitionAndPayload({"recipe" : recipe})
+            recipe_handle = self.project.get_recipe(recipe["name"])
+            fake_rap = DSSRecipeDefinitionAndPayload(recipe_handle, {"recipe" : recipe})
             if fake_rap.has_input(current_ref):
                 logging.info("Recipe %s has %s as input, performing the replacement by %s"% \
                     (recipe["name"], current_ref, new_ref))

dataikuapi/dss/managedfolder.py

@@ -15,6 +15,10 @@ def __init__(self, client, project_key, odb_id):
         self.project_key = project_key
         self.odb_id = odb_id
 
+    @property
+    def id(self):
+        return self.odb_id
+    
     ########################################################
     # Managed folder deletion
     ########################################################