Merge remote-tracking branch 'origin/master' into feature/new-dataset-management-apis-patch + cleanup

Author: cstenac

dataikuapi/dss/dataset.py

@@ -3,7 +3,7 @@
 from ..utils import DataikuStreamedHttpUTF8CSVReader
 from .future import DSSFuture
 import json, warnings
-from .utils import DSSTaggableObjectListItem
+from .utils import DSSTaggableObjectListItem, DSSTaggableObjectSettings
 from .future import DSSFuture
 from .metrics import ComputedMetrics
 from .discussion import DSSObjectDiscussions
@@ -402,13 +402,31 @@ def create_analysis(self):
         """
         return self.project_create_analysis(self.dataset_name)
 
-    def list_analyses(self):
+    def list_analyses(self, as_type="listitems"):
         """
         List the visual analyses on this dataset
-        :return list of dicts
+        :param str as_type: How to return the list. Supported values are "listitems" and "objects".
+        :returns: The list of the analyses. If "as_type" is "listitems", each one as a dict,
+                  If "as_type" is "objects", each one as a :class:`dataikuapi.dss.analysis.DSSAnalysis` 
+        :rtype: list
         """
-        analysis_list = self.project.list_analyses()
-        return [desc for desc in analysis_list if self.dataset_name == desc.get('inputDataset')]
+        analysis_list = [al for al in self.project.list_analyses() if self.dataset_name == al.get('inputDataset')]
+
+        if as_type == "listitems" or as_type == "listitem":
+            return analysis_list
+        elif as_type == "objects" or as_type == "object":
+            return [self.project.get_analysis(item["analysisId"])for item in analysis_list]
+        else:
+            raise ValueError("Unknown as_type")
+
+    def delete_analyses(self, drop_data=False):
+        """
+        Deletes all analyses that have this dataset as input dataset. Also deletes
+        ML tasks that are part of the analysis
+
+        :param: bool drop_data: whether to drop data for all ML tasks in the analysis
+        """
+        [analysis.delete(drop_data=drop_data) for analysis in self.list_analyses(as_type="objects")]
 
     ########################################################
     # Statistics worksheets
@@ -607,36 +625,9 @@ def new_recipe(self, type, recipe_name=None):
         builder.with_input(self.dataset_name)
         return builder
 
-    ########################################################
-    # Creation of analyses
-    ########################################################
-
-    def new_analysis(self):
-        analysis = self.project.create_analysis(self.name)
-        return analysis
-
-    def list_analyses(self):
-        """Returns a list of json short description description of analysis that has this dataset as inputDataset
-
-        :return: list of dict with keys {'analysisId', 'analysisName', 'inputDataset'}
-
-        """
-        project_analysis_desc_list = self.project.list_analyses()
-        return [desc for desc in project_analysis_desc_list if self.name == desc.get('inputDataset')]
-
-    def delete_analyses(self, drop_data=False):
-        """Deletes all analyses that have this dataset as inputDataset
-
-        :param: bool drop_data: will drop analysis data if True. Default is False
-        """
-
-        desc_list = self.list_analyses()
-        dss_analysis_list = [self.project.get_analysis(desc['analysisId']) for desc in desc_list]
-        return [analysis.delete(drop_data=drop_data) for analysis in dss_analysis_list]
-
-
-class DSSDatasetSettings(object):
+class DSSDatasetSettings(DSSTaggableObjectSettings):
     def __init__(self, dataset, settings):
+        super(DSSDatasetSettings, self).__init__(settings)
         self.dataset = dataset
         self.settings = settings
 

dataikuapi/dss/flow.py

@@ -41,7 +41,8 @@ def replace_input_computable(self, current_ref, new_ref, type="DATASET"):
             settings.save()
 
         for recipe in self.project.list_recipes():
-            fake_rap = DSSRecipeDefinitionAndPayload({"recipe" : recipe})
+            recipe_handle = self.project.get_recipe(recipe["name"])
+            fake_rap = DSSRecipeDefinitionAndPayload(recipe_handle, {"recipe" : recipe})
             if fake_rap.has_input(current_ref):
                 logging.info("Recipe %s has %s as input, performing the replacement by %s"% \
                     (recipe["name"], current_ref, new_ref))

dataikuapi/dss/ml.py

@@ -125,13 +125,13 @@ def set_split_explicit(self, train_selection, test_selection, dataset_name=None,
 
         return self
 
-    def set_order_by(self, feature_name, ascending=True):
+    def set_time_ordering(self, feature_name, ascending=True):
         """
-        Uses a variable to sort the data for train/test split and hyperparameter optimization
+        Uses a variable to sort the data for train/test split and hyperparameter optimization by time
         :param str feature_name: Name of the variable to use
         :param bool ascending: True iff the test set is expected to have larger time values than the train set
         """
-        self.unset_order_by()
+        self.unset_time_ordering()
         if not feature_name in self.mltask_settings["preprocessing"]["per_feature"]:
             raise ValueError("Feature %s doesn't exist in this ML task, can't use as time" % feature_name)
         self.mltask_settings['time']['enabled'] = True
@@ -148,9 +148,9 @@ def set_order_by(self, feature_name, ascending=True):
 
         return self
 
-    def unset_order_by(self):
+    def unset_time_ordering(self):
         """
-        Remove time-based ordering.
+        Remove time-based ordering for train/test split and hyperparameter optimization
         """
         self.mltask_settings['time']['enabled'] = False
         self.mltask_settings['time']['timeVariable'] = None
@@ -187,28 +187,6 @@ def get_raw(self):
         """
         return self.mltask_settings
 
-    def get_split_params(self):
-        """
-        Gets a handle to modify train/test splitting params.
-
-        :rtype: :class:`PredictionSplitParamsHandler`
-        """
-        raise NotImplementedError("get_split_params not available for class {}".format(self.__class__))
-
-    def split_ordered_by(self, feature_name, ascending=True):
-        """
-        Uses a variable to sort the data for train/test split and hyperparameter optimization
-        :param str feature_name: Name of the variable to use
-        :param bool ascending: True iff the test set is expected to have larger time values than the train set
-        """
-        raise NotImplementedError("split_ordered_by not available for class {}".format(self.__class__))
-
-    def remove_ordered_split(self):
-        """
-        Remove time-based ordering.
-        """
-        raise NotImplementedError("remove_ordered_split not available for class {}".format(self.__class__))
-
     def get_feature_preprocessing(self, feature_name):
         """
         Gets the feature preprocessing params for a particular feature. This returns a reference to the
@@ -249,18 +227,6 @@ def use_feature(self, feature_name):
         """
         self.get_feature_preprocessing(feature_name)["role"] = "INPUT"
 
-    def use_sample_weighting(self, feature_name):
-        """
-        Deprecated. Will be removed from DSSMLTaskSettings class
-        """
-        raise NotImplementedError("use_sample_weighting() not available for class {}".format(self.__class__))
-
-    def remove_sample_weighting(self):
-        """
-        Deprecated. Will be removed from DSSMLTaskSettings class
-        """
-        raise NotImplementedError("remove_sample_weighting() not available for class {}".format(self.__class__))
- 
     def get_algorithm_settings(self, algorithm_name):
         """
         Gets the training settings for a particular algorithm. This returns a reference to the
@@ -419,31 +385,21 @@ def get_split_params(self):
         """
         return PredictionSplitParamsHandler(self.mltask_settings)
 
-    @split_params.setter
-    def split_params(self, value):
-        raise AttributeError("split_params reference cannot be overwritten, get a handle and modify it with a set method instead")
-
     def split_ordered_by(self, feature_name, ascending=True):
         """
-        Uses a variable to sort the data for train/test split and hyperparameter optimization
-        :param str feature_name: Name of the variable to use
-        :param bool ascending: True iff the test set is expected to have larger time values than the train set
-
-        :rtype: self
+        Deprecated. Use split_params.set_time_ordering()
         """
-        warnings.warn("split_ordered_by() is deprecated, please use split_params.set_order_by() instead", DeprecationWarning)
-        self.split_params.set_order_by(feature_name, ascending=True)
+        warnings.warn("split_ordered_by() is deprecated, please use split_params.set_time_ordering() instead", DeprecationWarning)
+        self.split_params.set_time_ordering(feature_name, ascending=ascending)
 
         return self
 
     def remove_ordered_split(self):
         """
-        Remove time-based ordering.
-
-        :rtype: self
+        Deprecated. Use split_params.unset_time_ordering()
         """
-        warnings.warn("remove_ordered_split() is deprecated, please use split_params.unset_order_by() instead", DeprecationWarning)
-        self.split_params.unset_order_by()
+        warnings.warn("remove_ordered_split() is deprecated, please use split_params.unset_time_ordering() instead", DeprecationWarning)
+        self.split_params.unset_time_ordering()
 
         return self
 
@@ -456,8 +412,10 @@ def use_sample_weighting(self, feature_name):
 
     def set_weighting(self, method, feature_name=None):
         """
-        Uses a feature as sample weight
-        :param str feature_name: Name of the feature to use
+        Sets the method to weight samples. 
+        :param str method: Method to use. One of NO_WEIGHTING, SAMPLE_WEIGHT (must give a feature name), 
+                        CLASS_WEIGHT or CLASS_AND_SAMPLE_WEIGHT (must give a feature name)
+        :param str feature_name: Name of the feature to use as sample weight
         """
         self.unset_weighting()