add some helpers for the analysis api

Author: FChataigner

dataikuapi/dss/analysis.py

@@ -5,6 +5,99 @@
 import time
 from .metrics import ComputedMetrics
 from .ml import DSSMLTask
+from .utils import DSSDatasetSelectionBuilder
+
+class DSSAnalysisStepBuilder(object):
+    def __init__(self, step_type=None, step_name=None):
+        self.step = {'metaType':'PROCESSOR', 'type':step_type, 'name':step_name, 'params':{}}
+
+    def build(self):
+        """Returns the built step dict"""
+        return self.step
+
+    def with_type(self, step_type):
+        """Sets the step's type"""
+        self.step["type"] = step_type
+        return self
+
+    def with_name(self, step_name):
+        """Sets the step's name"""
+        self.step["name"] = step_name
+        return self
+
+
+class DSSAnalysisDefinition():
+    """
+    Object to manipulate the definition of a visual analysis
+    """
+
+    def __init__(self, analysis, acp):
+    	self.analysis = analysis
+        self.acp = acp
+
+    def get_raw(self):
+        """
+        Gets the raw dictionary of the visual analysis definition
+        """
+        return self.acp
+
+    def get_raw_script(self):
+        """
+        Gets the raw dictionary of visual analysis' script settings (including steps, sampling, ...)
+        """
+        acp = self.get_raw()
+        if not 'script' in acp:
+        	acp['script'] = {'steps':[]}
+        return acp['script']
+
+    def get_raw_script_steps(self):
+        """
+        Gets the raw dictionary of visual analysis' script steps
+        """
+        script = self.get_raw_script()
+        if not 'steps' in script:
+        	script['steps'] = []
+        return script['steps']
+
+    def get_raw_script_sampling(self):
+        """
+        Gets the raw dictionary of visual analysis' script sampling
+        """
+        script = self.get_raw_script()
+        if not 'explorationSampling' in script:
+        	script['explorationSampling'] = {}
+        return script['explorationSampling']
+
+    def save(self):
+    	"""
+    	Shortcut to :meth:`DSSAnalysis.set_definition()`
+    	"""
+    	self.analysis.set_definition(self)
+
+    def add_step(self, step):
+    	"""
+    	Add a step to the script
+
+        :param object selection: A :class:`DSSAnalysisStepBuilder` to build the settings of the step.
+    	"""
+        steps = self.get_raw_script_steps()
+        if isinstance(step, DSSAnalysisStepBuilder):
+        	steps.append(step.build())
+        else:
+        	steps.append(step)
+
+    def set_script_sampling_selection(self, selection):    
+        """
+        Sets the sampling for the script
+
+        :param object selection: A :class:`DSSDatasetSelectionBuilder` to build the settings of the extract of the dataset.
+        """
+        sampling = self.get_raw_script_sampling()
+        if isinstance(selection, DSSDatasetSelectionBuilder):
+            sampling['selection'] = selection.build()
+        else:
+            sampling['selection'] = selection
+
 
 class DSSAnalysis(object):
     """A handle to interact with a DSS visual analysis"""
@@ -34,20 +127,25 @@ def get_definition(self):
         """
         Get the definition of the analysis
 
-        Returns:
-            the definition, as a JSON object
+        :return: a DSSAnalysisDefinition object to interact with the settings
+        :rtype: :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition`
         """
-        return self.client._perform_json("GET", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id))
+        acp = self.client._perform_json("GET", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id))
+        return DSSAnalysisDefinition(self, acp)
 
     def set_definition(self, definition):
         """
         Set the definition of the analysis
         
         Args:
-            definition: the definition, as a JSON object. You should only set a definition object 
-            that has been retrieved using the get_definition call.
+            definition: the definition, as a JSON object or a :class:`dataikuapi.dss.analysis.DSSAnalysisDefinition`.
+            You should only set a definition object that has been retrieved using the get_definition call.
         """
-        return self.client._perform_json("PUT", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id), body=definition)
+        if isinstance(definition, DSSAnalysisDefinition):
+        	acp = definition.get_raw()
+        else:
+        	acp = definition
+        return self.client._perform_json("PUT", "/projects/%s/lab/%s/" % (self.project_key, self.analysis_id), body=acp)
 
 
     ########################################################
@@ -134,3 +232,119 @@ def get_ml_task(self, mltask_id):
         """
         return DSSMLTask(self.client, self.project_key, self.analysis_id, mltask_id)
 
+
+# some basic steps
+class DSSFormulaStepBuilder(DSSAnalysisStepBuilder):
+    def __init__(self, step_name=None):
+    	super(DSSFormulaStepBuilder, self).__init__(step_type='CreateColumnWithGREL', step_name=step_name)
+
+    def with_output_column(self, column_name):
+        """Sets the step's output column's name"""
+        self.step["params"]["column"] = column_name
+        return self
+
+    def with_error_column(self, column_name):
+        """Sets the step's output column's name"""
+        self.step["params"]["errorColumn"] = column_name
+        return self
+
+    def with_expression(self, expression):
+        """Sets the step's expression"""
+        self.step["params"]["expression"] = expression
+        return self
+
+class AppliesToStepBuilder(DSSAnalysisStepBuilder):
+    def __init__(self, step_type=None, step_name=None):
+    	super(AppliesToStepBuilder, self).__init__(step_type=step_type, step_name=step_name)
+    	self.step["params"]["appliesTo"] = 'SINGLE_COLUMN'
+
+    def with_column_selection_mode(self, column_selection_mode):
+        """Sets the step's column selection mode (SINGLE_COLUMN, COLUMNS, PATTERN, ALL)"""
+        self.step["params"]["appliesTo"] = column_selection_mode
+        return self
+
+    def with_columns(self, *column_names):
+        """Sets the step's selected columns"""
+        self.step["params"]["columns"] = [c for c in column_names]
+        return self
+
+    def with_column_regex(self, regex):
+        """Sets the step's column selection regular expression"""
+        self.step["params"]["appliesToPattern"] = regex
+        return self
+
+    def with_single_column_selection(self, column_name):
+        """Sets the step's as applying to a single column"""
+        return self.with_column_selection_mode('SINGLE_COLUMN').with_columns(column_name)
+
+    def with_multiple_column_selection(self, *column_names):
+        """Sets the step's as applying to a single column"""
+        return self.with_column_selection_mode('COLUMNS').with_columns(column_names)
+
+    def with_regex_column_selection(self, regex):
+        """Sets the step's as applying to a single column"""
+        return self.with_column_selection_mode('PATTERN').with_column_regex(regex)
+
+    def with_all_column_selection(self, column_name):
+        """Sets the step's as applying to all columns"""
+        return self.with_column_selection_mode('ALL')
+
+class FilterAndFlagStepBuilder(AppliesToStepBuilder):
+    def __init__(self, step_type=None, step_name=None):
+    	super(FilterAndFlagStepBuilder, self).__init__(step_type=step_type, step_name=step_name)
+    	self.step["params"]["booleanMode"] = 'AND'
+    	self.step["params"]["action"] = 'REMOVE_ROW'
+
+    def with_action(self, action):
+        """Sets the step's action on match (KEEP_ROW, REMOVE_ROW, CLEAR_CELL, DONTCLEAR_CELL, FLAG)"""
+        self.step["params"]["action"] = column_selection_mode
+        return self
+
+    def with_boolean_mode(self, boolean_mode):
+        """Sets the step's mode for combining matches in different columns (AND, OR)"""
+        self.step["params"]["booleanMode"] = boolean_mode
+        return self
+
+    def with_flag_column(self, column_name):
+        """Sets the step's column for outputing the flag"""
+        self.step["params"]["flagColumn"] = column_name
+        return self
+
+class FilterOnValueStepBuilder(FilterAndFlagStepBuilder):
+    def __init__(self, step_name=None):
+    	super(FilterOnValueStepBuilder, self).__init__(step_type='FlagOnValue', step_name=step_name)
+
+    def with_values(self, *values):
+        """Sets the step's flagged values"""
+        self.step["params"]["values"] = [v for v in values]
+        return self
+
+    def with_matching_mode(self, matching_mode):
+        """Sets the step's matching_mode (FULL_STRING, SUBSTRING, PATTERN)"""
+        self.step["params"]["matchingMode"] = matching_mode
+        return self
+
+    def with_normalization_mode(self, normalization_mode):
+        """Sets the step's normalization_mode (EXACT, LOWERCASE, NORMALIZED)"""
+        self.step["params"]["normalizationMode"] = normalization_mode
+        return self
+
+class FilterOnBadTypeStepBuilder(FilterAndFlagStepBuilder):
+    def __init__(self, step_name=None):
+    	super(FilterOnBadTypeStepBuilder, self).__init__(step_type='FilterOnBadType', step_name=step_name)
+
+    def with_meaning(self, meaning):
+        """Sets the step's meaning to check"""
+        self.step["params"]["type"] = meaning
+        return self
+
+class RemoveRowsStepBuilder(AppliesToStepBuilder):
+    def __init__(self, step_name=None):
+    	super(RemoveRowsStepBuilder, self).__init__(step_type='RemoveRowsOnEmpty', step_name=step_name)
+
+    def with_meaning(self, keep):
+        """Sets the step's behavior when an empty value is found : True=keep, False=drop (default)"""
+        self.step["params"]["keep"] = keep
+        return self
+
+

dataikuapi/dss/ml.py

@@ -4,50 +4,7 @@
 import json
 import time
 from .metrics import ComputedMetrics
-
-class DSSDatasetSelectionBuilder(object):
-    """Builder for a "dataset selection". In DSS, a dataset selection is used to select a part of a dataset for processing.
-
-    Depending on the location where it is used, a selection can include:
-    * Sampling
-    * Filtering by partitions (for partitioned datasets)
-    * Filtering by an expression
-    * Selection of columns
-    * Ordering
-
-    Please see the sampling documentation of DSS for a detailed explanation of the sampling methods.
-
-    """
-    def __init__(self):
-        self.selection = {}
-
-    def build(self):
-        """Returns the built selection dict"""
-        return self.selection
-
-    def with_head_sampling(self, limit):
-        """Sets the sampling to 'first records' mode"""
-        self.selection["samplingMethod"] = "HEAD_SEQUENTIAL"
-        self.selection["maxRecords"] = limit
-        return self
-
-    def with_all_data_sampling(self):
-        """Sets the sampling to 'no sampling, all data' mode"""
-        self.selection["samplingMethod"] = "FULL"
-        return self
-
-    def with_random_fixed_nb_sampling(self, nb):
-        """Sets the sampling to 'Random sampling, fixed number of records' mode"""
-        self.selection["samplingMethod"] = "RANDOM_FIXED_NB"
-        self.selection["maxRecords"] = nb
-        return self
-
-    def with_selected_partitions(self, ids):
-        """Sets partition filtering on the given partition identifiers. The dataset to select must be partitioned."""
-        self.selection["partitionSelectionMethod"] = "SELECTED"
-        self.selection["selectedPartitions"] = ids
-        return self
-
+from .utils import DSSDatasetSelectionBuilder
 
 class PredictionSplitParamsHandler(object):
     """Object to modify the train/test splitting params."""

dataikuapi/dss/utils.py

@@ -0,0 +1,44 @@
+class DSSDatasetSelectionBuilder(object):
+    """Builder for a "dataset selection". In DSS, a dataset selection is used to select a part of a dataset for processing.
+
+    Depending on the location where it is used, a selection can include:
+    * Sampling
+    * Filtering by partitions (for partitioned datasets)
+    * Filtering by an expression
+    * Selection of columns
+    * Ordering
+
+    Please see the sampling documentation of DSS for a detailed explanation of the sampling methods.
+
+    """
+    def __init__(self):
+        self.selection = {}
+
+    def build(self):
+        """Returns the built selection dict"""
+        return self.selection
+
+    def with_head_sampling(self, limit):
+        """Sets the sampling to 'first records' mode"""
+        self.selection["samplingMethod"] = "HEAD_SEQUENTIAL"
+        self.selection["maxRecords"] = limit
+        return self
+
+    def with_all_data_sampling(self):
+        """Sets the sampling to 'no sampling, all data' mode"""
+        self.selection["samplingMethod"] = "FULL"
+        return self
+
+    def with_random_fixed_nb_sampling(self, nb):
+        """Sets the sampling to 'Random sampling, fixed number of records' mode"""
+        self.selection["samplingMethod"] = "RANDOM_FIXED_NB"
+        self.selection["maxRecords"] = nb
+        return self
+
+    def with_selected_partitions(self, ids):
+        """Sets partition filtering on the given partition identifiers. The dataset to select must be partitioned."""
+        self.selection["partitionSelectionMethod"] = "SELECTED"
+        self.selection["selectedPartitions"] = ids
+        return self
+
+