doc recipe creation

Author: FChataigner

dataikuapi/dss/recipe.py

@@ -85,32 +85,64 @@ def __init__(self, data):
         self.data = data
 
     def get_recipe_raw_definition(self):
+        """
+        Get the recipe definition as a raw JSON object
+        """
         return self.data.get('recipe', None)
 
     def get_recipe_inputs(self):
+        """
+        Get the list of inputs of this recipe
+        """
         return self.data.get('recipe').get('inputs')
 
     def get_recipe_outputs(self):
+        """
+        Get the list of outputs of this recipe
+        """
         return self.data.get('recipe').get('outputs')
 
     def get_recipe_params(self):
+        """
+        Get the parameters of this recipe, as a raw JSON object
+        """
         return self.data.get('recipe').get('params')
 
     def get_payload(self):
+        """
+        Get the payload or script of this recipe, as a raw string
+        """
         return self.data.get('payload', None)
 
     def get_json_payload(self):
+        """
+        Get the payload or script of this recipe, as a JSON object
+        """
         return json.loads(self.data.get('payload', None))
 
     def set_payload(self, payload):
+        """
+        Set the raw payload of this recipe
+
+        :param str payload: the payload, as a string
+        """
         self.data['payload'] = payload
 
     def set_json_payload(self, payload):
+        """
+        Set the raw payload of this recipe
+
+        :param dict payload: the payload, as a dict. The payload will be converted to a JSON string internally
+        """
         self.data['payload'] = json.dumps(payload)
 
 class DSSRecipeCreator(object):
     """
     Helper to create new recipes
+
+    :param str type: type of the recipe
+    :param str name: name for the recipe
+    :param :class:`dataikuapi.dss.project.DSSProject` project: project in which the recipe will be created
     """
     def __init__(self, type, name, project):
         self.project = project
@@ -146,11 +178,27 @@ def _with_output(self, dataset_name, append=False, role="main"):
         return self
 
     def with_input(self, dataset_name, project_key=None, role="main"):
+        """
+        Add an existing object as input to the recipe-to-be-created
+
+        :param dataset_name: name of the dataset, or identifier of the managed folder
+                             or identifier of the saved model
+        :param project_key: project containing the object, if different from the one where the recipe is created
+        :param str role: the role of the recipe in which the input should be added
+        """
         return self._with_input(dataset_name, project_key, role)
 
     def with_output(self, dataset_name, append=False, role="main"):
-        """The output dataset must already exist. If you are creating a visual recipe with a single
-        output, use with_existing_output"""
+        """
+        The output dataset must already exist. If you are creating a visual recipe with a single
+        output, use with_existing_output
+
+        :param dataset_name: name of the dataset, or identifier of the managed folder
+                             or identifier of the saved model
+        :param append: whether the recipe should append or overwrite the output when running
+                       (note: not available for all dataset types)
+        :param str role: the role of the recipe in which the input should be added
+        """
         return self._with_output(dataset_name, append, role)
 
     def build(self):
@@ -167,6 +215,10 @@ def _finish_creation_settings(self):
         pass
 
 class SingleOutputRecipeCreator(DSSRecipeCreator):
+    """
+    Create a recipe that has a single output
+    """
+
     def __init__(self, type, name, project):
         DSSRecipeCreator.__init__(self, type, name, project)
         self.create_output_dataset = None
@@ -175,12 +227,38 @@ def __init__(self, type, name, project):
         self.output_folder_settings = None
 
     def with_existing_output(self, dataset_name, append=False):
+        """
+        Add an existing object as output to the recipe-to-be-created
+
+        :param dataset_name: name of the dataset, or identifier of the managed folder
+                             or identifier of the saved model
+        :param append: whether the recipe should append or overwrite the output when running
+                       (note: not available for all dataset types)
+        """
         assert self.create_output_dataset is None
         self.create_output_dataset = False
         self._with_output(dataset_name, append)
         return self
 
     def with_new_output(self, name, connection_id, typeOptionId=None, format_option_id=None, override_sql_schema=None, partitioning_option_id=None, append=False, object_type='DATASET'):
+        """
+        Create a new dataset as output to the recipe-to-be-created. The dataset is not created immediately,
+        but when the recipe is created (ie in the build() method)
+
+        :param str name: name of the dataset or identifier of the managed folder
+        :param str connection_id: name of the connection to create the dataset on
+        :param str typeOptionId: sub-type of dataset, for connection where the type could be ambiguous. Typically,
+                                 this is SCP or SFTP, for SSH connection
+        :param str format_option_id: name of a format preset relevant for the dataset type. Possible values are: CSV_ESCAPING_NOGZIP_FORHIVE, 
+                                     CSV_UNIX_GZIP, CSV_EXCEL_GZIP, CSV_EXCEL_GZIP_BIGQUERY, CSV_NOQUOTING_NOGZIP_FORPIG, PARQUET_HIVE, 
+                                     AVRO, ORC 
+        :param override_sql_schema: schema to force dataset, for SQL dataset. If left empty, will be autodetected
+        :param str partitioning_option_id: to copy the partitioning schema of an existing dataset 'foo', pass a
+                                           value of 'copy:foo'
+        :param append: whether the recipe should append or overwrite the output when running
+                       (note: not available for all dataset types)
+        :param str object_type: DATASET or MANAGED_FOLDER
+        """
         if object_type == 'DATASET':
             assert self.create_output_dataset is None
             self.create_output_dataset = True
@@ -194,6 +272,7 @@ def with_new_output(self, name, connection_id, typeOptionId=None, format_option_
         return self
 
     def with_output(self, dataset_name, append=False):
+        """Alias of with_existing_output"""
         return self.with_existing_output(dataset_name, append)
 
     def _finish_creation_settings(self):
@@ -203,6 +282,10 @@ def _finish_creation_settings(self):
         self.creation_settings['outputFolderSettings'] = self.output_folder_settings
 
 class VirtualInputsSingleOutputRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a recipe that has a single output and several inputs
+    """
+
     def __init__(self, type, name, project):
         SingleOutputRecipeCreator.__init__(self, type, name, project)
         self.virtual_inputs = []
@@ -221,31 +304,54 @@ def _finish_creation_settings(self):
 #
 ########################
 class WindowRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Window recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'window', name, project)
 
 class SyncRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Sync recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'sync', name, project)
 
 class SortRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Sort recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'sort', name, project)
 
 class TopNRecipeCreator(DSSRecipeCreator):
+    """
+    Create a TopN recipe
+    """
     def __init__(self, name, project):
         DSSRecipeCreator.__init__(self, 'topn', name, project)
 
 class DistinctRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Distinct recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'distinct', name, project)
 
 class GroupingRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Group recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'grouping', name, project)
         self.group_key = None
 
     def with_group_key(self, group_key):
+        """
+        Set a column as grouping key
+
+        :param str group_key: name of a column in the input
+        """
         self.group_key = group_key
         return self
 
@@ -254,37 +360,61 @@ def _finish_creation_settings(self):
         self.creation_settings['groupKey'] = self.group_key
 
 class JoinRecipeCreator(VirtualInputsSingleOutputRecipeCreator):
+    """
+    Create a Join recipe
+    """
     def __init__(self, name, project):
         VirtualInputsSingleOutputRecipeCreator.__init__(self, 'join', name, project)
 
 class StackRecipeCreator(VirtualInputsSingleOutputRecipeCreator):
+    """
+    Create a Stack recipe
+    """
     def __init__(self, name, project):
         VirtualInputsSingleOutputRecipeCreator.__init__(self, 'vstack', name, project)
 
 class SamplingRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Sample/Filter recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'sampling', name, project)
 
 class CodeRecipeCreator(DSSRecipeCreator):
     def __init__(self, name, type, project):
+        """
+        Create a recipe running a script
+
+        :param str type: the type of the recipe (possible values : python, r, hive, impala, spark_scala, pyspark, sparkr)
+        """
         DSSRecipeCreator.__init__(self, type, name, project)
         self.script = None
 
     def with_script(self, script):
+        """
+        Set the code of the recipe
+
+        :param str script: the script of the recipe
+        """
         self.script = script
         return self
 
     def _finish_creation_settings(self):
         super(CodeRecipeCreator, self)._finish_creation_settings()
-        # DSSRecipeCreator._finish_creation_settings(self)
         self.creation_settings['script'] = self.script
 
 
 class SQLQueryRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a SQL query recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'sql_query', name, project)
 
 class SplitRecipeCreator(DSSRecipeCreator):
+    """
+    Create a Split recipe
+    """
     def __init__(self, name, project):
         DSSRecipeCreator.__init__(self, "split", name, project)
 
@@ -356,5 +486,8 @@ def with_input_model(self, model_id):
 
 
 class DownloadRecipeCreator(SingleOutputRecipeCreator):
+    """
+    Create a Download recipe
+    """
     def __init__(self, name, project):
         SingleOutputRecipeCreator.__init__(self, 'download', name, project)