New helpers for app handling and start adding flow manipulation helpers

Author: cstenac

dataikuapi/dss/app.py

@@ -2,7 +2,10 @@
 import os.path as osp
 from .future import DSSFuture
 from dataikuapi.utils import DataikuException
+import random, string
 
+def random_string(length):
+    return ''.join(random.choice(string.ascii_letters) for _ in range(length))
 
 class DSSApp(object):
     """
@@ -13,27 +16,115 @@ def __init__(self, client, app_id):
        self.client = client
        self.app_id = app_id
 
-
     ########################################################
     # Instances
     ########################################################
 
+    def create_instance(self, instance_key, instance_name, wait=True):
+        """
+        Creates a new instance of this app. Each instance. must have a globally unique
+        instance key, separate from any project key across the whole DSS instance
+
+        :return: 
+        """
+        future_resp = self.client._perform_json(
+            "POST", "/apps/%s/instances" % self.app_id, body={
+            "targetProjectKey" : instance_key,
+            "targetProjectName" : instance_name
+        })
+        future = DSSFuture(self.client, future_resp.get("jobId", None), future_resp)
+        if wait:
+            result = future.wait_for_result()
+            return DSSAppInstance(self.client, instance_key)
+        else:
+            return future
+
+    def create_temporary_instance(self):
+        """
+        Creates a new temporary instance of this app.
+        The return value should be used as a Python context manager. Upon exit, the temporary app
+        instance is deleted
+        :return a :class:`TemporaryDSSAppInstance`
+        """
+        key = "%s_tmp_%s" % (self.app_id, random_string(10))
+        instance = self.create_instance(key, key, True)
+        return TemporaryDSSAppInstance(self.client, key)
+
+    def list_instance_keys(self):
+        """
+        List the existing instances of this app
+
+        :return a list of instance keys, each as a string
+        """
+        return [x["projectKey"] for x in self.list_instances()]
+
     def list_instances(self):
         """
-        List the instances in this project
+        List the existing instances of this app
         
-        Returns:
-            the list of the instances, each one as a JSON object
+        :rtype: list of dicts
+        :return a list of instances, each as a dict containing at least a "projectKey" field
         """
         return self.client._perform_json(
             "GET", "/apps/%s/instances/" % self.app_id)
 
+    def get_instance(self, instance_key):
+        return DSSAppInstance(self.client, instance_key)
+
+
+    def get_manifest(self):
+        raw_data = self.client._perform_json("GET", "/apps/%s/" % self.app_id)
+        return DSSAppManifest(self.client, raw_data)
+
+
+class DSSAppManifest(object):
 
-class DSSDataikuAppInstance(object):
+    def __init__(self, client, raw_data):
+        """The manifest for an app. Do not create this class directly"""
+        self.client = client
+        self.raw_data = raw_data
+
+    def get_raw(self):
+        return self.raw_data
+
+    def get_all_actions(self):
+        return [x for section in self.raw_data["homepageSections"] for x in section["tiles"]]
+
+    def get_runnable_scenarios(self):
+        """Return the scenario identifiers that are declared as actions for this app"""
+        return [x["scenarioId"] for x in self.get_all_actions() if x["type"] == "SCENARIO_RUN"]
+
+class DSSAppInstance(object):
 
     def __init__(self, client, project_key):
        self.client = client
        self.project_key = project_key
 
     def get_as_project(self):
-        self
+        """
+        Get the :class:`dataikuapi.dss.project DSSProject` corresponding to this app instance
+        """
+        return self.client.get_project(self.project_key)
+
+    def get_manifest(self):
+        """
+        Get the app manifest for this instance, as a :class:`DSSAppManifest`
+        """
+        raw_data = self.client._perform_json("GET", "/projects/%s/app-manifest" % self.project_key)
+        return DSSAppManifest(self.client, raw_data)
+
+class TemporaryDSSAppInstance(DSSAppInstance):
+    """internal class"""
+
+    def __init__(self, client, project_key):
+        DSSAppInstance.__init__(self, client,project_key)
+
+
+    def close(self):
+        self.get_as_project().delete()
+
+    def __enter__(self,):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()

dataikuapi/dss/flow.py

@@ -0,0 +1,45 @@
+from .utils import AnyLoc
+from .recipe import DSSRecipeDefinitionAndPayload
+import logging
+class DSSProjectFlow(object):
+    def __init__(self, client, project):
+        self.client = client
+        self.project = project
+
+
+    def replace_input_computable(self, current_ref, new_ref, type="DATASET"):
+        """
+        This method replaces all references to a "computable" (Dataset, Managed Folder or Saved Model)
+        as input of recipes in the whole Flow by a reference to another computable.
+
+        No specific checks are performed. It is your responsibility to ensure that the schema
+        of the new dataset will be compatible with the previous one (in the case of datasets).
+
+        If `new_ref` references an object in a foreign project, this method will automatically
+        ensure that `new_ref` is exposed to the current project
+
+        :param current_ref str: Either a "simple" object name (dataset name, model id, managed folder id)
+                    or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id")
+        :param new_ref str: Either a "simple" object name (dataset name, model id, managed folder id)
+                    or a foreign object reference in the form "FOREIGN_PROJECT_KEY.local_id")
+        :param type str: The type of object being replaced (DATASET, SAVED_MODEL or MANAGED_FOLDER)
+        """
+
+        new_loc = AnyLoc.from_ref(self.project.project_key, new_ref)
+
+        if new_loc.project_key != self.project.project_key:
+            logging.debug("New ref is in project %s, exposing it to project %s" % (new_loc.project_key, self.project.project_key))
+            new_ref_src_project = self.client.get_project(new_loc.project_key)
+            settings = new_ref_src_project.get_settings()
+            settings.add_exposed_object(type, new_loc.object_id, self.project.project_key)
+            settings.save()
+
+        for recipe in self.project.list_recipes():
+            fake_rap = DSSRecipeDefinitionAndPayload({"recipe" : recipe})
+            if fake_rap.has_input(current_ref):
+                logging.debug("Recipe %s has %s as input, performing the replacement by %s"% \
+                    (recipe["name"], current_ref, new_ref))
+                recipe_obj = self.project.get_recipe(recipe["name"])
+                dap = recipe_obj.get_definition_and_payload()
+                dap.replace_input(current_ref, new_ref)
+                recipe_obj.set_definition_and_payload(dap)

dataikuapi/dss/project.py

@@ -15,6 +15,7 @@
 from .discussion import DSSObjectDiscussions
 from .ml import DSSMLTask
 from .analysis import DSSAnalysis
+from .flow import DSSProjectFlow
 from dataikuapi.utils import DataikuException
 
 
@@ -755,6 +756,13 @@ def create_recipe(self, recipe_proto, creation_settings):
                        body = definition)['name']
         return DSSRecipe(self.client, self.project_key, recipe_name)
 
+    ########################################################
+    # Flow
+    ########################################################
+
+    def get_flow(self):
+        return DSSProjectFlow(self.client, self)
+
     ########################################################
     # Security
     ########################################################
@@ -1021,7 +1029,6 @@ def add_exposed_object(self, object_type, object_id, target_project):
         Exposes an object from this project to another project.
         Does nothing if the object was already exposed to the target project
         """
-
         found_eo = None
         for eo in self.settings["exposedObjects"]["objects"]:
             if eo["type"] == object_type and eo["localName"] == object_id:

dataikuapi/dss/recipe.py

@@ -211,6 +211,40 @@ def set_json_payload(self, payload):
         """
         self.data['payload'] = json.dumps(payload)
 
+    def has_input(self, input_ref):
+        """Returns whether this recipe has a given ref as input"""
+        inputs = self.get_recipe_inputs()
+        for (input_role_name, input_role) in inputs.items():
+            for item in input_role.get("items", []):
+                if item.get("ref", None) == input_ref:
+                    return True
+        return False
+
+    def has_output(self, output_ref):
+        """Returns whether this recipe has a given ref as output"""
+        outputs = self.get_recipe_outputs()
+        for (output_role_name, output_role) in outputs.items():
+            for item in output_role.get("items", []):
+                if item.get("ref", None) == output_ref:
+                    return True
+        return False
+
+    def replace_input(self, current_input_ref, new_input_ref):
+        """Replaces an object reference as input of this recipe by another"""
+        inputs = self.get_recipe_inputs()
+        for (input_role_name, input_role) in inputs.items():
+            for item in input_role.get("items", []):
+                if item.get("ref", None) == current_input_ref:
+                    item["ref"] = new_input_ref
+
+    def replace_output(self, current_output_ref, new_output_ref):
+        """Replaces an object reference as output of this recipe by another"""
+        outputs = self.get_recipe_outputs()
+        for (output_role_name, output_role) in outputs.items():
+            for item in output_role.get("items", []):
+                if item.get("ref", None) == current_output_ref:
+                    item["ref"] = new_output_ref
+
 class DSSRecipeCreator(object):
     """
     Helper to create new recipes

dataikuapi/dssclient.py

@@ -191,7 +191,7 @@ def list_apps(self):
         """
         List the apps
 
-        :returns: a list of apps, each as a dict. Each dictcontains at least a 'appId' field
+        :returns: a list of apps, each as a dict. Each dict contains at least a 'appId' field
         :rtype: list of dicts
         """
         return self._perform_json("GET", "/apps/")