Merge remote-tracking branch 'origin/release/10.0' into feature/dss100-sc-58209-public-api-for-api-keys

Author: qcastel

dataikuapi/apinode_client.py

@@ -6,15 +6,16 @@ class APINodeClient(DSSBaseClient):
     This is an API client for the user-facing API of DSS API Node server (user facing API)
     """
 
-    def __init__(self, uri, service_id, api_key=None):
+    def __init__(self, uri, service_id, api_key=None, bearer_token=None):
         """
         Instantiate a new DSS API client on the given base URI with the given API key.
 
         :param str uri: Base URI of the DSS API node server (http://host:port/ or https://host:port/)
         :param str service_id: Identifier of the service to query
-        :param str api_key: Optional, API key for the service. Only required if the service has authentication
+        :param str api_key: Optional, API key for the service. Only required if the service has its authorization setup to API keys
+        :param str bearer_token: Optional, The bearer token. Only required if the service has its authorization setup to OAuth2/JWT
         """
-        DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key)
+        DSSBaseClient.__init__(self, "%s/%s" % (uri, "public/api/v1/%s" % service_id), api_key=api_key, bearer_token=bearer_token)
 
     def predict_record(self, endpoint_id, features, forced_generation=None, dispatch_key=None, context=None,
                        with_explanations=None, explanation_method=None, n_explanations=None, n_explanations_mc_steps=None):

dataikuapi/base_client.py

@@ -5,8 +5,9 @@
 from .utils import DataikuException
 
 class DSSBaseClient(object):
-    def __init__(self, base_uri, api_key=None, internal_ticket=None):
+    def __init__(self, base_uri, api_key=None, internal_ticket=None, bearer_token=None):
         self.api_key = api_key
+        self.bearer_token = bearer_token
         self.base_uri = base_uri
         self._session = Session()
 
@@ -18,12 +19,18 @@ def _perform_http(self, method, path, params=None, body=None, stream=False):
         if body:
             body = json.dumps(body)
 
-        auth = HTTPBasicAuth(self.api_key, "")
+        headers = None
+        auth = None
+
+        if self.api_key:
+            auth = HTTPBasicAuth(self.api_key, "")
+        elif self.bearer_token:
+            headers = {"Authorization": "Bearer " + self.bearer_token}
 
         try:
             http_res = self._session.request(
                     method, "%s/%s" % (self.base_uri, path),
-                    params=params, data=body,
+                    params=params, data=body, headers=headers,
                     auth=auth, stream = stream)
             http_res.raise_for_status()
             return http_res

dataikuapi/dss/admin.py

@@ -1,3 +1,5 @@
+import datetime
+
 from .future import DSSFuture
 import json, warnings
 
@@ -164,11 +166,22 @@ def delete(self):
     def get_settings(self):
         """
         Gets the settings of the user
+
         :rtype: :class:`DSSUserSettings`
         """
         raw = self.client._perform_json("GET", "/admin/users/%s" % self.login)
         return DSSUserSettings(self.client, self.login, raw)
 
+    def get_activity(self):
+        """
+        Gets the activity of the user
+
+        :return: the user's activity
+        :rtype: :class:`DSSUserActivity`
+        """
+        activity = self.client._perform_json("GET", "/admin/users/%s/activity" % self.login)
+        return DSSUserActivity(self.client, self.login, activity)
+
     ########################################################
     # Legacy
     ########################################################
@@ -224,6 +237,7 @@ def get_client_as(self):
         else:
             raise ValueError("Don't know how to proxy this client")
 
+
 class DSSOwnUser(object):
     """
     A handle to interact with your own user
@@ -250,8 +264,7 @@ def __init__(self, settings):
 
     def get_raw(self):
         """
-        :return: the raw settings of the user, as a dict. Modifications made to the returned object 
-        are reflected when saving
+        :return: the raw settings of the user, as a dict. Modifications made to the returned object are reflected when saving
         :rtype: dict
         """
         return self.settings
@@ -331,15 +344,15 @@ def admin_properties(self):
         """
         The user properties (not editable by the user) for this user. Do not set this property, modify the dict in place
 
-        :rtype dict
+        :rtype: dict
         """
         return self.settings["adminProperties"]
 
     @property
     def enabled(self):
         """
         Whether this user is enabled
-        :rtype boolean
+        :rtype: boolean
         """
         return self.settings["enabled"]
 
@@ -365,6 +378,67 @@ def save(self):
         self.client._perform_empty("PUT", "/current-user", body = self.settings)
 
 
+class DSSUserActivity(object):
+    """
+    Activity for a DSS user.
+    Do not call this directly, use :meth:`DSSUser.get_activity` or :meth:`DSSClient.list_users_activity`
+    """
+
+    def __init__(self, client, login, activity):
+        self.client = client
+        self.login = login
+        self.activity = activity
+
+    def get_raw(self):
+        """
+        Get the raw activity of the user as a dict.
+
+        :return: the raw activity
+        :rtype: dict
+        """
+        return self.activity
+
+    @property
+    def last_successful_login(self):
+        """
+        Get the last successful login of the user as a :class:`datetime.datetime`
+        
+        Returns None if there was no successful login for this user.
+
+        :return: the last successful login
+        :rtype: :class:`datetime.datetime` or None
+        """
+        timestamp = self.activity["lastSuccessfulLogin"]
+        return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None
+
+    @property
+    def last_failed_login(self):
+        """
+        Get the last failed login of the user as a :class:`datetime.datetime`
+
+        Returns None if there were no failed login for this user.
+
+        :return: the last failed login
+        :rtype: :class:`datetime.datetime` or None
+        """
+        timestamp = self.activity["lastFailedLogin"]
+        return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None
+
+    @property
+    def last_session_activity(self):
+        """
+        Get the last session activity of the user as a :class:`datetime.datetime`, i.e. the last time
+        the user opened a new DSS tab or refreshed his session.
+
+        Returns None if there is no session activity yet.
+
+        :return: the last session activity
+        :rtype: :class:`datetime.datetime` or None
+        """
+        timestamp = self.activity["lastSessionActivity"]
+        return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp > 0 else None
+
+
 class DSSGroup(object):
     """
     A group on the DSS instance.

dataikuapi/dss/dataset.py

@@ -1,3 +1,5 @@
+import datetime
+
 from ..utils import DataikuException
 from ..utils import DataikuUTF8CSVReader
 from ..utils import DataikuStreamedHttpUTF8CSVReader
@@ -511,7 +513,6 @@ def get_last_metric_values(self, partition=''):
         return ComputedMetrics(self.client._perform_json(
                 "GET", "/projects/%s/datasets/%s/metrics/last/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition)))
 
-
     def get_metric_history(self, metric, partition=''):
         """
         Get the history of the values of the metric on this dataset
@@ -523,6 +524,16 @@ def get_metric_history(self, metric, partition=''):
                 "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition),
                 params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)})
 
+    def get_info(self):
+        """
+        Retrieve all the information about a dataset
+
+        :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset.
+        :rtype: :class:`DSSDatasetInfo`
+        """
+        data = self.client._perform_json("GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name))
+        return DSSDatasetInfo(self, data)
+
     ########################################################
     # Misc
     ########################################################
@@ -542,7 +553,7 @@ def move_to_zone(self, zone):
         :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object
         """
         if isinstance(zone, basestring):
-           zone = self.project.get_flow().get_zone(zone)
+            zone = self.project.get_flow().get_zone(zone)
         zone.add_item(self)
 
     def share_to_zone(self, zone):
@@ -859,3 +870,59 @@ def already_exists(self):
             return True
         except Exception as e:
             return False
+
+
+class DSSDatasetInfo(object):
+    """
+    Info class for a DSS dataset (Read-Only).
+    Do not instantiate this class directly, use :meth:`DSSDataset.get_info`
+    """
+
+    def __init__(self, dataset, info):
+        self.dataset = dataset
+        self.info = info
+
+    def get_raw(self):
+        """
+        Get the raw dataset full information as a dict
+
+        :return: the raw dataset full information
+        :rtype: dict
+        """
+        return self.info
+
+    @property
+    def last_build_start_time(self):
+        """
+        The last build start time of the dataset as a :class:`datetime.datetime` or None if there is no last build information.
+
+        :return: the last build start time
+        :rtype: :class:`datetime.datetime` or None
+        """
+        last_build_info = self.info.get("lastBuild", dict())
+        timestamp = last_build_info.get("buildStartTime", None)
+        return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None
+
+    @property
+    def last_build_end_time(self):
+        """
+        The last build end time of the dataset as a :class:`datetime.datetime` or None if there is no last build information.
+
+        :return: the last build end time
+        :rtype: :class:`datetime.datetime` or None
+        """
+        last_build_info = self.info.get("lastBuild", dict())
+        timestamp = last_build_info.get("buildEndTime", None)
+        return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None
+
+    @property
+    def is_last_build_successful(self):
+        """
+        Get whether the last build of the dataset is successful.
+
+        :return: True if the last build is successful
+        :rtype: bool
+        """
+        last_build_info = self.info.get("lastBuild", dict())
+        success = last_build_info.get("buildSuccess", False)
+        return success

dataikuapi/dss/modelcomparison.py

@@ -1,4 +1,4 @@
-from dataikuapi.dss.discussion import DSSObjectDiscussions
+from .discussion import DSSObjectDiscussions
 import re
 
 

dataikuapi/dss/modelevaluationstore.py

@@ -1,7 +1,7 @@
 import json
 from io import BytesIO
 
-from dataikuapi.dss.metrics import ComputedMetrics
+from .metrics import ComputedMetrics
 from .discussion import DSSObjectDiscussions
 from .future import DSSFuture
 

dataikuapi/dss/recipe.py

@@ -1411,6 +1411,12 @@ class StandaloneEvaluationRecipeCreator(DSSRecipeCreator):
         builder.with_input("scored_dataset_to_evaluate")
         builder.with_output_evaluation_store(evaluation_store_id)
 
+        # Add a reference dataset (optional) to compute data drift
+
+        builder.with_reference_dataset("reference_dataset")
+
+        # Finish creation of the recipe
+
         new_recipe = builder.create()
 
         # Modify the model parameters in the SER settings
@@ -1465,6 +1471,10 @@ def with_output_evaluation_store(self, mes_id):
         """Sets the output model evaluation store"""
         return self._with_output(mes_id, role="main")
 
+    def with_reference_dataset(self, dataset_name):
+        """Sets the dataset to use as a reference in data drift computation (optional)."""
+        return self._with_input(dataset_name, self.project.project_key, role="reference")
+
 
 class ClusteringScoringRecipeCreator(SingleOutputRecipeCreator):
     """

dataikuapi/dss/savedmodel.py

@@ -6,7 +6,7 @@
 from .ml import DSSTrainedPredictionModelDetails
 from .managedfolder import DSSManagedFolder
 
-from ..utils import _make_zipfile
+from ..utils import _make_zipfile, dku_basestring_type
 
 try:
     basestring
@@ -166,10 +166,13 @@ def import_mlflow_version_from_managed_folder(self, version_id, managed_folder,
         """
         # TODO: Add a check that it's indeed a MLFlow model folder
         folder_ref = None
-        if type(managed_folder) is DSSManagedFolder:
+        if isinstance(managed_folder, DSSManagedFolder):
             folder_ref = "{}.{}".format(managed_folder.project_key, managed_folder.id)
-        else:
+        elif isinstance(managed_folder, dku_basestring_type):
             folder_ref = managed_folder
+        else:
+            raise Exception("managed_folder should either be a string representing the identifier of the managed folder"
+                            " or an instance of dataikuapi.dss.managedfolder.DSSManagedFolder")
 
         self.client._perform_empty(
             "POST", "/projects/{project_id}/savedmodels/{saved_model_id}/versions/{version_id}?codeEnvName={codeEnvName}".format(

dataikuapi/dss/scenario.py

@@ -134,15 +134,17 @@ def get_run(self, run_id):
     def get_status(self):
         """
         Returns the status of this scenario
-        :rtype :class:`DSSScenarioStatus`
+
+        :rtype: :class:`DSSScenarioStatus`
         """
         data = self.client._perform_json("GET", "/projects/%s/scenarios/%s/light" % (self.project_key, self.id))
         return DSSScenarioStatus(self, data)
 
     def get_settings(self):
         """
         Returns the settings of this scenario
-        :rtype :class:`StepBasedDSSScenarioSettings` or :class:`PythonScriptBasedScenarioSettings`
+
+        :rtype: :class:`StepBasedScenarioSettings` or :class:`PythonScriptBasedScenarioSettings`
         """
         data = self.client._perform_json("GET", "/projects/%s/scenarios/%s" % (self.project_key, self.id))