Skip to content

Commit 152bfef

Browse files
EmanAbdelhaleemEmanAbdelhaleem
committed
Update list - sync PR
1 parent 150c7d6 commit 152bfef

File tree

3 files changed

+190
-195
lines changed

3 files changed

+190
-195
lines changed

‎openml/_api/resources/base.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from openml._api.http import HTTPClient
1010
from openml.datasets.dataset import OpenMLDataset
11+
from openml.evaluations import OpenMLEvaluation
1112
from openml.tasks.task import OpenMLTask
1213

1314

@@ -33,13 +34,21 @@ def get(
3334

3435
class EvaluationsAPI(ResourceAPI, ABC):
3536
@abstractmethod
36-
def list(
37+
def list( # noqa: PLR0913
3738
self,
3839
limit: int,
3940
offset: int,
41+
*,
4042
function: str,
43+
tasks: list | None = None,
44+
setups: list | None = None,
45+
flows: list | None = None,
46+
runs: list | None = None,
47+
uploaders: list | None = None,
48+
study: int | None = None,
49+
sort_order: str | None = None,
4150
**kwargs: Any,
42-
) -> dict: ...
51+
) -> list[OpenMLEvaluation]: ...
4352

4453
@abstractmethod
4554
def get_users(self, uploader_ids: list[str]) -> dict: ...

‎openml/_api/resources/evaluations.py‎

Lines changed: 173 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,34 @@
11
from __future__ import annotations
22

3+
import json
34
from typing import Any
45

56
import xmltodict
67

78
from openml._api.resources.base import EvaluationsAPI
9+
from openml.evaluations import OpenMLEvaluation
810

911

1012
class EvaluationsV1(EvaluationsAPI):
1113
"""V1 API implementation for evaluations.
1214
Fetches evaluations from the v1 XML API endpoint.
1315
"""
1416

15-
def list(
17+
def list( # noqa: PLR0913
1618
self,
1719
limit: int,
1820
offset: int,
21+
*,
1922
function: str,
23+
tasks: list | None = None,
24+
setups: list | None = None,
25+
flows: list | None = None,
26+
runs: list | None = None,
27+
uploaders: list | None = None,
28+
study: int | None = None,
29+
sort_order: str | None = None,
2030
**kwargs: Any,
21-
) -> dict:
31+
) -> list[OpenMLEvaluation]:
2232
"""Retrieve evaluations from the OpenML v1 XML API.
2333
2434
This method builds an evaluation query URL based on the provided
@@ -28,36 +38,38 @@ def list(
2838
2939
Parameters
3040
----------
41+
The arguments that are lists are separated from the single value
42+
ones which are put into the kwargs.
43+
3144
limit : int
32-
Maximum number of evaluations to return.
45+
the number of evaluations to return
3346
offset : int
34-
Offset for pagination.
47+
the number of evaluations to skip, starting from the first
3548
function : str
3649
the evaluation function. e.g., predictive_accuracy
37-
**kwargs
38-
Optional filters supported by the OpenML evaluation API, such as:
39-
- tasks
40-
- setups
41-
- flows
42-
- runs
43-
- uploaders
44-
- tag
45-
- study
46-
- sort_order
50+
51+
tasks : list[int,str], optional
52+
the list of task IDs
53+
setups: list[int,str], optional
54+
the list of setup IDs
55+
flows : list[int,str], optional
56+
the list of flow IDs
57+
runs :list[int,str], optional
58+
the list of run IDs
59+
uploaders : list[int,str], optional
60+
the list of uploader IDs
61+
62+
study : int, optional
63+
64+
kwargs: dict, optional
65+
Legal filter operators: tag, per_fold
66+
67+
sort_order : str, optional
68+
order of sorting evaluations, ascending ("asc") or descending ("desc")
4769
4870
Returns
4971
-------
50-
dict
51-
A dictionary containing:
52-
- Parsed evaluation data from the XML response
53-
- A "users" key mapping uploader IDs to usernames
54-
55-
Raises
56-
------
57-
ValueError
58-
If the XML response does not contain the expected structure.
59-
AssertionError
60-
If the evaluation data is not in list format as expected.
72+
list of OpenMLEvaluation objects
6173
6274
Notes
6375
-----
@@ -67,15 +79,112 @@ def list(
6779
6880
The user information is used to map uploader IDs to usernames.
6981
"""
70-
api_call = self._build_url(limit, offset, function, **kwargs)
82+
api_call = self._build_url(
83+
limit,
84+
offset,
85+
function=function,
86+
tasks=tasks,
87+
setups=setups,
88+
flows=flows,
89+
runs=runs,
90+
uploaders=uploaders,
91+
study=study,
92+
sort_order=sort_order,
93+
**kwargs,
94+
)
95+
7196
eval_response = self._http.get(api_call)
7297
xml_content = eval_response.text
7398

99+
return self._parse_list_xml(xml_content)
100+
101+
def _build_url( # noqa: PLR0913
102+
self,
103+
limit: int,
104+
offset: int,
105+
*,
106+
function: str,
107+
tasks: list | None = None,
108+
setups: list | None = None,
109+
flows: list | None = None,
110+
runs: list | None = None,
111+
uploaders: list | None = None,
112+
study: int | None = None,
113+
sort_order: str | None = None,
114+
**kwargs: Any,
115+
) -> str:
116+
"""
117+
Construct an OpenML evaluation API URL with filtering parameters.
118+
119+
Parameters
120+
----------
121+
The arguments that are lists are separated from the single value
122+
ones which are put into the kwargs.
123+
124+
limit : int
125+
the number of evaluations to return
126+
offset : int
127+
the number of evaluations to skip, starting from the first
128+
function : str
129+
the evaluation function. e.g., predictive_accuracy
130+
131+
tasks : list[int,str], optional
132+
the list of task IDs
133+
setups: list[int,str], optional
134+
the list of setup IDs
135+
flows : list[int,str], optional
136+
the list of flow IDs
137+
runs :list[int,str], optional
138+
the list of run IDs
139+
uploaders : list[int,str], optional
140+
the list of uploader IDs
141+
142+
study : int, optional
143+
144+
kwargs: dict, optional
145+
Legal filter operators: tag, per_fold
146+
147+
sort_order : str, optional
148+
order of sorting evaluations, ascending ("asc") or descending ("desc")
149+
150+
Returns
151+
-------
152+
str
153+
A relative API path suitable for an OpenML HTTP request.
154+
"""
155+
api_call = f"evaluation/list/function/{function}"
156+
if limit is not None:
157+
api_call += f"/limit/{limit}"
158+
if offset is not None:
159+
api_call += f"/offset/{offset}"
160+
if kwargs is not None:
161+
for operator, value in kwargs.items():
162+
if value is not None:
163+
api_call += f"/{operator}/{value}"
164+
if tasks is not None:
165+
api_call += f"/task/{','.join([str(int(i)) for i in tasks])}"
166+
if setups is not None:
167+
api_call += f"/setup/{','.join([str(int(i)) for i in setups])}"
168+
if flows is not None:
169+
api_call += f"/flow/{','.join([str(int(i)) for i in flows])}"
170+
if runs is not None:
171+
api_call += f"/run/{','.join([str(int(i)) for i in runs])}"
172+
if uploaders is not None:
173+
api_call += f"/uploader/{','.join([str(int(i)) for i in uploaders])}"
174+
if study is not None:
175+
api_call += f"/study/{study}"
176+
if sort_order is not None:
177+
api_call += f"/sort_order/{sort_order}"
178+
179+
return api_call
180+
181+
def _parse_list_xml(self, xml_content: str) -> list[OpenMLEvaluation]:
182+
"""Helper function to parse API calls which are lists of runs"""
74183
evals_dict: dict[str, Any] = xmltodict.parse(xml_content, force_list=("oml:evaluation",))
75184
# Minimalistic check if the XML is useful
76185
if "oml:evaluations" not in evals_dict:
77186
raise ValueError(
78-
"Error in return XML, does not contain " f'"oml:evaluations": {evals_dict!s}',
187+
f'Error in return XML, does not contain "oml:evaluations": {evals_dict!s}',
79188
)
80189

81190
assert isinstance(evals_dict["oml:evaluations"]["oml:evaluation"], list), (
@@ -87,9 +196,34 @@ def list(
87196
{eval_["oml:uploader"] for eval_ in evals_dict["oml:evaluations"]["oml:evaluation"]},
88197
)
89198
user_dict = self.get_users(uploader_ids)
90-
evals_dict["users"] = user_dict
91199

92-
return evals_dict
200+
evals = []
201+
for eval_ in evals_dict["oml:evaluations"]["oml:evaluation"]:
202+
run_id = int(eval_["oml:run_id"])
203+
value = float(eval_["oml:value"]) if "oml:value" in eval_ else None
204+
values = json.loads(eval_["oml:values"]) if eval_.get("oml:values", None) else None
205+
array_data = eval_.get("oml:array_data")
206+
207+
evals.append(
208+
OpenMLEvaluation(
209+
run_id=run_id,
210+
task_id=int(eval_["oml:task_id"]),
211+
setup_id=int(eval_["oml:setup_id"]),
212+
flow_id=int(eval_["oml:flow_id"]),
213+
flow_name=eval_["oml:flow_name"],
214+
data_id=int(eval_["oml:data_id"]),
215+
data_name=eval_["oml:data_name"],
216+
function=eval_["oml:function"],
217+
upload_time=eval_["oml:upload_time"],
218+
uploader=int(eval_["oml:uploader"]),
219+
uploader_name=user_dict[eval_["oml:uploader"]],
220+
value=value,
221+
values=values,
222+
array_data=array_data,
223+
)
224+
)
225+
226+
return evals
93227

94228
def get_users(self, uploader_ids: list[str]) -> dict:
95229
"""
@@ -112,80 +246,27 @@ def get_users(self, uploader_ids: list[str]) -> dict:
112246
users = xmltodict.parse(xml_content_user, force_list=("oml:user",))
113247
return {user["oml:id"]: user["oml:username"] for user in users["oml:users"]["oml:user"]}
114248

115-
def _build_url(
116-
self,
117-
limit: int,
118-
offset: int,
119-
function: str,
120-
**kwargs: Any,
121-
) -> str:
122-
"""
123-
Construct an OpenML evaluation API URL with filtering parameters.
124-
125-
Parameters
126-
----------
127-
limit : int
128-
Maximum number of evaluations to return.
129-
offset : int
130-
Offset for pagination.
131-
function : str
132-
the evaluation function. e.g., predictive_accuracy
133-
**kwargs
134-
Evaluation filters such as task IDs, flow IDs,
135-
uploader IDs, study name, and sorting options.
136-
137-
Returns
138-
-------
139-
str
140-
A relative API path suitable for an OpenML HTTP request.
141-
"""
142-
api_call = f"evaluation/list/function/{function}"
143-
if limit is not None:
144-
api_call += f"/limit/{limit}"
145-
if offset is not None:
146-
api_call += f"/offset/{offset}"
147-
148-
# List-based filters
149-
list_filters = {
150-
"task": kwargs.get("tasks"),
151-
"setup": kwargs.get("setups"),
152-
"flow": kwargs.get("flows"),
153-
"run": kwargs.get("runs"),
154-
"uploader": kwargs.get("uploaders"),
155-
}
156-
157-
for name, values in list_filters.items():
158-
if values is not None:
159-
api_call += f"/{name}/" + ",".join(str(int(v)) for v in values)
160-
161-
# Single-value filters
162-
if kwargs.get("study") is not None:
163-
api_call += f"/study/{kwargs['study']}"
164-
165-
if kwargs.get("sort_order") is not None:
166-
api_call += f"/sort_order/{kwargs['sort_order']}"
167-
168-
# Extra filters (tag, per_fold, future-proof)
169-
for key in ("tag", "per_fold"):
170-
value = kwargs.get(key)
171-
if value is not None:
172-
api_call += f"/{key}/{value}"
173-
174-
return api_call
175-
176249

177250
class EvaluationsV2(EvaluationsAPI):
178251
"""V2 API implementation for evaluations.
179252
Fetches evaluations from the v2 json API endpoint.
180253
"""
181254

182-
def list(
255+
def list( # noqa: PLR0913
183256
self,
184257
limit: int,
185258
offset: int,
259+
*,
186260
function: str,
261+
tasks: list | None = None,
262+
setups: list | None = None,
263+
flows: list | None = None,
264+
runs: list | None = None,
265+
uploaders: list | None = None,
266+
study: int | None = None,
267+
sort_order: str | None = None,
187268
**kwargs: Any,
188-
) -> dict:
269+
) -> list[OpenMLEvaluation]:
189270
"""
190271
Retrieve evaluation results from the OpenML v2 JSON API.
191272

0 commit comments

Comments
 (0)