From 1b541a483b89d62d4adc32290a4c5dc6a29787d8 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Mon, 17 Feb 2025 17:36:54 +0100 Subject: [PATCH 1/6] Add logic and tests for ListJobs --- .../com/databricks/sdk/mixin/JobsExt.java | 29 +++++ .../com/databricks/sdk/mixin/JobsExtTest.java | 105 ++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java index 8dfd0a7a8..ebb094eb6 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java @@ -3,6 +3,7 @@ import com.databricks.sdk.core.ApiClient; import com.databricks.sdk.service.jobs.*; import java.util.Collection; +import java.util.Iterator; public class JobsExt extends JobsAPI { @@ -14,6 +15,34 @@ public JobsExt(JobsService mock) { super(mock); } + public Iterable list(ListJobsRequest request) { + Iterable jobsList = super.list(request); + + if (!request.getExpandTasks()) { + return jobsList; + } + + Iterator iterator = jobsList.iterator(); + return () -> new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public BaseJob next() { + BaseJob job = iterator.next(); + GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); + Job fullJob = get(getJobRequest); + job.getSettings().setTasks(fullJob.getSettings().getTasks()); + job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); + job.getSettings().setParameters(fullJob.getSettings().getParameters()); + job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); + return job; + } + }; + } + /** * Wrap the {@code JobsApi.getRun} operation to retrieve paginated content without breaking the * response contract. diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java index 46b04d912..6068a0876 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java @@ -202,4 +202,109 @@ private void addJobEnvironments(Job job, String... environmentKeys) { } job.getSettings().setEnvironments(environments); } + + @Test + public void testListJobsWithoutExpandTasks() { + JobsService service = Mockito.mock(JobsService.class); + BaseJob job1 = new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + BaseJob job2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + BaseJob job3 = new BaseJob().setJobId(300L).setSettings(new JobSettings().setName("job3")); + BaseJob job4 = new BaseJob().setJobId(400L).setSettings(new JobSettings().setName("job4")); + + List jobsOnFirstPage = new ArrayList<>(); + jobsOnFirstPage.add(job1); + jobsOnFirstPage.add(job2); + List jobsOnSecondPage = new ArrayList<>(); + jobsOnSecondPage.add(job3); + jobsOnSecondPage.add(job4); + when(service.list(any())) + .thenReturn(new ListJobsResponse().setJobs(jobsOnFirstPage)) + .thenReturn(new ListJobsResponse().setJobs(jobsOnSecondPage)); + JobsExt jobsExt = new JobsExt(service); + + ListJobsRequest request = new ListJobsRequest().setExpandTasks(false); + Iterable jobsList = jobsExt.list(request); + + List expectedJobsList = new ArrayList<>(); + expectedJobsList.add(job1); + expectedJobsList.add(job2); + expectedJobsList.add(job3); + expectedJobsList.add(job4); + for (BaseJob job : jobsList) { + BaseJob expectedJob = expectedJobsList.stream() + .filter(e -> e.getJobId().equals(job.getJobId())) + .findFirst() + .orElse(null); + assertEquals(expectedJob, job); + } + verify(service, times(0)).get(any()); + } + + @Test + public void testListJobs() { + JobsService service = Mockito.mock(JobsService.class); + BaseJob job1 = new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + addTasks(job1, "job1_taskKey1", "job1_taskKey2"); + BaseJob job2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + addTasks(job2, "job2_taskKey1", "job2_taskKey2"); + + Job getJob1_page1 = new Job() + .setJobId(100L) + .setNextPageToken("job1_page2token") + .setSettings(new JobSettings().setName("job1")); + addTasks(getJob1_page1, "job1_taskKey1", "job1_taskKey2"); + Job getJob1_page2 = new Job() + .setJobId(100L) + .setSettings(new JobSettings().setName("job1")); + addTasks(getJob1_page2, "job1_taskKey3", "job1_taskKey4"); + + Job getJob2_page1 = new Job() + .setJobId(200L) + .setNextPageToken("job2_page2token") + .setSettings(new JobSettings().setName("job2")); + addTasks(getJob2_page1, "job2_taskKey1", "job2_taskKey2"); + Job getJob2_page2 = new Job() + .setJobId(200L) + .setSettings(new JobSettings().setName("job2")); + addTasks(getJob2_page2, "job2_taskKey3", "job2_taskKey4"); + + doReturn(getJob1_page1).when(service).get(argThat(request -> request != null && request.getJobId() == 100L && request.getPageToken() == null)); + doReturn(getJob1_page2).when(service).get(argThat(request -> request != null && request.getJobId() == 100L && "job1_page2token".equals(request.getPageToken()))); + doReturn(getJob2_page1).when(service).get(argThat(request -> request != null && request.getJobId() == 200L && request.getPageToken() == null)); + doReturn(getJob2_page2).when(service).get(argThat(request -> request != null && request.getJobId() == 200L && "job2_page2token".equals(request.getPageToken()))); + + List jobsOnFirstPage = new ArrayList<>(); + jobsOnFirstPage.add(job1); + jobsOnFirstPage.add(job2); + ListJobsResponse response = new ListJobsResponse().setJobs(jobsOnFirstPage); + when(service.list(any())).thenReturn(response); + JobsExt jobsExt = new JobsExt(service); + + ListJobsRequest request = new ListJobsRequest().setExpandTasks(true); + Iterable jobsList = jobsExt.list(request); + + BaseJob expectedJob1 = new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + addTasks(expectedJob1, "job1_taskKey1", "job1_taskKey2", "job1_taskKey3", "job1_taskKey4"); + BaseJob expectedJob2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + addTasks(expectedJob2, "job2_taskKey1", "job2_taskKey2", "job2_taskKey3", "job2_taskKey4"); + List expectedJobsList = new ArrayList<>(); + expectedJobsList.add(expectedJob1); + expectedJobsList.add(expectedJob2); + for (BaseJob job : jobsList) { + BaseJob expectedJob = expectedJobsList.stream() + .filter(e -> e.getJobId().equals(job.getJobId())) + .findFirst() + .orElse(null); + assertEquals(expectedJob, job); + } + verify(service, times(4)).get(any()); + } + + private void addTasks(BaseJob job, String... taskKeys) { + Collection tasks = new ArrayList<>(); + for (String taskKey : taskKeys) { + tasks.add(new Task().setTaskKey(taskKey)); + } + job.getSettings().setTasks(tasks); + } } From 6142aedc715a0ecdb705e68d2eeef86b2397b349 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Mon, 17 Feb 2025 18:03:39 +0100 Subject: [PATCH 2/6] Add comments --- .../src/main/java/com/databricks/sdk/mixin/JobsExt.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java index ebb094eb6..ee2d812af 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java @@ -15,7 +15,14 @@ public JobsExt(JobsService mock) { super(mock); } + /** + * List jobs. + * + *

Retrieves a list of jobs. If the job has multiple pages of tasks, job_clusters, parameters or environments, + * it will paginate through all pages and aggregate the results. + */ public Iterable list(ListJobsRequest request) { + // fetch jobs with limited elements in top level arrays Iterable jobsList = super.list(request); if (!request.getExpandTasks()) { @@ -32,6 +39,7 @@ public boolean hasNext() { @Override public BaseJob next() { BaseJob job = iterator.next(); + // fully fetch all top level arrays for the job GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); Job fullJob = get(getJobRequest); job.getSettings().setTasks(fullJob.getSettings().getTasks()); From 8f7c832820d0d6ff69f46057ba4b653896b9b25a Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Mon, 17 Feb 2025 18:04:50 +0100 Subject: [PATCH 3/6] fmt --- .../com/databricks/sdk/mixin/JobsExt.java | 45 ++++++------- .../com/databricks/sdk/mixin/JobsExtTest.java | 66 ++++++++++++++----- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java index ee2d812af..4014217e8 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java @@ -18,37 +18,38 @@ public JobsExt(JobsService mock) { /** * List jobs. * - *

Retrieves a list of jobs. If the job has multiple pages of tasks, job_clusters, parameters or environments, - * it will paginate through all pages and aggregate the results. + *

Retrieves a list of jobs. If the job has multiple pages of tasks, job_clusters, parameters + * or environments, it will paginate through all pages and aggregate the results. */ public Iterable list(ListJobsRequest request) { // fetch jobs with limited elements in top level arrays - Iterable jobsList = super.list(request); + Iterable jobsList = super.list(request); if (!request.getExpandTasks()) { return jobsList; } Iterator iterator = jobsList.iterator(); - return () -> new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public BaseJob next() { - BaseJob job = iterator.next(); - // fully fetch all top level arrays for the job - GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); - Job fullJob = get(getJobRequest); - job.getSettings().setTasks(fullJob.getSettings().getTasks()); - job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); - job.getSettings().setParameters(fullJob.getSettings().getParameters()); - job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); - return job; - } - }; + return () -> + new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public BaseJob next() { + BaseJob job = iterator.next(); + // fully fetch all top level arrays for the job + GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); + Job fullJob = get(getJobRequest); + job.getSettings().setTasks(fullJob.getSettings().getTasks()); + job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); + job.getSettings().setParameters(fullJob.getSettings().getParameters()); + job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); + return job; + } + }; } /** diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java index 6068a0876..f014dd739 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java @@ -218,8 +218,8 @@ public void testListJobsWithoutExpandTasks() { jobsOnSecondPage.add(job3); jobsOnSecondPage.add(job4); when(service.list(any())) - .thenReturn(new ListJobsResponse().setJobs(jobsOnFirstPage)) - .thenReturn(new ListJobsResponse().setJobs(jobsOnSecondPage)); + .thenReturn(new ListJobsResponse().setJobs(jobsOnFirstPage)) + .thenReturn(new ListJobsResponse().setJobs(jobsOnSecondPage)); JobsExt jobsExt = new JobsExt(service); ListJobsRequest request = new ListJobsRequest().setExpandTasks(false); @@ -231,7 +231,8 @@ public void testListJobsWithoutExpandTasks() { expectedJobsList.add(job3); expectedJobsList.add(job4); for (BaseJob job : jobsList) { - BaseJob expectedJob = expectedJobsList.stream() + BaseJob expectedJob = + expectedJobsList.stream() .filter(e -> e.getJobId().equals(job.getJobId())) .findFirst() .orElse(null); @@ -248,30 +249,56 @@ public void testListJobs() { BaseJob job2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); addTasks(job2, "job2_taskKey1", "job2_taskKey2"); - Job getJob1_page1 = new Job() + Job getJob1_page1 = + new Job() .setJobId(100L) .setNextPageToken("job1_page2token") .setSettings(new JobSettings().setName("job1")); addTasks(getJob1_page1, "job1_taskKey1", "job1_taskKey2"); - Job getJob1_page2 = new Job() - .setJobId(100L) - .setSettings(new JobSettings().setName("job1")); + Job getJob1_page2 = new Job().setJobId(100L).setSettings(new JobSettings().setName("job1")); addTasks(getJob1_page2, "job1_taskKey3", "job1_taskKey4"); - Job getJob2_page1 = new Job() + Job getJob2_page1 = + new Job() .setJobId(200L) .setNextPageToken("job2_page2token") .setSettings(new JobSettings().setName("job2")); addTasks(getJob2_page1, "job2_taskKey1", "job2_taskKey2"); - Job getJob2_page2 = new Job() - .setJobId(200L) - .setSettings(new JobSettings().setName("job2")); + Job getJob2_page2 = new Job().setJobId(200L).setSettings(new JobSettings().setName("job2")); addTasks(getJob2_page2, "job2_taskKey3", "job2_taskKey4"); - doReturn(getJob1_page1).when(service).get(argThat(request -> request != null && request.getJobId() == 100L && request.getPageToken() == null)); - doReturn(getJob1_page2).when(service).get(argThat(request -> request != null && request.getJobId() == 100L && "job1_page2token".equals(request.getPageToken()))); - doReturn(getJob2_page1).when(service).get(argThat(request -> request != null && request.getJobId() == 200L && request.getPageToken() == null)); - doReturn(getJob2_page2).when(service).get(argThat(request -> request != null && request.getJobId() == 200L && "job2_page2token".equals(request.getPageToken()))); + doReturn(getJob1_page1) + .when(service) + .get( + argThat( + request -> + request != null + && request.getJobId() == 100L + && request.getPageToken() == null)); + doReturn(getJob1_page2) + .when(service) + .get( + argThat( + request -> + request != null + && request.getJobId() == 100L + && "job1_page2token".equals(request.getPageToken()))); + doReturn(getJob2_page1) + .when(service) + .get( + argThat( + request -> + request != null + && request.getJobId() == 200L + && request.getPageToken() == null)); + doReturn(getJob2_page2) + .when(service) + .get( + argThat( + request -> + request != null + && request.getJobId() == 200L + && "job2_page2token".equals(request.getPageToken()))); List jobsOnFirstPage = new ArrayList<>(); jobsOnFirstPage.add(job1); @@ -283,15 +310,18 @@ public void testListJobs() { ListJobsRequest request = new ListJobsRequest().setExpandTasks(true); Iterable jobsList = jobsExt.list(request); - BaseJob expectedJob1 = new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + BaseJob expectedJob1 = + new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); addTasks(expectedJob1, "job1_taskKey1", "job1_taskKey2", "job1_taskKey3", "job1_taskKey4"); - BaseJob expectedJob2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + BaseJob expectedJob2 = + new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); addTasks(expectedJob2, "job2_taskKey1", "job2_taskKey2", "job2_taskKey3", "job2_taskKey4"); List expectedJobsList = new ArrayList<>(); expectedJobsList.add(expectedJob1); expectedJobsList.add(expectedJob2); for (BaseJob job : jobsList) { - BaseJob expectedJob = expectedJobsList.stream() + BaseJob expectedJob = + expectedJobsList.stream() .filter(e -> e.getJobId().equals(job.getJobId())) .findFirst() .orElse(null); From 29e3cc1cb32c7046e101aaacfbc44fa5dcfed1c7 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Mon, 17 Feb 2025 20:24:01 +0100 Subject: [PATCH 4/6] Set hasMore to false --- .../com/databricks/sdk/mixin/JobsExt.java | 1 + .../com/databricks/sdk/mixin/JobsExtTest.java | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java index 4014217e8..2a0404516 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java @@ -47,6 +47,7 @@ public BaseJob next() { job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); job.getSettings().setParameters(fullJob.getSettings().getParameters()); job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); + job.setHasMore(false); return job; } }; diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java index f014dd739..378eeda48 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java @@ -244,9 +244,17 @@ public void testListJobsWithoutExpandTasks() { @Test public void testListJobs() { JobsService service = Mockito.mock(JobsService.class); - BaseJob job1 = new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + BaseJob job1 = + new BaseJob() + .setJobId(100L) + .setSettings(new JobSettings().setName("job1")) + .setHasMore(true); addTasks(job1, "job1_taskKey1", "job1_taskKey2"); - BaseJob job2 = new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + BaseJob job2 = + new BaseJob() + .setJobId(200L) + .setSettings(new JobSettings().setName("job2")) + .setHasMore(true); addTasks(job2, "job2_taskKey1", "job2_taskKey2"); Job getJob1_page1 = @@ -311,10 +319,16 @@ public void testListJobs() { Iterable jobsList = jobsExt.list(request); BaseJob expectedJob1 = - new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); + new BaseJob() + .setJobId(100L) + .setSettings(new JobSettings().setName("job1")) + .setHasMore(false); addTasks(expectedJob1, "job1_taskKey1", "job1_taskKey2", "job1_taskKey3", "job1_taskKey4"); BaseJob expectedJob2 = - new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); + new BaseJob() + .setJobId(200L) + .setSettings(new JobSettings().setName("job2")) + .setHasMore(false); addTasks(expectedJob2, "job2_taskKey1", "job2_taskKey2", "job2_taskKey3", "job2_taskKey4"); List expectedJobsList = new ArrayList<>(); expectedJobsList.add(expectedJob1); From 878f970e8427db4ba85496e827a387195cb0cfa0 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Thu, 20 Feb 2025 16:34:31 +0100 Subject: [PATCH 5/6] Set has_next to null --- .../com/databricks/sdk/mixin/JobsExt.java | 24 ++++++++++------ .../com/databricks/sdk/mixin/JobsExtTest.java | 28 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java index 2a0404516..fea5e6733 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/JobsExt.java @@ -40,14 +40,22 @@ public boolean hasNext() { @Override public BaseJob next() { BaseJob job = iterator.next(); - // fully fetch all top level arrays for the job - GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); - Job fullJob = get(getJobRequest); - job.getSettings().setTasks(fullJob.getSettings().getTasks()); - job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); - job.getSettings().setParameters(fullJob.getSettings().getParameters()); - job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); - job.setHasMore(false); + + // The has_more field is only present in jobs with 100+ tasks, that is served from Jobs + // API 2.2. + // Extra tasks and other fields need to be fetched only when has_more is true. + if (job.getHasMore() != null && job.getHasMore()) { + // fully fetch all top level arrays for the job + GetJobRequest getJobRequest = new GetJobRequest().setJobId(job.getJobId()); + Job fullJob = get(getJobRequest); + job.getSettings().setTasks(fullJob.getSettings().getTasks()); + job.getSettings().setJobClusters(fullJob.getSettings().getJobClusters()); + job.getSettings().setParameters(fullJob.getSettings().getParameters()); + job.getSettings().setEnvironments(fullJob.getSettings().getEnvironments()); + } + // Set the has_more field to false to indicate that there are no more tasks and other + // fields to fetch. + job.setHasMore(null); return job; } }; diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java index 378eeda48..c4bb49033 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/JobsExtTest.java @@ -1,6 +1,7 @@ package com.databricks.sdk.mixin; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -256,6 +257,8 @@ public void testListJobs() { .setSettings(new JobSettings().setName("job2")) .setHasMore(true); addTasks(job2, "job2_taskKey1", "job2_taskKey2"); + BaseJob job3 = new BaseJob().setJobId(300L).setSettings(new JobSettings().setName("job3")); + addTasks(job3, "job3_taskKey1", "job3_taskKey2"); Job getJob1_page1 = new Job() @@ -311,28 +314,31 @@ public void testListJobs() { List jobsOnFirstPage = new ArrayList<>(); jobsOnFirstPage.add(job1); jobsOnFirstPage.add(job2); - ListJobsResponse response = new ListJobsResponse().setJobs(jobsOnFirstPage); - when(service.list(any())).thenReturn(response); + ListJobsResponse responsePage1 = + new ListJobsResponse().setJobs(jobsOnFirstPage).setNextPageToken("page2token"); + List jobsOnSecondPage = new ArrayList<>(); + jobsOnSecondPage.add(job3); + ListJobsResponse responsePage2 = new ListJobsResponse().setJobs(jobsOnSecondPage); + JobsExt jobsExt = new JobsExt(service); + when(service.list(any())).thenReturn(responsePage1).thenReturn(responsePage2); ListJobsRequest request = new ListJobsRequest().setExpandTasks(true); Iterable jobsList = jobsExt.list(request); BaseJob expectedJob1 = - new BaseJob() - .setJobId(100L) - .setSettings(new JobSettings().setName("job1")) - .setHasMore(false); + new BaseJob().setJobId(100L).setSettings(new JobSettings().setName("job1")); addTasks(expectedJob1, "job1_taskKey1", "job1_taskKey2", "job1_taskKey3", "job1_taskKey4"); BaseJob expectedJob2 = - new BaseJob() - .setJobId(200L) - .setSettings(new JobSettings().setName("job2")) - .setHasMore(false); + new BaseJob().setJobId(200L).setSettings(new JobSettings().setName("job2")); addTasks(expectedJob2, "job2_taskKey1", "job2_taskKey2", "job2_taskKey3", "job2_taskKey4"); + BaseJob expectedJob3 = + new BaseJob().setJobId(300L).setSettings(new JobSettings().setName("job3")); + addTasks(expectedJob3, "job3_taskKey1", "job3_taskKey2"); List expectedJobsList = new ArrayList<>(); expectedJobsList.add(expectedJob1); expectedJobsList.add(expectedJob2); + expectedJobsList.add(expectedJob3); for (BaseJob job : jobsList) { BaseJob expectedJob = expectedJobsList.stream() @@ -340,7 +346,9 @@ public void testListJobs() { .findFirst() .orElse(null); assertEquals(expectedJob, job); + assertNull(expectedJob.getHasMore()); } + // 2 getRun calls for job1, 2 getRun calls for job2, 0 getRun call for job3 verify(service, times(4)).get(any()); } From ee8f619e3e9aa676e06d449107ed6b458ea41662 Mon Sep 17 00:00:00 2001 From: Giorgi Kikolashvili Date: Thu, 20 Feb 2025 16:41:54 +0100 Subject: [PATCH 6/6] Add changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 584443e52..8a67400d9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Documentation ### Internal Changes +* Update Jobs ListJobs API to support paginated responses ([#411](https://github.com/databricks/databricks-sdk-java/pull/411)) * Introduce automated tagging ([#409](https://github.com/databricks/databricks-sdk-java/pull/409)). * Update Jobs GetJob API to support paginated responses ([#403](https://github.com/databricks/databricks-sdk-java/pull/403)). * Update Jobs GetRun API to support paginated responses ([#402](https://github.com/databricks/databricks-sdk-java/pull/402)).