diff --git a/src/main/java/com/treasuredata/client/TDClient.java b/src/main/java/com/treasuredata/client/TDClient.java index 306cf76a..266ac793 100644 --- a/src/main/java/com/treasuredata/client/TDClient.java +++ b/src/main/java/com/treasuredata/client/TDClient.java @@ -43,7 +43,6 @@ import com.treasuredata.client.model.TDJobRequest; import com.treasuredata.client.model.TDJobSubmitResult; import com.treasuredata.client.model.TDJobSummary; -import com.treasuredata.client.model.TDPartialDeleteJob; import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDSaveQueryRequest; import com.treasuredata.client.model.TDSavedQuery; @@ -564,33 +563,6 @@ public void deleteTableIfExists(String databaseName, String tableName) } } - @Override - public TDPartialDeleteJob partialDelete(String databaseName, String tableName, long from, long to) - throws TDClientException - { - return partialDelete(databaseName, tableName, from, to, null); - } - - @Override - public TDPartialDeleteJob partialDelete(String databaseName, String tableName, long from, long to, String domainKey) - throws TDClientException - { - if ((from % 3600 != 0) || (to % 3600 != 0)) { - throw new TDClientException(TDClientException.ErrorType.INVALID_INPUT, String.format("from/to value must be a multiple of 3600: [%s, %s)", from, to)); - } - - Map queryParams = new HashMap<>(); - queryParams.put("from", Long.toString(from)); - queryParams.put("to", Long.toString(to)); - - if (domainKey != null) { - queryParams.put("domain_key", domainKey); - } - - TDPartialDeleteJob job = doPost(buildUrl("/v3/table/partialdelete", databaseName, tableName), Collections.unmodifiableMap(queryParams), TDPartialDeleteJob.class); - return job; - } - @Override public void swapTables(String databaseName, String tableName1, String tableName2) { diff --git a/src/main/java/com/treasuredata/client/TDClientApi.java b/src/main/java/com/treasuredata/client/TDClientApi.java index 7cbf86c4..7093431d 100644 --- a/src/main/java/com/treasuredata/client/TDClientApi.java +++ b/src/main/java/com/treasuredata/client/TDClientApi.java @@ -33,7 +33,6 @@ import com.treasuredata.client.model.TDJobList; import com.treasuredata.client.model.TDJobRequest; import com.treasuredata.client.model.TDJobSummary; -import com.treasuredata.client.model.TDPartialDeleteJob; import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDSaveQueryRequest; import com.treasuredata.client.model.TDSavedQuery; @@ -215,10 +214,6 @@ public interface TDClientApi void deleteTableIfExists(String databaseName, String tableName); - TDPartialDeleteJob partialDelete(String databaseName, String tableName, long from, long to); - - TDPartialDeleteJob partialDelete(String databaseName, String tableName, long from, long to, String domainKey); - void swapTables(String databaseName, String tableName1, String tableName2); // schema API diff --git a/src/main/java/com/treasuredata/client/model/TDPartialDeleteJob.java b/src/main/java/com/treasuredata/client/model/TDPartialDeleteJob.java deleted file mode 100644 index 83fcfdb1..00000000 --- a/src/main/java/com/treasuredata/client/model/TDPartialDeleteJob.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.treasuredata.client.model; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * - */ -public class TDPartialDeleteJob -{ - private final String jobId; - private final String database; - private final String table; - private final long from; - private final long to; - - public TDPartialDeleteJob( - @JsonProperty("job_id") String jobId, - @JsonProperty("database") String database, - @JsonProperty("table") String table, - @JsonProperty("long") long from, - @JsonProperty("to") long to) - { - this.jobId = jobId; - this.database = database; - this.table = table; - this.from = from; - this.to = to; - } - - public String getJobId() - { - return jobId; - } - - public String getDatabase() - { - return database; - } - - public String getTable() - { - return table; - } - - public long getFrom() - { - return from; - } - - public long getTo() - { - return to; - } - - @Override - public String toString() - { - return "TDPartialDeleteJob{" + - "jobId='" + jobId + '\'' + - ", database='" + database + '\'' + - ", table='" + table + '\'' + - ", from=" + from + - ", to=" + to + - '}'; - } -} diff --git a/src/test/java/com/treasuredata/client/TestTDClient.java b/src/test/java/com/treasuredata/client/TestTDClient.java index 241a9c1e..1a5391a7 100644 --- a/src/test/java/com/treasuredata/client/TestTDClient.java +++ b/src/test/java/com/treasuredata/client/TestTDClient.java @@ -41,7 +41,6 @@ import com.treasuredata.client.model.TDJobRequest; import com.treasuredata.client.model.TDJobRequestBuilder; import com.treasuredata.client.model.TDJobSummary; -import com.treasuredata.client.model.TDPartialDeleteJob; import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDSaveQueryRequest; import com.treasuredata.client.model.TDSavedQuery; @@ -1001,85 +1000,6 @@ private static String newTemporaryName(String prefix) return prefix + "_" + dateStr; } - @Test - public void partialDeleteTest() - throws Exception - { - String t = newTemporaryName("td_client_test"); - try { - client.deleteTableIfExists(SAMPLE_DB, t); - - String jobId = client.submit(TDJobRequest.newPrestoQuery(SAMPLE_DB, - String.format("CREATE TABLE %s AS SELECT * FROM (VALUES TD_TIME_PARSE('2015-01-01', 'UTC'), TD_TIME_PARSE('2015-02-01', 'UTC')) as sample(time)", t, t))); - - waitJobCompletion(jobId); - - String before = queryResult(SAMPLE_DB, String.format("SELECT * FROM %s", t)); - - assertTrue(before.contains("1420070400")); - assertTrue(before.contains("1422748800")); - - // delete 2015-01-01 entry - try { - client.partialDelete(SAMPLE_DB, t, 1420070400, 1420070400 + 1); - fail("should not reach here"); - } - catch (TDClientException e) { - assertEquals(TDClientException.ErrorType.INVALID_INPUT, e.getErrorType()); - } - long from = 1420070400 - (1420070400 % 3600); - long to = from + 3600; - TDPartialDeleteJob partialDeleteJob = client.partialDelete(SAMPLE_DB, t, from, to); - logger.debug("partial delete job: " + partialDeleteJob); - assertEquals(from, partialDeleteJob.getFrom()); - assertEquals(to, partialDeleteJob.getTo()); - assertEquals(SAMPLE_DB, partialDeleteJob.getDatabase()); - assertEquals(t, partialDeleteJob.getTable()); - - waitJobCompletion(partialDeleteJob.getJobId()); - - String after = queryResult(SAMPLE_DB, String.format("SELECT * FROM %s", t)); - assertFalse(after.contains("1420070400")); - assertTrue(after.contains("1422748800")); - } - finally { - client.deleteTableIfExists(SAMPLE_DB, t); - } - } - - @Test - public void partialDeleteWithDomainKeyTest() - throws Exception - { - String domainKey = randomDomainKey(); - - String t = newTemporaryName("td_client_test"); - try { - client.deleteTableIfExists(SAMPLE_DB, t); - - String jobId = client.submit(TDJobRequest.newPrestoQuery(SAMPLE_DB, - String.format("CREATE TABLE %s AS SELECT * FROM (VALUES TD_TIME_PARSE('2015-01-01', 'UTC'), TD_TIME_PARSE('2015-02-01', 'UTC')) as sample(time)", t, t))); - - waitJobCompletion(jobId); - - int from = 1420070400; - int to = from + 3600; - - TDPartialDeleteJob deleteJob = client.partialDelete(SAMPLE_DB, t, from, to, domainKey); - - try { - client.partialDelete(SAMPLE_DB, t, from, to, domainKey); - fail("Expected " + TDClientHttpConflictException.class.getName()); - } - catch (TDClientHttpConflictException e) { - assertThat(e.getConflictsWith(), is(Optional.of(deleteJob.getJobId()))); - } - } - finally { - client.deleteTableIfExists(SAMPLE_DB, t); - } - } - @Test public void swapTest() throws Exception @@ -1248,9 +1168,23 @@ public void testBulkImport() } // Check the data + // It seems it needs some time to reflect the data in TD, + // so we will wait for few mins before checking the data + deadline = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5); + TDTable imported = client.listTables(SAMPLE_DB).stream().filter(input -> { return input.getName().equals(bulkImportTable); }).findFirst().get(); + while (imported.getRowCount() == 0) { + if (System.currentTimeMillis() > deadline) { + throw new IllegalStateException("timeout error: data is not imported yet"); + } + logger.info("Waiting data import step completion"); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + imported = client.listTables(SAMPLE_DB).stream().filter(input -> { + return input.getName().equals(bulkImportTable); + }).findFirst().get(); + } assertEquals(numRowsInPart * 2, imported.getRowCount()); List columns = imported.getColumns();