diff --git a/.github/workflows/cleanup-test-resources.yml b/.github/workflows/cleanup-test-resources.yml
new file mode 100644
index 00000000..6d405cb5
--- /dev/null
+++ b/.github/workflows/cleanup-test-resources.yml
@@ -0,0 +1,76 @@
+name: Cleanup Test Resources
+
+permissions:
+ contents: read
+
+on:
+ # Scheduled execution - daily at 2 AM UTC
+ schedule:
+ - cron: '0 2 * * *'
+
+ # Manual trigger with optional inputs
+ workflow_dispatch:
+ inputs:
+ age_threshold_days:
+ description: 'Minimum age in days for resources to be deleted'
+ required: false
+ default: '1'
+ type: number
+ dry_run:
+ description: 'Preview deletions without executing (dry-run mode)'
+ required: false
+ default: false
+ type: boolean
+
+jobs:
+ cleanup:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up JDK 8
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '8'
+ cache: 'gradle'
+
+ - name: Build project
+ run: ./gradlew build -x test
+
+ - name: Run cleanup utility
+ env:
+ PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+ run: |
+ # Determine parameters based on trigger type
+ if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+ AGE_THRESHOLD=${{ inputs.age_threshold_days }}
+ DRY_RUN=${{ inputs.dry_run }}
+ else
+ # Scheduled run uses default values
+ AGE_THRESHOLD=1
+ DRY_RUN=false
+ fi
+
+ # Build command with arguments
+ ARGS="--age-threshold-days $AGE_THRESHOLD"
+ if [ "$DRY_RUN" = "true" ]; then
+ ARGS="$ARGS --dry-run"
+ fi
+
+ echo "Running cleanup with: $ARGS"
+
+ # Run the cleanup utility
+ java -cp "build/libs/*:build/classes/java/main" \
+ io.pinecone.helpers.IndexCleanupUtility $ARGS
+
+ - name: Summary
+ if: always()
+ run: |
+ if [ "${{ job.status }}" = "success" ]; then
+ echo "✅ Cleanup completed successfully"
+ else
+ echo "❌ Cleanup failed - check logs for details"
+ fi
diff --git a/build.gradle b/build.gradle
index 738a537a..1c055651 100644
--- a/build.gradle
+++ b/build.gradle
@@ -70,6 +70,7 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat
import org.gradle.api.tasks.testing.logging.TestLogEvent
tasks.withType(Test) {
+ systemProperty "net.bytebuddy.experimental", "true"
testLogging {
// set options for log level LIFECYCLE
events TestLogEvent.FAILED,
diff --git a/src/main/java/io/pinecone/helpers/IndexCleanupUtility.java b/src/main/java/io/pinecone/helpers/IndexCleanupUtility.java
index 72d73d3d..c71cf878 100644
--- a/src/main/java/io/pinecone/helpers/IndexCleanupUtility.java
+++ b/src/main/java/io/pinecone/helpers/IndexCleanupUtility.java
@@ -1,38 +1,302 @@
package io.pinecone.helpers;
import io.pinecone.clients.Pinecone;
+import org.openapitools.db_control.client.model.CollectionList;
+import org.openapitools.db_control.client.model.CollectionModel;
+import org.openapitools.db_control.client.model.IndexList;
import org.openapitools.db_control.client.model.IndexModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Utility for cleaning up Pinecone indexes and collections.
+ *
+ * This utility can be used to clean up resources in a Pinecone project, with support for:
+ * - Deleting all indexes and collections
+ * - Dry-run mode to preview deletions without executing them
+ * - Automatic handling of deletion protection
+ * - Age-based filtering (when timestamp information becomes available)
+ *
+ * Command-line arguments:
+ * --age-threshold-days <number> - Minimum age in days for resources to be deleted (default: 1)
+ * --dry-run - Preview deletions without actually deleting resources
+ *
+ * Example usage:
+ * {@code
+ * java io.pinecone.helpers.IndexCleanupUtility --age-threshold-days 2 --dry-run
+ * }
+ */
public class IndexCleanupUtility {
private static final Logger logger = LoggerFactory.getLogger(IndexCleanupUtility.class);
+ private final Pinecone pinecone;
+ private final int ageThresholdDays;
+ private final boolean dryRun;
+
+ /**
+ * Constructs a new IndexCleanupUtility.
+ *
+ * @param pinecone The Pinecone client instance
+ * @param ageThresholdDays Minimum age in days for resources to be deleted
+ * @param dryRun If true, preview deletions without executing them
+ */
+ public IndexCleanupUtility(Pinecone pinecone, int ageThresholdDays, boolean dryRun) {
+ this.pinecone = pinecone;
+ this.ageThresholdDays = ageThresholdDays;
+ this.dryRun = dryRun;
+ }
+
+ /**
+ * Main entry point for the cleanup utility.
+ *
+ * @param args Command-line arguments
+ */
public static void main(String[] args) {
try {
- logger.info("Starting Pinecone index cleanup...");
- Pinecone pinecone = new Pinecone.Builder(System.getenv("PINECONE_API_KEY")).build();
+ // Parse command-line arguments
+ int ageThresholdDays = 1; // Default: 1 day
+ boolean dryRun = false;
- for(IndexModel model : pinecone.listIndexes().getIndexes()) {
- String indexName = model.getName();
- if(model.getDeletionProtection().equals("enabled")) {
+ for (int i = 0; i < args.length; i++) {
+ if ("--age-threshold-days".equals(args[i]) && i + 1 < args.length) {
try {
- model.getSpec().getIndexModelPodBased();
- pinecone.configurePodsIndex(indexName, "disabled");
- } catch (ClassCastException e) {
- // Not a pod-based index, continue
+ ageThresholdDays = Integer.parseInt(args[i + 1]);
+ i++; // Skip the next argument since we've consumed it
+ } catch (NumberFormatException e) {
+ logger.error("Invalid value for --age-threshold-days: {}", args[i + 1]);
+ printUsage();
+ System.exit(1);
}
- pinecone.configureServerlessIndex(indexName, "disabled", null, null);
+ } else if ("--dry-run".equals(args[i])) {
+ dryRun = true;
+ } else {
+ logger.warn("Unknown argument: {}", args[i]);
}
- Thread.sleep(5000);
- pinecone.deleteIndex(indexName);
}
- logger.info("Index cleanup completed");
+ logger.info("Starting Pinecone resource cleanup...");
+ logger.info("Age threshold: {} days", ageThresholdDays);
+ logger.info("Dry-run mode: {}", dryRun);
+
+ // Initialize Pinecone client
+ String apiKey = System.getenv("PINECONE_API_KEY");
+ if (apiKey == null || apiKey.isEmpty()) {
+ logger.error("PINECONE_API_KEY environment variable is not set");
+ System.exit(1);
+ }
+
+ Pinecone pinecone = new Pinecone.Builder(apiKey).build();
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, ageThresholdDays, dryRun);
+
+ // Execute cleanup
+ CleanupResult result = utility.cleanup();
+
+ // Log summary
+ logger.info("=== Cleanup Summary ===");
+ logger.info("Indexes processed: {}", result.getIndexesProcessed());
+ logger.info("Indexes deleted: {}", result.getIndexesDeleted());
+ logger.info("Indexes failed: {}", result.getIndexesFailed());
+ logger.info("Collections processed: {}", result.getCollectionsProcessed());
+ logger.info("Collections deleted: {}", result.getCollectionsDeleted());
+ logger.info("Collections failed: {}", result.getCollectionsFailed());
+
+ if (dryRun) {
+ logger.info("DRY-RUN MODE: No resources were actually deleted");
+ }
+
+ logger.info("Cleanup completed");
+
+ // Exit with error code if any deletions failed
+ if (result.getIndexesFailed() > 0 || result.getCollectionsFailed() > 0) {
+ System.exit(1);
+ }
} catch (Exception e) {
logger.error("Error during cleanup: {}", e.getMessage(), e);
System.exit(1);
}
}
+
+ private static void printUsage() {
+ System.err.println("Usage: java io.pinecone.helpers.IndexCleanupUtility [OPTIONS]");
+ System.err.println("Options:");
+ System.err.println(" --age-threshold-days Minimum age in days for resources to be deleted (default: 1)");
+ System.err.println(" --dry-run Preview deletions without executing them");
+ }
+
+ /**
+ * Executes the cleanup operation, deleting indexes and collections that match the criteria.
+ *
+ * @return A CleanupResult containing statistics about the cleanup operation
+ * @throws Exception if an error occurs during cleanup
+ */
+ public CleanupResult cleanup() throws Exception {
+ CleanupResult result = new CleanupResult();
+
+ // Clean up indexes
+ logger.info("Listing indexes...");
+ IndexList indexList = pinecone.listIndexes();
+ List indexes = indexList != null ? indexList.getIndexes() : null;
+ if (indexes == null) {
+ indexes = new ArrayList<>();
+ }
+ logger.info("Found {} indexes", indexes.size());
+
+ for (IndexModel index : indexes) {
+ result.incrementIndexesProcessed();
+ try {
+ boolean deletionInitiated = cleanupIndex(index);
+ if (deletionInitiated) {
+ result.incrementIndexesDeleted();
+ }
+ } catch (Exception e) {
+ logger.error("Failed to delete index {}: {}", index.getName(), e.getMessage(), e);
+ result.incrementIndexesFailed();
+ }
+ }
+
+ // Clean up collections
+ logger.info("Listing collections...");
+ CollectionList collectionList = pinecone.listCollections();
+ List collections = collectionList != null ? collectionList.getCollections() : null;
+ if (collections == null) {
+ collections = new ArrayList<>();
+ }
+ logger.info("Found {} collections", collections.size());
+
+ for (CollectionModel collection : collections) {
+ result.incrementCollectionsProcessed();
+ try {
+ boolean deletionInitiated = cleanupCollection(collection);
+ if (deletionInitiated) {
+ result.incrementCollectionsDeleted();
+ }
+ } catch (Exception e) {
+ logger.error("Failed to delete collection {}: {}", collection.getName(), e.getMessage(), e);
+ result.incrementCollectionsFailed();
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Cleans up a single index.
+ *
+ * @param index The index to clean up
+ * @throws Exception if an error occurs during cleanup
+ */
+ private boolean cleanupIndex(IndexModel index) throws Exception {
+ String indexName = index.getName();
+ String status = index.getStatus() != null && index.getStatus().getState() != null
+ ? index.getStatus().getState()
+ : "unknown";
+
+ logger.info("Processing index: {} (status: {})", indexName, status);
+
+ // Skip indexes that are already terminating
+ if ("Terminating".equalsIgnoreCase(status)) {
+ logger.info("Skipping index {} - already terminating", indexName);
+ return false;
+ }
+
+ // Note: Age-based filtering would go here when timestamp information becomes available
+ // For now, we process all indexes that aren't already terminating
+
+ if (dryRun) {
+ logger.info("DRY-RUN: Would delete index: {}", indexName);
+ return false;
+ }
+
+ // Handle deletion protection
+ if ("enabled".equals(index.getDeletionProtection())) {
+ logger.info("Index {} has deletion protection enabled, disabling...", indexName);
+ try {
+ // Try pod-based configuration first
+ index.getSpec().getIndexModelPodBased();
+ pinecone.configurePodsIndex(indexName, "disabled");
+ } catch (ClassCastException e) {
+ // Not a pod-based index, try serverless
+ pinecone.configureServerlessIndex(indexName, "disabled", null, null);
+ }
+
+ // Wait for configuration to take effect
+ logger.info("Waiting 5 seconds for deletion protection to be disabled...");
+ Thread.sleep(5000);
+ }
+
+ // Delete the index
+ logger.info("Deleting index: {}", indexName);
+ pinecone.deleteIndex(indexName);
+ logger.info("Successfully initiated deletion of index: {}", indexName);
+
+ // Add small delay to avoid rate limiting
+ Thread.sleep(1000);
+ return true;
+ }
+
+ /**
+ * Cleans up a single collection.
+ *
+ * @param collection The collection to clean up
+ * @throws Exception if an error occurs during cleanup
+ */
+ private boolean cleanupCollection(CollectionModel collection) throws Exception {
+ String collectionName = collection.getName();
+ String status = collection.getStatus() != null ? collection.getStatus() : "unknown";
+
+ logger.info("Processing collection: {} (status: {})", collectionName, status);
+
+ // Skip collections that are already terminating
+ if ("Terminating".equalsIgnoreCase(status)) {
+ logger.info("Skipping collection {} - already terminating", collectionName);
+ return false;
+ }
+
+ // Note: Age-based filtering would go here when timestamp information becomes available
+ // For now, we process all collections that aren't already terminating
+
+ if (dryRun) {
+ logger.info("DRY-RUN: Would delete collection: {}", collectionName);
+ return false;
+ }
+
+ // Delete the collection
+ logger.info("Deleting collection: {}", collectionName);
+ pinecone.deleteCollection(collectionName);
+ logger.info("Successfully initiated deletion of collection: {}", collectionName);
+
+ // Add small delay to avoid rate limiting
+ Thread.sleep(1000);
+ return true;
+ }
+
+ /**
+ * Result of a cleanup operation, containing statistics about what was processed and deleted.
+ */
+ public static class CleanupResult {
+ private int indexesProcessed = 0;
+ private int indexesDeleted = 0;
+ private int indexesFailed = 0;
+ private int collectionsProcessed = 0;
+ private int collectionsDeleted = 0;
+ private int collectionsFailed = 0;
+
+ public int getIndexesProcessed() { return indexesProcessed; }
+ public int getIndexesDeleted() { return indexesDeleted; }
+ public int getIndexesFailed() { return indexesFailed; }
+ public int getCollectionsProcessed() { return collectionsProcessed; }
+ public int getCollectionsDeleted() { return collectionsDeleted; }
+ public int getCollectionsFailed() { return collectionsFailed; }
+
+ void incrementIndexesProcessed() { indexesProcessed++; }
+ void incrementIndexesDeleted() { indexesDeleted++; }
+ void incrementIndexesFailed() { indexesFailed++; }
+ void incrementCollectionsProcessed() { collectionsProcessed++; }
+ void incrementCollectionsDeleted() { collectionsDeleted++; }
+ void incrementCollectionsFailed() { collectionsFailed++; }
+ }
}
diff --git a/src/test/java/io/pinecone/helpers/IndexCleanupUtilityTest.java b/src/test/java/io/pinecone/helpers/IndexCleanupUtilityTest.java
new file mode 100644
index 00000000..df4faee3
--- /dev/null
+++ b/src/test/java/io/pinecone/helpers/IndexCleanupUtilityTest.java
@@ -0,0 +1,159 @@
+package io.pinecone.helpers;
+
+import io.pinecone.clients.Pinecone;
+import org.junit.jupiter.api.Test;
+import org.openapitools.db_control.client.model.CollectionList;
+import org.openapitools.db_control.client.model.CollectionModel;
+import org.openapitools.db_control.client.model.IndexList;
+import org.openapitools.db_control.client.model.IndexModel;
+import org.openapitools.db_control.client.model.IndexModelStatus;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class IndexCleanupUtilityTest {
+
+ @Test
+ void cleanup_nullIndexesList_doesNotThrowAndProcessesZero() throws Exception {
+ Pinecone pinecone = mock(Pinecone.class);
+
+ when(pinecone.listIndexes()).thenReturn(new IndexList());
+ when(pinecone.listCollections()).thenReturn(new CollectionList().collections(Collections.emptyList()));
+
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, 1, false);
+ IndexCleanupUtility.CleanupResult result = utility.cleanup();
+
+ assertEquals(0, result.getIndexesProcessed());
+ assertEquals(0, result.getIndexesDeleted());
+ assertEquals(0, result.getIndexesFailed());
+
+ assertEquals(0, result.getCollectionsProcessed());
+ assertEquals(0, result.getCollectionsDeleted());
+ assertEquals(0, result.getCollectionsFailed());
+
+ verify(pinecone, never()).deleteIndex(anyString());
+ verify(pinecone, never()).deleteCollection(anyString());
+ }
+
+ @Test
+ void cleanup_nullCollectionsListObject_doesNotThrowAndProcessesZero() throws Exception {
+ Pinecone pinecone = mock(Pinecone.class);
+
+ when(pinecone.listIndexes()).thenReturn(new IndexList().indexes(Collections.emptyList()));
+ when(pinecone.listCollections()).thenReturn(null);
+
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, 1, false);
+ IndexCleanupUtility.CleanupResult result = utility.cleanup();
+
+ assertEquals(0, result.getIndexesProcessed());
+ assertEquals(0, result.getIndexesDeleted());
+ assertEquals(0, result.getIndexesFailed());
+
+ assertEquals(0, result.getCollectionsProcessed());
+ assertEquals(0, result.getCollectionsDeleted());
+ assertEquals(0, result.getCollectionsFailed());
+
+ verify(pinecone, never()).deleteIndex(anyString());
+ verify(pinecone, never()).deleteCollection(anyString());
+ }
+
+ @Test
+ void cleanup_dryRun_doesNotIncrementDeletedCounts() throws Exception {
+ Pinecone pinecone = mock(Pinecone.class);
+
+ IndexList indexes = new IndexList().indexes(Arrays.asList(
+ new IndexModel().name("ready-index").status(new IndexModelStatus().state("Ready").ready(true)),
+ new IndexModel().name("terminating-index").status(new IndexModelStatus().state("Terminating").ready(false))
+ ));
+ when(pinecone.listIndexes()).thenReturn(indexes);
+
+ CollectionList collections = new CollectionList().collections(Arrays.asList(
+ new CollectionModel().name("ready-collection").status("Ready"),
+ new CollectionModel().name("terminating-collection").status("Terminating")
+ ));
+ when(pinecone.listCollections()).thenReturn(collections);
+
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, 1, true);
+ IndexCleanupUtility.CleanupResult result = utility.cleanup();
+
+ assertEquals(2, result.getIndexesProcessed());
+ assertEquals(0, result.getIndexesDeleted());
+ assertEquals(0, result.getIndexesFailed());
+
+ assertEquals(2, result.getCollectionsProcessed());
+ assertEquals(0, result.getCollectionsDeleted());
+ assertEquals(0, result.getCollectionsFailed());
+
+ verify(pinecone, never()).deleteIndex(anyString());
+ verify(pinecone, never()).deleteCollection(anyString());
+ }
+
+ @Test
+ void cleanup_terminatingResources_doesNotIncrementDeletedCounts() throws Exception {
+ Pinecone pinecone = mock(Pinecone.class);
+
+ IndexList indexes = new IndexList().indexes(Collections.singletonList(
+ new IndexModel().name("terminating-index").status(new IndexModelStatus().state("Terminating").ready(false))
+ ));
+ when(pinecone.listIndexes()).thenReturn(indexes);
+
+ CollectionList collections = new CollectionList().collections(Collections.singletonList(
+ new CollectionModel().name("terminating-collection").status("Terminating")
+ ));
+ when(pinecone.listCollections()).thenReturn(collections);
+
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, 1, false);
+ IndexCleanupUtility.CleanupResult result = utility.cleanup();
+
+ assertEquals(1, result.getIndexesProcessed());
+ assertEquals(0, result.getIndexesDeleted());
+ assertEquals(0, result.getIndexesFailed());
+
+ assertEquals(1, result.getCollectionsProcessed());
+ assertEquals(0, result.getCollectionsDeleted());
+ assertEquals(0, result.getCollectionsFailed());
+
+ verify(pinecone, never()).deleteIndex(anyString());
+ verify(pinecone, never()).deleteCollection(anyString());
+ }
+
+ @Test
+ void cleanup_nonDryRun_incrementsDeletedCountsWhenDeletionIsInitiated() throws Exception {
+ Pinecone pinecone = mock(Pinecone.class);
+
+ IndexList indexes = new IndexList().indexes(Collections.singletonList(
+ new IndexModel()
+ .name("ready-index")
+ .deletionProtection("disabled")
+ .status(new IndexModelStatus().state("Ready").ready(true))
+ ));
+ when(pinecone.listIndexes()).thenReturn(indexes);
+
+ CollectionList collections = new CollectionList().collections(Collections.singletonList(
+ new CollectionModel().name("ready-collection").status("Ready")
+ ));
+ when(pinecone.listCollections()).thenReturn(collections);
+
+ IndexCleanupUtility utility = new IndexCleanupUtility(pinecone, 1, false);
+ IndexCleanupUtility.CleanupResult result = utility.cleanup();
+
+ assertEquals(1, result.getIndexesProcessed());
+ assertEquals(1, result.getIndexesDeleted());
+ assertEquals(0, result.getIndexesFailed());
+
+ assertEquals(1, result.getCollectionsProcessed());
+ assertEquals(1, result.getCollectionsDeleted());
+ assertEquals(0, result.getCollectionsFailed());
+
+ verify(pinecone).deleteIndex("ready-index");
+ verify(pinecone).deleteCollection("ready-collection");
+ }
+}
+