Skip to content

Commit

Permalink
Update index operations (#62)
Browse files Browse the repository at this point in the history
## Problem

After adding open api generated code for control plane, there's a need
for a wrapper for the index operations.

## Solution

This PR adds a wrapper for index operations and associated integration
tests. Couple of the integration tests are currently commented out and
will be handled in the upcoming PR's.

## Type of Change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

## Test Plan

Ran integration tests.
  • Loading branch information
rohanshah18 authored Feb 2, 2024
1 parent 12a2ee5 commit 20e5494
Show file tree
Hide file tree
Showing 20 changed files with 743 additions and 1,241 deletions.
6 changes: 5 additions & 1 deletion src/integration/java/io/pinecone/helpers/AssertRetry.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ public class AssertRetry {
private static int delay = 1500;

public static void assertWithRetry(AssertionRunnable assertionRunnable) throws InterruptedException {
assertWithRetry(assertionRunnable, 2);
}

public static void assertWithRetry(AssertionRunnable assertionRunnable, int backOff) throws InterruptedException {
int retryCount = 0;
boolean success = false;

Expand All @@ -17,7 +21,7 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable) throws I
success = true;
} catch (AssertionError | ExecutionException | IOException e) {
retryCount++;
delay*=2;
delay*=backOff;
Thread.sleep(delay);
}
}
Expand Down
79 changes: 44 additions & 35 deletions src/integration/java/io/pinecone/helpers/IndexManager.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package io.pinecone.helpers;

import io.pinecone.*;
import io.pinecone.model.CreateIndexRequest;
import io.pinecone.model.IndexMeta;
import org.openapitools.client.model.*;

import java.io.IOException;
import java.util.List;
Expand All @@ -12,69 +11,79 @@
public class IndexManager {
private static PineconeClientConfig config;

public static PineconeConnection createIndexIfNotExistsDataPlane(int dimension) throws IOException, InterruptedException {
config = new PineconeClientConfig()
.withApiKey(System.getenv("PINECONE_API_KEY"))
.withEnvironment(System.getenv("PINECONE_ENVIRONMENT"));
public static PineconeConnection createIndexIfNotExistsDataPlane(int dimension, String indexType) throws IOException, InterruptedException {
config = new PineconeClientConfig().withApiKey(System.getenv("PINECONE_API_KEY")).withEnvironment(System.getenv("PINECONE_ENVIRONMENT"));
PineconeIndexOperationClient controlPlaneClient = new PineconeIndexOperationClient(config);
List<String> indexList = controlPlaneClient.listIndexes();
IndexList indexList = controlPlaneClient.listIndexes();

String indexName = findIndexWithDimensionAndPodType(indexList, dimension, controlPlaneClient);
if(indexName.isEmpty()) indexName = createNewIndex(dimension, controlPlaneClient);
String indexName = findIndexWithDimensionAndType(indexList, dimension, controlPlaneClient, indexType);
if (indexName.isEmpty()) indexName = createNewIndex(controlPlaneClient, indexType, dimension);

// Do not proceed until the newly created index is ready
isIndexReady(indexName, controlPlaneClient);
PineconeClient dataPlaneClient = new PineconeClient(config);
IndexMeta indexMeta = controlPlaneClient.describeIndex(indexName);
String host = indexMeta.getStatus().getHost();
String host = controlPlaneClient.describeIndex(indexName).getHost();

return dataPlaneClient.connect(
new PineconeConnectionConfig()
.withConnectionUrl("https://" + host));
}

public static String createIndexIfNotExistsControlPlane(PineconeClientConfig config, int dimension) throws IOException, InterruptedException {
PineconeIndexOperationClient controlPlaneClient = new PineconeIndexOperationClient(config);
List<String> indexList = controlPlaneClient.listIndexes();
String indexName = findIndexWithDimensionAndPodType(indexList, dimension, controlPlaneClient);
public static String createIndexIfNotExistsControlPlane(PineconeIndexOperationClient controlPlaneClient, int dimension, String indexType) throws IOException, InterruptedException {
IndexList indexList = controlPlaneClient.listIndexes();
String indexName = findIndexWithDimensionAndType(indexList, dimension, controlPlaneClient, indexType);

return (indexName.isEmpty()) ? createNewIndex(dimension, controlPlaneClient) : indexName;
return (indexName.isEmpty()) ? createNewIndex(controlPlaneClient, indexType, dimension) : indexName;
}

private static String findIndexWithDimensionAndPodType(List<String> indexList, int dimension, PineconeIndexOperationClient controlPlaneClient)
throws IOException, InterruptedException {
private static String findIndexWithDimensionAndType(IndexList indexList, int dimension, PineconeIndexOperationClient controlPlaneClient, String indexType)
throws InterruptedException {
int i = 0;
while (i < indexList.size()) {
IndexMeta indexMeta = isIndexReady(indexList.get(i), controlPlaneClient);
if (indexMeta.getDatabase().getDimension() == dimension
&& (indexMeta.getDatabase().getPodType().equals("p1.x1") || config.getEnvironment().equals("gcp-starter"))) {
return indexList.get(i);
List<IndexModel> indexModels = indexList.getIndexes();
while (i < indexModels.size()) {
IndexModel indexModel = isIndexReady(indexModels.get(i).getName(), controlPlaneClient);
// ToDo: add pod type support
if (indexModel.getDimension() == dimension
&& ((indexType.equalsIgnoreCase(IndexModelSpec.SERIALIZED_NAME_POD) && indexModel.getSpec().getPod() != null && indexModel.getSpec().getPod().getPodType().equalsIgnoreCase("p1.x1"))
|| (indexType.equalsIgnoreCase(IndexModelSpec.SERIALIZED_NAME_SERVERLESS)))) {
return indexModel.getName();
}
i++;
}
return "";
}

private static String createNewIndex(int dimension, PineconeIndexOperationClient controlPlaneClient) throws IOException {
private static String createNewIndex(PineconeIndexOperationClient controlPlaneClient, String indexType, int dimension) throws IOException {
String indexName = RandomStringBuilder.build("index-name", 8);
String environment = System.getenv("PINECONE_ENVIRONMENT");
CreateIndexRequestSpec createIndexRequestSpec;

if (indexType.equalsIgnoreCase(IndexModelSpec.SERIALIZED_NAME_POD)) {
CreateIndexRequestSpecPod podSpec = new CreateIndexRequestSpecPod().environment(environment).podType("p1.x1");
createIndexRequestSpec = new CreateIndexRequestSpec().pod(podSpec);
} else {
ServerlessSpec serverlessSpec = new ServerlessSpec().cloud(ServerlessSpec.CloudEnum.AWS).region(environment);
createIndexRequestSpec = new CreateIndexRequestSpec().serverless(serverlessSpec);
}

CreateIndexRequest createIndexRequest = new CreateIndexRequest()
.withIndexName(indexName)
.withDimension(dimension)
.withMetric("euclidean")
.withPodType("p1.x1");
.name(indexName)
.dimension(dimension)
.metric(IndexMetric.EUCLIDEAN)
.spec(createIndexRequestSpec);
controlPlaneClient.createIndex(createIndexRequest);

return indexName;
}

public static IndexMeta isIndexReady(String indexName, PineconeIndexOperationClient indexOperationClient)
throws IOException, InterruptedException {
final IndexMeta[] indexMeta = new IndexMeta[1];
public static IndexModel isIndexReady(String indexName, PineconeIndexOperationClient controlPlaneClient)
throws InterruptedException {
final IndexModel[] indexModels = new IndexModel[1];
assertWithRetry(() -> {
indexMeta[0] = indexOperationClient.describeIndex(indexName);
assert (indexMeta[0].getStatus().getState().equalsIgnoreCase("ready"));
});
indexModels[0] = controlPlaneClient.describeIndex(indexName);
assert (indexModels[0].getStatus().getReady());
}, 1);

return indexMeta[0];
return indexModels[0];
}
}

This file was deleted.

This file was deleted.

Loading

0 comments on commit 20e5494

Please sign in to comment.