From 931134b36c08b851aeaa855210548d1a5d9939ce Mon Sep 17 00:00:00 2001 From: Rohan Shah Date: Tue, 16 Jan 2024 15:15:19 -0500 Subject: [PATCH] Add source_collection to indexMetaDatabase object and ignore newly added fields (#58) ## Problem `source_collection` is a newly added field in `indexMetaDatabase` object of describeIndex response body. But without adding it to the response body of describe index call, it breaks. ## Solution Added the field as well as added support to ignore newly added fields. ## Type of Change - [X] Bug fix (non-breaking change which fixes an issue) - [X] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update - [ ] Infrastructure change (CI configs, etc) - [ ] Non-code change (docs, etc) - [ ] None of the above: (explain here) ## Test Plan Ran integration tests and added unit test for the same --- CHANGELOG.md | 3 + README.md | 6 +- gradle.properties | 2 +- .../{RetryAssert.java => AssertRetry.java} | 7 ++- .../io/pinecone/helpers/IndexManager.java | 17 +++--- .../controlPlane/ConfigureIndexTest.java | 16 +++-- .../PineconeClientLiveIntegTest.java | 8 +-- .../UpsertAndDescribeIndexStatsTest.java | 2 +- .../io/pinecone/PineconeClientConfig.java | 2 +- .../io/pinecone/model/CreateIndexRequest.java | 2 +- .../io/pinecone/model/IndexMetaDatabase.java | 15 +++++ .../PineconeIndexOperationClientTest.java | 60 +++++++++++++++++-- src/test/resources/indexJsonString.json | 1 + .../indexJsonStringWithExtraFields.json | 1 + 14 files changed, 104 insertions(+), 38 deletions(-) rename src/integration/java/io/pinecone/helpers/{RetryAssert.java => AssertRetry.java} (81%) create mode 100644 src/test/resources/indexJsonString.json create mode 100644 src/test/resources/indexJsonStringWithExtraFields.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 535c011a..072121eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ [comment]: <> (When bumping [pc:VERSION_LATEST_RELEASE] create a new entry below) ### Unreleased version +### v0.7.4 +- Add source_collection and support to ignore newly added fields to the response body for describeIndex's indexMetaDatabase object + ### v0.7.3 - Add assert with retry mechanism - Add deprecation warning for queries parameter diff --git a/README.md b/README.md index f004ddd0..1108c7ea 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Maven: io.pinecone pinecone-client - 0.7.3 + 0.7.4 ``` @@ -23,12 +23,12 @@ Maven: Gradle: ``` -implementation "io.pinecone:pinecone-client:0.7.3" +implementation "io.pinecone:pinecone-client:0.7.4" ``` [comment]: <> (^ [pc:VERSION_LATEST_RELEASE]) -Alternatively, you can use our standalone uberjar [pinecone-client-0.7.3-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/0.7.3/pinecone-client-0.7.3-all.jar), which bundles the pinecone client and all dependencies together inside a single jar. You can include this on your classpath like any 3rd party JAR without having to obtain the *pinecone-client* dependencies separately. +Alternatively, you can use our standalone uberjar [pinecone-client-0.7.4-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/0.7.4/pinecone-client-0.7.4-all.jar), which bundles the pinecone client and all dependencies together inside a single jar. You can include this on your classpath like any 3rd party JAR without having to obtain the *pinecone-client* dependencies separately. [comment]: <> (^ [pc:VERSION_LATEST_RELEASE]) diff --git a/gradle.properties b/gradle.properties index 521ff6ec..65120d1f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -pineconeClientVersion = 0.7.3 +pineconeClientVersion = 0.7.4 diff --git a/src/integration/java/io/pinecone/helpers/RetryAssert.java b/src/integration/java/io/pinecone/helpers/AssertRetry.java similarity index 81% rename from src/integration/java/io/pinecone/helpers/RetryAssert.java rename to src/integration/java/io/pinecone/helpers/AssertRetry.java index bf2dbbd1..8b263d23 100644 --- a/src/integration/java/io/pinecone/helpers/RetryAssert.java +++ b/src/integration/java/io/pinecone/helpers/AssertRetry.java @@ -1,8 +1,9 @@ package io.pinecone.helpers; +import java.io.IOException; import java.util.concurrent.ExecutionException; -public class RetryAssert { +public class AssertRetry { private static final int maxRetry = 4; private static int delay = 1500; @@ -14,7 +15,7 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable) throws I try { assertionRunnable.run(); success = true; - } catch (AssertionError | ExecutionException e) { + } catch (AssertionError | ExecutionException | IOException e) { retryCount++; delay*=2; Thread.sleep(delay); @@ -24,6 +25,6 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable) throws I @FunctionalInterface public interface AssertionRunnable { - void run() throws AssertionError, ExecutionException, InterruptedException; + void run() throws AssertionError, ExecutionException, InterruptedException, IOException; } } diff --git a/src/integration/java/io/pinecone/helpers/IndexManager.java b/src/integration/java/io/pinecone/helpers/IndexManager.java index 9e9d21e5..0f5dc3c3 100644 --- a/src/integration/java/io/pinecone/helpers/IndexManager.java +++ b/src/integration/java/io/pinecone/helpers/IndexManager.java @@ -7,6 +7,8 @@ import java.io.IOException; import java.util.List; +import static io.pinecone.helpers.AssertRetry.assertWithRetry; + public class IndexManager { private static PineconeClientConfig config; @@ -67,15 +69,12 @@ private static String createNewIndex(int dimension, PineconeIndexOperationClient public static IndexMeta isIndexReady(String indexName, PineconeIndexOperationClient indexOperationClient) throws IOException, InterruptedException { - IndexMeta indexMeta; - while (true) { - indexMeta = indexOperationClient.describeIndex(indexName); - if (indexMeta.getStatus().getState().equalsIgnoreCase("ready")) { - break; - } - Thread.sleep(1000); - } + final IndexMeta[] indexMeta = new IndexMeta[1]; + assertWithRetry(() -> { + indexMeta[0] = indexOperationClient.describeIndex(indexName); + assert (indexMeta[0].getStatus().getState().equalsIgnoreCase("ready")); + }); - return indexMeta; + return indexMeta[0]; } } diff --git a/src/integration/java/io/pinecone/integration/controlPlane/ConfigureIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/ConfigureIndexTest.java index 8555b35d..b0ce960a 100644 --- a/src/integration/java/io/pinecone/integration/controlPlane/ConfigureIndexTest.java +++ b/src/integration/java/io/pinecone/integration/controlPlane/ConfigureIndexTest.java @@ -43,8 +43,8 @@ public void configureIndexWithInvalidIndexName() { try { isIndexReady(indexName, indexOperationClient); indexOperationClient.configureIndex("non-existent-index", configureIndexRequest); - } catch (PineconeNotFoundException exception) { - assert (exception.getLocalizedMessage().contains("404: Not Found")); + } catch (PineconeNotFoundException notFoundException) { + assert (notFoundException.getLocalizedMessage().toLowerCase().contains("not found")); } catch (IOException | InterruptedException exception) { logger.error(exception.toString()); } @@ -57,10 +57,8 @@ public void configureIndexExceedingQuota() { try { isIndexReady(indexName, indexOperationClient); indexOperationClient.configureIndex(indexName, configureIndexRequest); - } catch (PineconeBadRequestException exception) { - assert (exception.getLocalizedMessage().contains("The index exceeds the project quota")); - assert (exception.getLocalizedMessage().contains("Upgrade your account or change" + - " the project settings to increase the quota.")); + } catch (PineconeBadRequestException badRequestException) { + assert (badRequestException.getLocalizedMessage().contains("Capacity Reached. Increase your quota or upgrade to create more indexes.")); } catch (Exception exception) { logger.error(exception.toString()); } @@ -110,8 +108,8 @@ public void changingBasePodType() { isIndexReady(indexName, indexOperationClient); indexOperationClient.configureIndex(indexName, configureIndexRequest); - } catch (PineconeBadRequestException pineconeBadRequestException) { - assertEquals(pineconeBadRequestException.getMessage(), "updating base pod type is not supported"); + } catch (PineconeBadRequestException badRequestException) { + assertEquals(badRequestException.getMessage(), "Bad request: Cannot change pod type."); } catch (Exception exception) { logger.error(exception.getLocalizedMessage()); } @@ -166,7 +164,7 @@ public void sizeDown() throws IOException, InterruptedException { Thread.sleep(3500); } catch (Exception exception) { assertEquals(exception.getClass(), PineconeBadRequestException.class); - assertEquals(exception.getMessage(), "scaling down pod type is not supported"); + assert(exception.getMessage().contains("Bad request: Cannot scale down an index, only up.")); } finally { // Delete this index since it'll be unused for other tests indexOperationClient.deleteIndex(indexName); diff --git a/src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java b/src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java index 5b30f94e..a95fd102 100644 --- a/src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java +++ b/src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java @@ -18,10 +18,7 @@ import java.util.List; import static io.pinecone.helpers.IndexManager.createIndexIfNotExistsDataPlane; -import static io.pinecone.helpers.RetryAssert.assertWithRetry; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; +import static io.pinecone.helpers.AssertRetry.assertWithRetry; public class PineconeClientLiveIntegTest { @@ -155,8 +152,7 @@ public void sanity() throws InterruptedException { assertWithRetry(() -> { QueryResponse queryResponse = blockingStub.query(queryByIdRequest); - assertThat(queryResponse, notNullValue()); - assertThat(queryResponse.getMatchesCount(), equalTo(1)); + Assertions.assertEquals(queryResponse.getMatchesCount(),1); assert (queryResponse.getMatches(0).getValuesList().containsAll(updateValueList)); }); diff --git a/src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java b/src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java index f6cc38d7..f54b8f95 100644 --- a/src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java +++ b/src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java @@ -6,7 +6,7 @@ import static io.pinecone.helpers.BuildUpsertRequest.*; import static io.pinecone.helpers.IndexManager.createIndexIfNotExistsDataPlane; -import static io.pinecone.helpers.RetryAssert.assertWithRetry; +import static io.pinecone.helpers.AssertRetry.assertWithRetry; import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; diff --git a/src/main/java/io/pinecone/PineconeClientConfig.java b/src/main/java/io/pinecone/PineconeClientConfig.java index abfb3671..67bcc76f 100644 --- a/src/main/java/io/pinecone/PineconeClientConfig.java +++ b/src/main/java/io/pinecone/PineconeClientConfig.java @@ -139,7 +139,7 @@ private String maskedApiKey() { } public String getUserAgent() { - String userAgentLanguage = "lang=java; pineconeClientVersion = v0.7.3"; + String userAgentLanguage = "lang=java; pineconeClientVersion = v0.7.4"; return (this.getUsageContext() != null) ? userAgentLanguage + "; usageContext=" + this.getUsageContext() : userAgentLanguage; } diff --git a/src/main/java/io/pinecone/model/CreateIndexRequest.java b/src/main/java/io/pinecone/model/CreateIndexRequest.java index a92c9eb3..f61f19f8 100644 --- a/src/main/java/io/pinecone/model/CreateIndexRequest.java +++ b/src/main/java/io/pinecone/model/CreateIndexRequest.java @@ -12,7 +12,7 @@ public class CreateIndexRequest { private String metric = "cosine"; - private Integer pods = 1; + private Integer pods; private Integer replicas = 1; diff --git a/src/main/java/io/pinecone/model/IndexMetaDatabase.java b/src/main/java/io/pinecone/model/IndexMetaDatabase.java index d477c139..c49b06dc 100644 --- a/src/main/java/io/pinecone/model/IndexMetaDatabase.java +++ b/src/main/java/io/pinecone/model/IndexMetaDatabase.java @@ -1,7 +1,9 @@ package io.pinecone.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +@JsonIgnoreProperties(ignoreUnknown = true) public class IndexMetaDatabase { private String name; @@ -18,6 +20,9 @@ public class IndexMetaDatabase { @JsonProperty("pod_type") private String podType; + @JsonProperty("source_collection") + private String sourceCollection; + private IndexMetadataConfig metadataConfig; public IndexMetaDatabase () { @@ -87,6 +92,15 @@ public IndexMetaDatabase withPodType(String podType) { return this; } + public String getSourceCollection() { + return sourceCollection; + } + + public IndexMetaDatabase withSourceCollection(String sourceCollection) { + this.sourceCollection = sourceCollection; + return this; + } + public IndexMetadataConfig getMetadataConfig() { return metadataConfig; } @@ -107,6 +121,7 @@ public String toString() { ", shards=" + shards + ", podType='" + podType + '\'' + ", metadataConfig=" + metadataConfig + + ", sourceCollection=" + sourceCollection + '}'; } } diff --git a/src/test/java/io/pinecone/PineconeIndexOperationClientTest.java b/src/test/java/io/pinecone/PineconeIndexOperationClientTest.java index eee6fc06..db9adaf1 100644 --- a/src/test/java/io/pinecone/PineconeIndexOperationClientTest.java +++ b/src/test/java/io/pinecone/PineconeIndexOperationClientTest.java @@ -10,6 +10,8 @@ import org.junit.jupiter.api.BeforeAll; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -176,10 +178,59 @@ public void testCreateIndexWithAllFields() throws IOException { @Test public void testDescribeIndex() throws IOException { String indexName = "testIndex"; - String indexJsonString = "{\"database\":{\"name\":\"testIndex\",\"metric\":\"cosine\",\"dimension\":3," + - "\"replicas\":1,\"shards\":1,\"pods\":1,\"pod_type\":\"p1.x1\"}," + - "\"status\":{\"waiting\":[],\"crashed\":[],\"host\":\"url\",\"port\":433," + - "\"state\":\"Ready\",\"ready\":true}}"; + String indexString = ""; + String filePath = "src/test/resources/indexJsonString.json"; + String indexJsonString = new String(Files.readAllBytes(Paths.get(filePath))); + + PineconeClientConfig clientConfig = new PineconeClientConfig() + .withApiKey("testApiKey") + .withEnvironment("testEnvironment"); + IndexMetaDatabase metaDatabase = new IndexMetaDatabase() + .withName("testIndex") + .withMetric("cosine") + .withDimension(3) + .withReplicas(1) + .withShards(1) + .withPods(1) + .withPodType("p1.x1") + .withSourceCollection("randomCollection"); + IndexStatus indexStatus = new IndexStatus() + .withHost("url") + .withState("Ready") + .withReady(true) + .withPort("433"); + + IndexMeta expectedIndexMeta = new IndexMeta() + .withMetaDatabase(metaDatabase) + .withStatus(indexStatus); + + Call mockCall = mock(Call.class); + Response mockResponse = new Response.Builder() + .request(new Request.Builder().url("http://localhost").build()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(indexJsonString, MediaType.parse("application/json"))) + .build(); + + when(mockCall.execute()).thenReturn(mockResponse); + + OkHttpClient mockClient = mock(OkHttpClient.class); + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponse); + + PineconeIndexOperationClient indexOperationClient = new PineconeIndexOperationClient(clientConfig, mockClient); + IndexMeta actualIndexMeta = indexOperationClient.describeIndex(indexName); + + assertEquals(expectedIndexMeta.toString(), actualIndexMeta.toString()); + } + + @Test + public void testDescribeIndexWithExtraFields() throws IOException { + String indexName = "testIndex"; + String filePath = "src/test/resources/indexJsonStringWithExtraFields.json"; + String indexJsonString = new String(Files.readAllBytes(Paths.get(filePath))); + PineconeClientConfig clientConfig = new PineconeClientConfig() .withApiKey("testApiKey") .withEnvironment("testEnvironment"); @@ -191,6 +242,7 @@ public void testDescribeIndex() throws IOException { .withShards(1) .withPods(1) .withPodType("p1.x1"); + IndexStatus indexStatus = new IndexStatus() .withHost("url") .withState("Ready") diff --git a/src/test/resources/indexJsonString.json b/src/test/resources/indexJsonString.json new file mode 100644 index 00000000..85647cda --- /dev/null +++ b/src/test/resources/indexJsonString.json @@ -0,0 +1 @@ +{"database":{"name":"testIndex","metric":"cosine","dimension":3,"replicas":1,"shards":1,"pods":1,"pod_type":"p1.x1", "source_collection":"randomCollection"},"status":{"waiting":[],"crashed":[],"host":"url","port":433,"state":"Ready","ready":true}} \ No newline at end of file diff --git a/src/test/resources/indexJsonStringWithExtraFields.json b/src/test/resources/indexJsonStringWithExtraFields.json new file mode 100644 index 00000000..4c369373 --- /dev/null +++ b/src/test/resources/indexJsonStringWithExtraFields.json @@ -0,0 +1 @@ +{"database":{"name":"testIndex","metric":"cosine","dimension":3,"replicas":1,"shards":1,"pods":1,"pod_type":"p1.x1","extra_field":"random1"},"status":{"waiting":[],"crashed":[],"host":"url","port":433,"state":"Ready","ready":true, "extra_field":"random2"}} \ No newline at end of file