diff --git a/docs/ServiceNow-batchsource.md b/docs/ServiceNow-batchsource.md index 6769bf41..116a78bd 100644 --- a/docs/ServiceNow-batchsource.md +++ b/docs/ServiceNow-batchsource.md @@ -67,20 +67,21 @@ ignored if the Mode is set to `Reporting`. Data Types Mapping ---------- - | ServiceNow Data Type | CDAP Schema Data Type | Comment | - | ------------------------------ | --------------------- | -------------------------------------------------- | - | decimal | double | | - | integer | int | | - | boolean | boolean | | - | reference | string | | - | currency | string | | - | glide_date | string | | - | glide_date_time | string | | - | sys_class_name | string | | - | domain_id | string | | - | domain_path | string | | - | guid | string | | - | translated_html | string | | - | journal | string | | - | string | string | | + | ServiceNow Data Type | CDAP Schema Data Type + | ------------------------------ | --------------------- + | decimal | double + | integer | int + | boolean | boolean + | reference | string + | currency | string + | glide_date | date + | glide_date_time | datetime + | glide_time | time + | sys_class_name | string + | domain_id | string + | domain_path | string + | guid | string + | translated_html | string + | journal | string + | string | string diff --git a/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature b/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature index 5a5d6937..39be0141 100644 --- a/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature +++ b/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature @@ -26,11 +26,6 @@ Feature: ServiceNow Multi Source - Design time validation scenarios And Click on the Validate button Then Verify mandatory property error for below listed properties: | referenceName | - | clientId | - | clientSecret | - | restApiEndpoint | - | user | - | password | @TS-SN-MULTI-DSGN-ERROR-02 Scenario: Verify validation message for invalid table name @@ -41,7 +36,7 @@ Feature: ServiceNow Multi Source - Design time validation scenarios | INVALID_TABLE | And fill Credentials section for pipeline user And Click on the Validate button - Then Verify that the Plugin is displaying an error message: "invalid.property.tablename" on the header + Then Verify that the Plugin Property: "tableNames" is displaying an in-line error message: "invalid.property.tablename" @TS-SN-MULTI-DSGN-ERROR-03 Scenario: Verify validation message for Start date and End date in invalid format diff --git a/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature b/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature index b6bd128f..ebfa7d7d 100644 --- a/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature +++ b/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature @@ -30,4 +30,4 @@ Feature: ServiceNow Multi Source - Design time scenarios (macro) And Click on the Macro button of Property: "restApiEndpoint" and set the value to: "restApiEndpoint" And Click on the Macro button of Property: "user" and set the value to: "username" And Click on the Macro button of Property: "password" and set the value to: "password" - Then Validate "ServiceNow Multi Source" plugin properties \ No newline at end of file + Then Validate "ServiceNow Multi Source" plugin properties diff --git a/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature index 80907fb0..dffc7a94 100644 --- a/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature +++ b/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature @@ -136,4 +136,4 @@ Feature: ServiceNow Multi Source - Run time scenarios (macro) And Verify the pipeline status is "Failed" Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | - | ERROR | invalid.filters.logsmessage | \ No newline at end of file + | ERROR | invalid.filters.logsmessage | diff --git a/src/e2e-test/features/servicenowsink/DesignTime.feature b/src/e2e-test/features/servicenowsink/DesignTime.feature index 0f201a58..98384825 100644 --- a/src/e2e-test/features/servicenowsink/DesignTime.feature +++ b/src/e2e-test/features/servicenowsink/DesignTime.feature @@ -15,7 +15,7 @@ @ServiceNow @SNSink @Smoke -@Regression +@RegressionSkip Feature: ServiceNow Sink - Design time scenarios @TS-SN-DSGN-SINK-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE diff --git a/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature b/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature index 28f9047d..f05e9004 100644 --- a/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature +++ b/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature @@ -15,7 +15,7 @@ @ServiceNow @SNSink @Smoke -@Regression +@RegressionSkip Feature: ServiceNow Sink - Design time validation scenarios @TS-SN-DSGN-SINK-ERROR-01 @@ -60,4 +60,4 @@ Feature: ServiceNow Sink - Design time validation scenarios And Enter input plugin property: "name" with value: "connection.name" And fill Credentials section with invalid credentials Then Click on the Test Connection button - Then Verify the invalid connection error message: "invalid.testconnection.logmessage" on the footer \ No newline at end of file + Then Verify the invalid connection error message: "invalid.testconnection.logmessage" on the footer diff --git a/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature b/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature index c2215a21..d8aae838 100644 --- a/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature +++ b/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature @@ -15,7 +15,7 @@ @ServiceNow @SNSink @Smoke -@@Regression +@@RegressionSkip Feature: ServiceNow Sink - Design time validation scenarios (macro) @TS-SN-DSGN-SINK-MACRO-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE @@ -69,4 +69,4 @@ Feature: ServiceNow Sink - Design time validation scenarios (macro) And fill Credentials section for pipeline user And Enter input plugin property: "tableName" with value: "receiving_slip_line" And Click on the Macro button of Property: "operation" and set the value to: "operation" - Then Validate "ServiceNow" plugin properties \ No newline at end of file + Then Validate "ServiceNow" plugin properties diff --git a/src/e2e-test/features/servicenowsink/RunTime.feature b/src/e2e-test/features/servicenowsink/RunTime.feature index f3379fc4..a2844b81 100644 --- a/src/e2e-test/features/servicenowsink/RunTime.feature +++ b/src/e2e-test/features/servicenowsink/RunTime.feature @@ -15,7 +15,7 @@ @ServiceNow @SNSink @Smoke -@Regression +@RegressionSkip Feature: ServiceNow Sink - Run time scenarios @TS-SN-RNTM-SINK-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE diff --git a/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature index cabb0023..863f44b4 100644 --- a/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature +++ b/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature @@ -15,7 +15,7 @@ @ServiceNow @SNSink @Smoke -@Regression +@RegressionSkip Feature: ServiceNow Sink - Run time scenarios (macro) @TS-SN-RNTM-SINK-MACRO-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE @BQ_SINK_CLEANUP @@ -197,4 +197,3 @@ Feature: ServiceNow Sink - Run time scenarios (macro) Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | invalid.credentials.logsmessage | - diff --git a/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature b/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature index 750409a4..5f330583 100644 --- a/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature +++ b/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature @@ -26,11 +26,6 @@ Feature: ServiceNow Source - Design time validation scenarios And Click on the Validate button Then Verify mandatory property error for below listed properties: | referenceName | - | clientId | - | clientSecret | - | restApiEndpoint | - | user | - | password | @TS-SN-DSGN-ERROR-02 Scenario: Verify invalid credentials validation messages diff --git a/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature b/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature index c7f55b0b..e501f8d6 100644 --- a/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature +++ b/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature @@ -46,4 +46,4 @@ Feature: ServiceNow Source - Design time scenarios (macro) And Click on the Macro button of Property: "restApiEndpoint" and set the value to: "restApiEndpoint" And Click on the Macro button of Property: "user" and set the value to: "username" And Click on the Macro button of Property: "password" and set the value to: "password" - Then Validate "ServiceNow" plugin properties \ No newline at end of file + Then Validate "ServiceNow" plugin properties diff --git a/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature index 7597dc04..38e6b471 100644 --- a/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature +++ b/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature @@ -184,4 +184,4 @@ Feature: ServiceNow Source - Run time scenarios (macro) And Verify the pipeline status is "Failed" Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | - | ERROR | invalid.filters.logsmessage | \ No newline at end of file + | ERROR | invalid.filters.logsmessage | diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java index 70bb124a..6216b8ee 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java @@ -44,7 +44,7 @@ public void configureBqSinkPlugin() { public void configureBqMultiTableSinkPlugin() { String referenceName = "Test" + RandomStringUtils.randomAlphanumeric(10); CdfBigQueryPropertiesActions.enterBigQueryReferenceName(referenceName); - CdfBigQueryPropertiesActions.enterBigQueryDataset(PluginPropertyUtils.pluginProp("bq.target.dataset")); + CdfBigQueryPropertiesActions.enterBigQueryDataset(PluginPropertyUtils.pluginProp("bq.target.dataset2")); } @Then("Verify count of no of records transferred to the target BigQuery Table") diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties index 41044729..b2e28d3a 100644 --- a/src/e2e-test/resources/errorMessage.properties +++ b/src/e2e-test/resources/errorMessage.properties @@ -4,7 +4,7 @@ validationSuccessMessage=No errors found. #Invalid value invalid.property.tablename=Bad Request. Table: invalid.property.startdate=Invalid format for Start date. Correct Format: yyyy-MM-dd -invalid.property.enddate=Invalid format for End date. Correct Format:yyyy-MM-dd +invalid.property.enddate=Invalid format for End date. Correct Format: yyyy-MM-dd invalid.property.credentials=Unable to connect to ServiceNow Instance. Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password are correct. #Logs error message diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 986ba223..d9baec91 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -10,6 +10,7 @@ pipeline.user.password = SERVICE_NOW_PASSWORD #Tables receiving_slip_line=proc_rec_slip_item +asset_covered=clm_m2m_contract_asset agent_assist_recommendation = agent_assist_recommendation vendor_catalog_item = pc_vendor_cat_item service_offering = service_offering @@ -38,6 +39,7 @@ connection.name = dummy projectId=cdf-athena datasetprojectId=cdf-athena bq.target.dataset=SN_test_automation +bq.target.dataset2=SN_Test_atm ##ServiceNowSink INSERT=insert @@ -45,19 +47,19 @@ UPDATE=update pagesize=200 ##ExpectedSchemaJSONs -schema.table.receiving.slip.line=[{"key":"cost","value":"string"},{"key":"quantity","value":"string"},\ - {"key":"purchase_line","value":"string"},{"key":"sys_mod_count","value":"string"},\ - {"key":"received","value":"string"},{"key":"requested_for","value":"string"},\ - {"key":"sys_updated_on","value":"string"},{"key":"sys_tags","value":"string"},{"key":"number","value":"string"},\ +schema.table.receiving.slip.line=[{"key":"quantity","value":"int"},{"key":"cost","value":"string"},\ + {"key":"purchase_line","value":"string"},{"key":"sys_mod_count","value":"int"},\ + {"key":"received","value":"datetime"},{"key":"requested_for","value":"string"},\ + {"key":"sys_updated_on","value":"datetime"},{"key":"sys_tags","value":"string"},{"key":"number","value":"string"},\ {"key":"sys_id","value":"string"},{"key":"received_by","value":"string"},{"key":"sys_updated_by","value":"string"},\ - {"key":"receiving_slip","value":"string"},{"key":"sys_created_on","value":"string"},\ + {"key":"receiving_slip","value":"string"},{"key":"sys_created_on","value":"datetime"},\ {"key":"sys_domain","value":"string"},{"key":"sys_created_by","value":"string"}] -schema.table.asset.covered=[{"key":"added","value":"string"},{"key":"contract","value":"string"},\ - {"key":"sys_mod_count","value":"string"},{"key":"sys_updated_on","value":"string"},\ +schema.table.asset.covered=[{"key":"added","value":"date"},{"key":"contract","value":"string"},\ + {"key":"sys_mod_count","value":"int"},{"key":"sys_updated_on","value":"datetime"},\ {"key":"sys_domain_path","value":"string"},{"key":"sys_tags","value":"string"},\ - {"key":"sys_id","value":"string"},{"key":"sys_updated_by","value":"string"},{"key":"removed","value":"string"},\ - {"key":"sys_created_on","value":"string"},{"key":"sys_domain","value":"string"},{"key":"asset","value":"string"},\ + {"key":"sys_id","value":"string"},{"key":"sys_updated_by","value":"string"},{"key":"removed","value":"date"},\ + {"key":"sys_created_on","value":"datetime"},{"key":"sys_domain","value":"string"},{"key":"asset","value":"string"},\ {"key":"sys_created_by","value":"string"}] schema.table.condition=[{"key":"condition_check","value":"string"},{"key":"sys_mod_count","value":"string"},\ diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 20c68a2b..3f61f553 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -23,7 +23,6 @@ import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.ConfigUtil; -import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -31,7 +30,6 @@ import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceValueType; -import io.cdap.plugin.servicenow.util.Util; import org.apache.http.HttpStatus; import org.apache.oltu.oauth2.common.exception.OAuthProblemException; import org.apache.oltu.oauth2.common.exception.OAuthSystemException; @@ -52,18 +50,17 @@ public class ServiceNowBaseConfig extends PluginConfig { @Macro @Nullable @Description("The existing connection to use.") - private ServiceNowConnectorConfig connection; - - @Nullable - public ServiceNowConnectorConfig getConnection() { - return connection; - } + private final ServiceNowConnectorConfig connection; public ServiceNowBaseConfig(String clientId, String clientSecret, String restApiEndpoint, String user, String password) { this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password); } + @Nullable + public ServiceNowConnectorConfig getConnection() { + return connection; + } /** * Validates {@link ServiceNowSourceConfig} instance. @@ -81,7 +78,7 @@ public void validateCredentials(FailureCollector collector) { validateServiceNowConnection(collector); } } - + @VisibleForTesting public void validateServiceNowConnection(FailureCollector collector) { try { @@ -89,8 +86,8 @@ public void validateServiceNowConnection(FailureCollector collector) { restApi.getAccessToken(); } catch (Exception e) { collector.addFailure("Unable to connect to ServiceNow Instance.", - "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password " + - "are correct.") + "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password " + + "are correct.") .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_ID) .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_SECRET) .withConfigProperty(ServiceNowConstants.PROPERTY_API_ENDPOINT) @@ -121,7 +118,8 @@ && shouldConnect() && !containsMacro(ServiceNowConstants.PROPERTY_VALUE_TYPE); } - public void validateTable(String tableName, SourceValueType valueType, FailureCollector collector) { + public void validateTable(String tableName, SourceValueType valueType, FailureCollector collector, + String tableField) { // Call API to fetch first record from the table ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( connection.getRestApiEndpoint(), tableName, false) @@ -142,11 +140,11 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo if (!apiResponse.isSuccess()) { if (apiResponse.getHttpStatus() == HttpStatus.SC_BAD_REQUEST) { collector.addFailure("Bad Request. Table: " + tableName + " is invalid.", "") - .withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME); + .withConfigProperty(tableField); } } else if (serviceNowTableAPIClient.parseResponseToResultListOfMap(apiResponse.getResponseBody()).isEmpty()) { - collector.addFailure("Table: " + tableName + " is empty.", "") - .withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME); + // Removed config property as in case of MultiSource, only first table error was populating. + collector.addFailure("Table: " + tableName + " is empty.", ""); } } catch (OAuthSystemException | OAuthProblemException e) { collector.addFailure("Unable to connect to ServiceNow Instance.", diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 1f9e76b3..a0be6fa0 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -29,11 +29,13 @@ import com.google.gson.reflect.TypeToken; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; -import io.cdap.plugin.servicenow.ServiceNowBaseConfig; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.sink.model.APIResponse; +import io.cdap.plugin.servicenow.sink.model.CreateRecordAPIResponse; import io.cdap.plugin.servicenow.sink.model.SchemaResponse; +import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; import io.cdap.plugin.servicenow.util.SchemaBuilder; import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; @@ -67,9 +69,9 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient { private static final String FIELD_CREATED_ON = "sys_created_on"; private static final String FIELD_UPDATED_ON = "sys_updated_on"; private static final String OAUTH_URL_TEMPLATE = "%s/oauth_token.do"; - private static final Gson gson = new Gson(); - public static JsonArray serviceNowJsonResultArray; + private static final Gson GSON = new Gson(); private final ServiceNowConnectorConfig conf; + public static JsonArray serviceNowJsonResultArray; public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf) { this.conf = conf; @@ -108,7 +110,7 @@ public String getAccessTokenRetryableMode() throws ExecutionException, RetryExce * @param limit The number of records to be fetched * @return The list of Map; each Map representing a table row */ - public List> fetchTableRecords(String tableName, SourceValueType valueType, String startDate, + public List> fetchTableRecords(String tableName, SourceValueType valueType, String startDate, String endDate, int offset, int limit) { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, false) @@ -144,144 +146,6 @@ public List> fetchTableRecords(String tableName, SourceValue } } - /** - * Create a new record in the ServiceNow Table - * - * @param tableName ServiceNow Table name - * @param entity Details of the Record to be created - */ - public String createRecord(String tableName, HttpEntity entity) throws IOException { - ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false); - String systemID; - RestAPIResponse apiResponse = null; - try { - String accessToken = getAccessToken(); - requestBuilder.setAuthHeader(accessToken); - requestBuilder.setAcceptHeader("application/json"); - requestBuilder.setContentTypeHeader("application/json"); - requestBuilder.setEntity(entity); - apiResponse = executePost(requestBuilder.build()); - - systemID = String.valueOf(getSystemId(apiResponse)); - - if (!apiResponse.isSuccess()) { - LOG.error("Error - {}", getErrorMessage(apiResponse.getResponseBody())); - } else { - LOG.info(apiResponse.getResponseBody()); - } - } catch (OAuthSystemException | OAuthProblemException | UnsupportedEncodingException e) { - LOG.error("Error in creating a new record", e); - throw new RuntimeException("Error in creating a new record"); - } - - return systemID; - } - - /** - * Fetches the System Id of a new Record. - * - * @param apiResponse API response after Creating a record - */ - - private String getSystemId(RestAPIResponse apiResponse) { - JsonObject jsonObject = gson.fromJson(apiResponse.getResponseBody(), JsonObject.class); - JsonObject result = (JsonObject) jsonObject.get(ServiceNowConstants.RESULT); - - return result.get(ServiceNowConstants.SYSTEM_ID).getAsString(); - } - - /** - * Return a record from ServiceNow application. - * - * @param tableName The ServiceNow table name - * @param query The query - */ - public Map getRecordFromServiceNowTable(String tableName, String query) - throws OAuthProblemException, OAuthSystemException { - - ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false) - .setQuery(query); - - RestAPIResponse apiResponse = null; - String accessToken = getAccessToken(); - requestBuilder.setAuthHeader(accessToken); - apiResponse = executeGet(requestBuilder.build()); - JsonObject jsonObject = gson.fromJson(apiResponse.getResponseBody(), JsonObject.class); - serviceNowJsonResultArray = jsonObject.getAsJsonArray(ServiceNowConstants.RESULT); - Map responseMap = gson.fromJson(serviceNowJsonResultArray.get(0), Map.class); - - return responseMap; - } - - - /** - * Fetches the table schema for ServiceNow table. - * - * @param tableName The ServiceNow table name - * @param valueType The value type - * @param startDate The start date - * @param endDate The end date - * @param fetchRecordCount A flag that decides whether to fetch total record count or not - * @return - */ - public ServiceNowTableDataResponse fetchTableSchema(String tableName, SourceValueType valueType, String startDate, - String endDate, boolean fetchRecordCount) { - return fetchTableSchemaUsingFirstRecord(tableName, valueType, startDate, endDate, fetchRecordCount); - } - - private ServiceNowTableDataResponse fetchTableSchemaUsingFirstRecord(String tableName, SourceValueType valueType, - String startDate, String endDate, - boolean fetchRecordCount) { - ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false) - .setExcludeReferenceLink(true) - .setDisplayValue(valueType) - .setLimit(1); - applyDateRangeToRequest(requestBuilder, startDate, endDate); - - RestAPIResponse apiResponse = null; - - try { - String accessToken = getAccessToken(); - requestBuilder.setAuthHeader(accessToken); - - // Get the response JSON and fetch the header X-Total-Count. Set the value to recordCount - if (fetchRecordCount) { - requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); - } - - apiResponse = executeGet(requestBuilder.build()); - if (!apiResponse.isSuccess()) { - LOG.error("Error - {}", getErrorMessage(apiResponse.getResponseBody())); - return null; - } - - ServiceNowTableDataResponse tableDataResponse = new ServiceNowTableDataResponse(); - - List> result = parseResponseToResultListOfMap(apiResponse.getResponseBody()); - List columns = new ArrayList<>(); - - if (result != null && !result.isEmpty()) { - Map firstRecord = result.get(0); - for (String key : firstRecord.keySet()) { - columns.add(new ServiceNowColumn(key, "string")); - } - } - - tableDataResponse.setColumns(columns); - if (fetchRecordCount) { - tableDataResponse.setTotalRecordCount(getRecordCountFromHeader(apiResponse)); - } - - return tableDataResponse; - } catch (OAuthSystemException | OAuthProblemException e) { - LOG.error("Error in fetchFirstRecordFromTable", e); - return null; - } - } - private void applyDateRangeToRequest(ServiceNowTableAPIRequestBuilder requestBuilder, String startDate, String endDate) { String dateRange = generateDateRangeQuery(startDate, endDate); @@ -312,23 +176,36 @@ private int getRecordCountFromHeader(RestAPIResponse apiResponse) { return Strings.isNullOrEmpty(headerValue) ? 0 : Integer.parseInt(headerValue); } - public List> parseResponseToResultListOfMap(String responseBody) { + public List> parseResponseToResultListOfMap(String responseBody) { - JsonObject jo = gson.fromJson(responseBody, JsonObject.class); + JsonObject jo = GSON.fromJson(responseBody, JsonObject.class); JsonArray ja = jo.getAsJsonArray(ServiceNowConstants.RESULT); Type type = new TypeToken>>() { }.getType(); - return gson.fromJson(ja, type); + return GSON.fromJson(ja, type); } private String getErrorMessage(String responseBody) { try { - JsonObject jo = gson.fromJson(responseBody, JsonObject.class); - return jo.getAsJsonObject(ServiceNowConstants.ERROR).get(ServiceNowConstants.MESSAGE).getAsString(); + JsonObject jo = GSON.fromJson(responseBody, JsonObject.class); + JsonObject error = jo.getAsJsonObject(ServiceNowConstants.ERROR); + if (error != null) { + String errorMessage = error.get(ServiceNowConstants.MESSAGE).getAsString(); + String errorDetail = error.get(ServiceNowConstants.ERROR_DETAIL).getAsString(); + if (errorMessage != null && errorDetail != null) { + return String.format("%s:%s", + jo.getAsJsonObject(ServiceNowConstants.ERROR).get(ServiceNowConstants.MESSAGE) + .getAsString(), + jo.getAsJsonObject(ServiceNowConstants.ERROR).get(ServiceNowConstants.ERROR_DETAIL) + .getAsString()); + } + } + return null; + } catch (Exception e) { - return e.getMessage(); + return String.format("%s:%s", e.getMessage(), responseBody); } } @@ -344,10 +221,10 @@ private String getErrorMessage(String responseBody) { * @param limit The number of records to be fetched * @return The list of Map; each Map representing a table row */ - public List> fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, + public List> fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, String startDate, String endDate, int offset, int limit) throws IOException { - final List> results = new ArrayList<>(); + final List> results = new ArrayList<>(); Callable fetchRecords = () -> { results.addAll(fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit)); return true; @@ -374,45 +251,137 @@ public List> fetchTableRecordsRetryableMode(String tableName * @return schema for given ServiceNow table */ @Nullable - public Schema fetchServiceNowTableSchema(String tableName, FailureCollector collector) { + public Schema fetchTableSchema(String tableName, FailureCollector collector) { + Schema schema = null; + try { + schema = fetchTableSchema(tableName); + } catch (OAuthProblemException | OAuthSystemException | RuntimeException e) { + LOG.error("Error in connection - {}", e.getMessage()); + collector.addFailure(String.format("Connection failed. Unable to fetch schema for table: %s. Cause: %s", + tableName, e.getStackTrace()), null); + } + return schema; + } + + @VisibleForTesting + public SchemaResponse parseSchemaResponse(String responseBody) { + return GSON.fromJson(responseBody, SchemaResponse.class); + } + + /** + * Fetches the table schema from ServiceNow + * + * @param tableName ServiceNow table name for which schema is getting fetched + * @return schema for given ServiceNow table + * @throws OAuthProblemException + * @throws OAuthSystemException + */ + public Schema fetchTableSchema(String tableName) throws OAuthProblemException, OAuthSystemException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, true) .setExcludeReferenceLink(true); RestAPIResponse apiResponse; + String accessToken = getAccessToken(); + requestBuilder.setAuthHeader(accessToken); + apiResponse = executeGet(requestBuilder.build()); + if (!apiResponse.isSuccess()) { + throw new RuntimeException("Error - " + getErrorMessage(apiResponse.getResponseBody())); + } + SchemaResponse response = parseSchemaResponse(apiResponse.getResponseBody()); + List columns = new ArrayList<>(); + + if (response.getResult() == null && response.getResult().isEmpty()) { + throw new RuntimeException("Error - Schema Response does not contain any result"); + } + for (ServiceNowSchemaField field : response.getResult()) { + columns.add(new ServiceNowColumn(field.getName(), field.getInternalType())); + } + return SchemaBuilder.constructSchema(tableName, columns); + } + + /** + * Get the total number of records in the table + * + * @param tableName ServiceNow table name for which record count is fetched. + * @return the table record count + * @throws OAuthProblemException + * @throws OAuthSystemException + */ + public int getTableRecordCount(String tableName) throws OAuthProblemException, OAuthSystemException { + ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( + this.conf.getRestApiEndpoint(), tableName, false) + .setExcludeReferenceLink(true) + .setDisplayValue(SourceValueType.SHOW_DISPLAY_VALUE) + .setLimit(1); + RestAPIResponse apiResponse = null; + String accessToken = getAccessToken(); + requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); + requestBuilder.setAuthHeader(accessToken); + apiResponse = executeGet(requestBuilder.build()); + if (!apiResponse.isSuccess()) { + throw new RuntimeException("Error : " + apiResponse); + } + return getRecordCountFromHeader(apiResponse); + } + + /** + * Create a new record in the ServiceNow Table + * + * @param tableName ServiceNow Table name + * @param entity Details of the Record to be created + * @description This function is being used in end-to-end (e2e) tests to fetch a record from the ServiceNow Table. + */ + public String createRecord(String tableName, HttpEntity entity) throws IOException { + ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( + this.conf.getRestApiEndpoint(), tableName, false); + String systemID; + RestAPIResponse apiResponse = null; try { String accessToken = getAccessToken(); requestBuilder.setAuthHeader(accessToken); - apiResponse = executeGet(requestBuilder.build()); + requestBuilder.setAcceptHeader("application/json"); + requestBuilder.setContentTypeHeader("application/json"); + requestBuilder.setEntity(entity); + apiResponse = executePost(requestBuilder.build()); + + systemID = String.valueOf(getSystemId(apiResponse)); + if (!apiResponse.isSuccess()) { LOG.error("Error - {}", getErrorMessage(apiResponse.getResponseBody())); - collector.addFailure("Unable to fetch schema for table " + tableName, null). - withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME); - return null; - } - List result = parseSchemaResponse(apiResponse.getResponseBody()); - List columns = new ArrayList<>(); - - if (result != null && !result.isEmpty()) { - for (SchemaResponse field : result) { - columns.add(new ServiceNowColumn(field.getName(), field.getInternalType())); - } } - return SchemaBuilder.constructSchema(tableName, columns); - } catch (OAuthSystemException | OAuthProblemException e) { - LOG.error("Error in connection - {}", e); - collector.addFailure("Connection failed. Unable to fetch schema for table " + tableName, null); + } catch (OAuthSystemException | OAuthProblemException | UnsupportedEncodingException e) { + throw new IOException("Error in creating a new record", e); } - return null; + return systemID; } - @VisibleForTesting - public List parseSchemaResponse(String responseBody) { - JsonObject jo = gson.fromJson(responseBody, JsonObject.class); - JsonArray ja = jo.getAsJsonArray(ServiceNowConstants.RESULT); - Type type = new TypeToken>() { - }.getType(); - return gson.fromJson(ja, type); + private String getSystemId(RestAPIResponse restAPIResponse) { + CreateRecordAPIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), + CreateRecordAPIResponse.class); + return apiResponse.getResult().get(ServiceNowConstants.SYSTEM_ID).toString(); } + /** + * This function is being used in end-to-end (e2e) tests to fetch a record + * Return a record from ServiceNow application. + * + * @param tableName The ServiceNow table name + * @param query The query + */ + public Map getRecordFromServiceNowTable(String tableName, String query) + throws OAuthProblemException, OAuthSystemException { + + ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( + this.conf.getRestApiEndpoint(), tableName, false) + .setQuery(query); + + RestAPIResponse restAPIResponse; + String accessToken = getAccessToken(); + requestBuilder.setAuthHeader(accessToken); + restAPIResponse = executeGet(requestBuilder.build()); + + APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class); + return apiResponse.getResult().get(0); + } } diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableDataResponse.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableDataResponse.java index 43e5cd0e..b42aef9f 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableDataResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableDataResponse.java @@ -29,7 +29,7 @@ public class ServiceNowTableDataResponse { private List columns; - private List> result; + private List> result; public int getTotalRecordCount() { return totalRecordCount; @@ -47,11 +47,11 @@ public void setColumns(List columns) { this.columns = columns; } - public List> getResult() { + public List> getResult() { return result; } - public void setResult(List> result) { + public void setResult(List> result) { this.result = result; } } diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index e485eec0..f7ddb976 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -58,7 +58,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import javax.annotation.Nullable; import javax.ws.rs.core.MediaType; @@ -153,7 +152,7 @@ public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSp specBuilder.setSchema(schema); } return specBuilder.addRelatedPlugin(new PluginSpec(ServiceNowConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, - properties)) + properties)) .addRelatedPlugin(new PluginSpec(ServiceNowConstants.PLUGIN_NAME, BatchSink.PLUGIN_TYPE, properties)).build(); } @@ -194,7 +193,7 @@ private List getTableData(String tableName, int limit) throws requestBuilder.setAuthHeader(accessToken); requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); apiResponse = serviceNowTableAPIClient.executeGet(requestBuilder.build()); - List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap + List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap (apiResponse.getResponseBody()); List recordList = new ArrayList<>(); Schema schema = getSchema(tableName); @@ -204,8 +203,7 @@ private List getTableData(String tableName, int limit) throws StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); for (Schema.Field field : tableFields) { String fieldName = field.getName(); - Object fieldValue = config.convertToValue(fieldName, field.getSchema(), result.get(i)); - recordBuilder.set(fieldName, fieldValue); + ServiceNowRecordConverter.convertToValue(fieldName, field.getSchema(), result.get(i), recordBuilder); } StructuredRecord structuredRecord = recordBuilder.build(); recordList.add(structuredRecord); @@ -214,14 +212,12 @@ private List getTableData(String tableName, int limit) throws return recordList; } - + @Nullable private Schema getSchema(String tableName) { SourceQueryMode mode = SourceQueryMode.TABLE; - SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; List tableInfo = ServiceNowInputFormat.fetchTableInfo(mode, config, tableName, - null, valueType, - null, null); + null); Schema schema = tableInfo.stream().findFirst().isPresent() ? tableInfo.stream().findFirst().get().getSchema() : null; return schema; diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java index a3dac2c4..61114276 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java @@ -15,47 +15,50 @@ */ package io.cdap.plugin.servicenow.connector; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; -import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.Util; -import java.util.Map; +import javax.annotation.Nullable; /** * PluginConfig for ServiceNow Connector */ public class ServiceNowConnectorConfig extends PluginConfig { + @Name(ServiceNowConstants.PROPERTY_CLIENT_ID) @Macro - @Description(" The Client ID for ServiceNow Instance.") + @Nullable + @Description("The Client ID for ServiceNow Instance.") private final String clientId; @Name(ServiceNowConstants.PROPERTY_CLIENT_SECRET) @Macro + @Nullable @Description("The Client Secret for ServiceNow Instance.") private final String clientSecret; @Name(ServiceNowConstants.PROPERTY_API_ENDPOINT) @Macro + @Nullable @Description("The REST API Endpoint for ServiceNow Instance. For example, https://instance.service-now.com") private final String restApiEndpoint; @Name(ServiceNowConstants.PROPERTY_USER) @Macro + @Nullable @Description("The user name for ServiceNow Instance.") private final String user; @Name(ServiceNowConstants.PROPERTY_PASSWORD) @Macro + @Nullable @Description("The password for ServiceNow Instance.") private final String password; @@ -136,61 +139,5 @@ public void validateConnection(FailureCollector collector) { } } - public Object convertToValue(String fieldName, Schema fieldSchema, Map record) { - Schema.Type fieldType = fieldSchema.getType(); - Object fieldValue = record.get(fieldName); - - switch (fieldType) { - case STRING: - return convertToStringValue(fieldValue); - case DOUBLE: - return convertToDoubleValue(fieldValue); - case INT: - return convertToIntegerValue(fieldValue); - case BOOLEAN: - return convertToBooleanValue(fieldValue); - case UNION: - if (fieldSchema.isNullable()) { - return convertToValue(fieldName, fieldSchema.getNonNullable(), record); - } - throw new IllegalStateException( - String.format("Field '%s' is of unexpected type '%s'. Declared 'complex UNION' types: %s", - fieldName, record.get(fieldName).getClass().getSimpleName(), fieldSchema.getUnionSchemas())); - default: - throw new IllegalStateException( - String.format("Record type '%s' is not supported for field '%s'", fieldType.name(), fieldName)); - } - } - - @VisibleForTesting - public String convertToStringValue(Object fieldValue) { - return String.valueOf(fieldValue); - } - - @VisibleForTesting - public Double convertToDoubleValue(Object fieldValue) { - if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { - return null; - } - - return Double.parseDouble(String.valueOf(fieldValue)); - } - - @VisibleForTesting - public Integer convertToIntegerValue(Object fieldValue) { - if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { - return null; - } - - return Integer.parseInt(String.valueOf(fieldValue)); - } - - @VisibleForTesting - public Boolean convertToBooleanValue(Object fieldValue) { - if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { - return null; - } - return Boolean.parseBoolean(String.valueOf(fieldValue)); - } } diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java new file mode 100644 index 00000000..9ff2dccc --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java @@ -0,0 +1,142 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.servicenow.connector; + +import com.google.common.annotations.VisibleForTesting; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.format.UnexpectedFormatException; +import io.cdap.cdap.api.data.schema.Schema; + +import java.text.NumberFormat; +import java.text.ParseException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Locale; +import java.util.Map; + +/** + * Utility class for converting the record from ServiceNow data type to CDAP schema data types + */ +public class ServiceNowRecordConverter { + private static final String DATE_PATTERN = "yyyy-MM-dd"; + private static final String DATE_TIME_PATTERN = "yyyy-MM-dd HH:mm:ss"; + private static final String TIME_PATTERN = "HH:mm:ss"; + + public static void convertToValue(String fieldName, Schema fieldSchema, Map record, + StructuredRecord.Builder recordBuilder) { + String fieldValue = record.get(fieldName); + if (fieldValue == null || fieldValue.isEmpty()) { + // Set 'null' value as it is + recordBuilder.set(fieldName, null); + return; + } + + fieldSchema = fieldSchema.isNullable() ? fieldSchema.getNonNullable() : fieldSchema; + Schema.LogicalType fieldLogicalType = fieldSchema.getLogicalType(); + // Get values of logical types properly + if (fieldLogicalType != null) { + switch (fieldLogicalType) { + case DATETIME: + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_TIME_PATTERN); + try { + recordBuilder.setDateTime(fieldName, LocalDateTime.parse(fieldValue, dateTimeFormatter)); + } catch (DateTimeParseException exception) { + throw new UnexpectedFormatException( + String.format("Field '%s' of type '%s' with value '%s' is not in ISO-8601 format.", + fieldName, fieldSchema.getDisplayName(), fieldValue), exception); + } + return; + case DATE: + DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern(DATE_PATTERN); + try { + recordBuilder.setDate(fieldName, LocalDate.parse(fieldValue, dateFormatter)); + } catch (DateTimeParseException exception) { + throw new UnexpectedFormatException( + String.format("Field '%s' of type '%s' with value '%s' is not in ISO-8601 format.", + fieldName, fieldSchema.getDisplayName(), fieldValue), exception); + } + return; + case TIME_MICROS: + DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern(TIME_PATTERN); + try { + recordBuilder.setTime(fieldName, LocalTime.parse(fieldValue, timeFormatter)); + } catch (DateTimeParseException exception) { + throw new UnexpectedFormatException( + String.format("Field '%s' of type '%s' with value '%s' is not in ISO-8601 format.", + fieldName, fieldSchema.getDisplayName(), fieldValue), exception); + } + return; + default: + throw new IllegalStateException(String.format("Field '%s' is of unsupported type '%s'", fieldName, + fieldLogicalType.name().toLowerCase())); + } + } + + Schema.Type fieldType = fieldSchema.getType(); + switch (fieldType) { + case STRING: + recordBuilder.set(fieldName, fieldValue); + return; + case DOUBLE: + recordBuilder.set(fieldName, convertToDoubleValue(fieldValue)); + return; + case INT: + recordBuilder.set(fieldName, convertToIntegerValue(fieldValue)); + return; + case BOOLEAN: + recordBuilder.set(fieldName, convertToBooleanValue(fieldValue)); + return; + default: + throw new IllegalStateException( + String.format("Record type '%s' is not supported for field '%s'", fieldType.name(), fieldName)); + } + + } + + @VisibleForTesting + public static Double convertToDoubleValue(String fieldValue) { + try { + return NumberFormat.getNumberInstance(Locale.US).parse(fieldValue).doubleValue(); + } catch (ParseException exception) { + throw new UnexpectedFormatException( + String.format("Field with value '%s' is not in valid format.", fieldValue), exception); + } + } + + @VisibleForTesting + public static Integer convertToIntegerValue(String fieldValue) { + try { + return NumberFormat.getNumberInstance(java.util.Locale.US).parse(fieldValue).intValue(); + } catch (ParseException exception) { + throw new UnexpectedFormatException( + String.format("Field with value '%s' is not in valid format.", fieldValue), exception); + } + } + + @VisibleForTesting + public static Boolean convertToBooleanValue(String fieldValue) { + if (fieldValue.equalsIgnoreCase(Boolean.TRUE.toString()) || + fieldValue.equalsIgnoreCase(Boolean.FALSE.toString())) { + return Boolean.parseBoolean(fieldValue); + } + throw new UnexpectedFormatException( + String.format("Field with value '%s' is not in valid format.", fieldValue)); + + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSink.java b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSink.java index 47522d42..d95f4b4d 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSink.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSink.java @@ -47,7 +47,7 @@ /** * A {@link BatchSink} that writes data into the specified table in ServiceNow. */ -@Plugin(type = BatchSink.PLUGIN_TYPE) +// ServiceNow Batch Sink Plugin is not ready for release yet. @Name(ServiceNowConstants.PLUGIN_NAME) @Description("Writes to the target table in ServiceNow.") @Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = ServiceNowConstants.PLUGIN_NAME)}) diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java index 874defc8..8438055c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java @@ -26,7 +26,6 @@ import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.servicenow.ServiceNowBaseConfig; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.Util; @@ -44,10 +43,6 @@ public class ServiceNowSinkConfig extends ServiceNowBaseConfig { public static final String PROPERTY_EXTERNAL_ID_FIELD = "externalIdField"; - @Name("referenceName") - @Description("This will be used to uniquely identify this source/sink for lineage, annotating metadata, etc.") - public String referenceName; - @Name(ServiceNowConstants.PROPERTY_TABLE_NAME) @Macro @Description("The name of the ServiceNow table to which data needs to be inserted.") @@ -59,6 +54,10 @@ public class ServiceNowSinkConfig extends ServiceNowBaseConfig { "must be present.") private final String operation; + @Name("referenceName") + @Description("This will be used to uniquely identify this source/sink for lineage, annotating metadata, etc.") + public String referenceName; + @Name(ServiceNowConstants.NAME_SCHEMA) @Macro @Nullable @@ -149,7 +148,7 @@ void validateSchema(Schema schema, FailureCollector collector) { return; } ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(this.getConnection()); - Schema tableSchema = restApi.fetchServiceNowTableSchema(tableName, collector); + Schema tableSchema = restApi.fetchTableSchema(tableName, collector); if (tableSchema == null) { throw collector.getOrThrowException(); } @@ -224,7 +223,7 @@ void checkCompatibility(Schema actualSchema, Schema providedSchema, FailureColle || !Objects.equals(actualFieldSchema.getLogicalType(), providedFieldSchema.getLogicalType())) { collector.addFailure( String.format("Expected field '%s' to be of '%s', but it is of '%s'", - providedField.getName(), providedFieldSchema, actualFieldSchema), null) + providedField.getName(), providedFieldSchema, actualFieldSchema), null) .withInputSchemaField(providedField.getName()); } } @@ -241,8 +240,7 @@ private Schema convertToServiceNowCompatibleDataTypes(Schema providedFieldSchema switch (providedFieldSchema.getType()) { case FLOAT: case DOUBLE: - providedFieldSchema = Schema.decimalOf(ServiceNowConstants.DEFAULT_PRECISION, - ServiceNowConstants.DEFAULT_SCALE); + providedFieldSchema = Schema.of(Schema.Type.DOUBLE); } if (providedFieldSchema.getLogicalType() != null) { diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/APIResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/APIResponse.java new file mode 100644 index 00000000..f3333c9e --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/APIResponse.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +import java.util.List; +import java.util.Map; + +/** + * Model class for API Response from ServiceNow Table APIs + */ +public class APIResponse { + + private List> result; + + public List> getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/CreateRecordAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/CreateRecordAPIResponse.java new file mode 100644 index 00000000..d02d1cae --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/CreateRecordAPIResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +import java.util.Map; + +/** + * Model class for API Response from ServiceNow Table Create Record APIs + */ +public class CreateRecordAPIResponse { + + private Map result; + + public Map getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java index a9a732ee..ea5191ef 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java @@ -16,44 +16,21 @@ package io.cdap.plugin.servicenow.sink.model; +import java.util.List; + /** * Model class for Schema Response from Schema API */ public class SchemaResponse { - private String label; - private String exampleValue; - private String internalType; - private String name; - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getExampleValue() { - return exampleValue; - } - - public void setExampleValue(String exampleValue) { - this.exampleValue = exampleValue; - } - public String getInternalType() { - return internalType; - } - - public void setInternalType(String internalType) { - this.internalType = internalType; - } + private final List result; - public String getName() { - return name; + public SchemaResponse(List result) { + this.result = result; } - public void setName(String name) { - this.name = name; + public List getResult() { + return result; } + } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java new file mode 100644 index 00000000..e3c44c38 --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +/** + * Model class for Schema Field from Schema API + */ +public class ServiceNowSchemaField { + private final String label; + private final String exampleValue; + private final String internalType; + private final String name; + + public ServiceNowSchemaField(String label, String exampleValue, String internalType, String name) { + this.label = label; + this.exampleValue = exampleValue; + this.internalType = internalType; + this.name = name; + } + + public String getLabel() { + return label; + } + + public String getExampleValue() { + return exampleValue; + } + + public String getInternalType() { + return internalType; + } + + public String getName() { + return name; + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 4d80cd3b..4cd903c4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -16,14 +16,10 @@ package io.cdap.plugin.servicenow.source; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; import java.io.IOException; import java.util.Iterator; @@ -41,18 +37,13 @@ public abstract class ServiceNowBaseRecordReader extends RecordReader> results; - protected Iterator> iterator; - protected Map row; + protected List> results; + protected Iterator> iterator; + protected Map row; public ServiceNowBaseRecordReader() { } - public void initialize(InputSplit split, TaskAttemptContext context) { - this.split = (ServiceNowInputSplit) split; - this.pos = 0; - } - public abstract boolean nextKeyValue() throws IOException; public NullWritable getCurrentKey() { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java index 1850d8d2..bad1302f 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java @@ -24,6 +24,7 @@ import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.servicenow.ServiceNowBaseConfig; + import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceValueType; import io.cdap.plugin.servicenow.util.Util; diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java index 3e07ef0f..371a8950 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java @@ -18,17 +18,12 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; - import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; -import io.cdap.plugin.servicenow.util.SchemaBuilder; -import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; -import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -36,6 +31,8 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.oltu.oauth2.common.exception.OAuthProblemException; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,9 +64,7 @@ public static List setInput(Configuration jobConfig, Source // Depending on conf value fetch the list of fields for each table and create schema object // return the schema object for each table as ServiceNowTableInfo List tableInfos = fetchTableInfo(mode, conf.getConnection(), conf.getTableName(), - conf.getApplicationName(), conf.getValueType(), - conf.getStartDate(), conf.getEndDate()); - + conf.getApplicationName()); jobConf.setTableInfos(tableInfos); return tableInfos; @@ -77,12 +72,10 @@ public static List setInput(Configuration jobConfig, Source public static List fetchTableInfo(SourceQueryMode mode, ServiceNowConnectorConfig conf, @Nullable String tableName, - @Nullable SourceApplication application, - SourceValueType valueType, @Nullable String startDate, - @Nullable String endDate) { + @Nullable SourceApplication application) { // When mode = Table, fetch details from the table name provided in plugin config if (mode == SourceQueryMode.TABLE) { - ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf, valueType, startDate, endDate); + ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -92,7 +85,7 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser List tableNames = application.getTableNames(); for (String table : tableNames) { - ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, valueType, startDate, endDate); + ServiceNowTableInfo tableInfo = getTableMetaData(table, conf); if (tableInfo == null) { continue; } @@ -102,25 +95,20 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser return tableInfos; } - private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf, - SourceValueType valueType, String startDate, String endDate) { + private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf) { // Call API to fetch first record from the table ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf); - ServiceNowTableDataResponse response = restApi.fetchTableSchema(tableName, valueType, startDate, endDate, - true); - if (response == null) { - return null; + Schema schema = null; + int recordCount = 0; + try { + schema = restApi.fetchTableSchema(tableName); + recordCount = restApi.getTableRecordCount(tableName); + } catch (OAuthProblemException | OAuthSystemException e) { + throw new RuntimeException(String.format("Error in fetching table metadata due to reason: %s", e.getMessage()), + e); } - - List columns = response.getColumns(); - if (columns == null || columns.isEmpty()) { - return null; - } - - Schema schema = SchemaBuilder.constructSchema(tableName, columns); - LOG.debug("table {}, rows = {}", tableName, response.getTotalRecordCount()); - return new ServiceNowTableInfo(tableName, schema, response.getTotalRecordCount()); + return new ServiceNowTableInfo(tableName, schema, recordCount); } @Override diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java index aa727ca9..f4e5410a 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java @@ -20,13 +20,9 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; -import io.cdap.plugin.servicenow.util.SchemaBuilder; -import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; -import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -34,6 +30,8 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.oltu.oauth2.common.exception.OAuthProblemException; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,23 +65,20 @@ public static Set setInput(Configuration jobConfig, // Depending on conf value fetch the list of fields for each table and create schema object // return the schema object for each table as ServiceNowTableInfo - Set tableInfos = fetchTablesInfo(conf.getConnection(), conf.getTableNames(), - conf.getValueType(), - conf.getStartDate(), conf.getEndDate()); + Set tableInfos = fetchTablesInfo(conf.getConnection(), conf.getTableNames()); jobConf.setTableInfos(tableInfos.stream().collect(Collectors.toList())); return tableInfos; } - static Set fetchTablesInfo(ServiceNowConnectorConfig conf, String tableNames, - SourceValueType valueType, String startDate, String endDate) { + static Set fetchTablesInfo(ServiceNowConnectorConfig conf, String tableNames) { Set tablesInfos = new LinkedHashSet<>(); Set tableNameSet = getList(tableNames); for (String table : tableNameSet) { - ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, valueType, startDate, endDate); + ServiceNowTableInfo tableInfo = getTableMetaData(table, conf); if (tableInfo == null) { continue; } @@ -93,25 +88,20 @@ static Set fetchTablesInfo(ServiceNowConnectorConfig conf, return tablesInfos; } - private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf, - SourceValueType valueType, String startDate, String endDate) { + private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf) { // Call API to fetch first record from the table ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf); - ServiceNowTableDataResponse response = restApi.fetchTableSchema(tableName, valueType, startDate, endDate, - true); - if (response == null) { - return null; + Schema schema; + int recordCount; + try { + schema = restApi.fetchTableSchema(tableName); + recordCount = restApi.getTableRecordCount(tableName); + } catch (OAuthProblemException | OAuthSystemException e) { + throw new RuntimeException(e); } - - List columns = response.getColumns(); - if (columns == null || columns.isEmpty()) { - return null; - } - - Schema schema = SchemaBuilder.constructSchema(tableName, columns); - LOG.debug("table {}, rows = {}", tableName, response.getTotalRecordCount()); - return new ServiceNowTableInfo(tableName, schema, response.getTotalRecordCount()); + LOG.debug("table {}, rows = {}", tableName, recordCount); + return new ServiceNowTableInfo(tableName, schema, recordCount); } public static Set getList(String value) { @@ -134,8 +124,8 @@ public List getSplits(JobContext jobContext) throws IOException, Int String tableName = tableInfo.getTableName(); int totalRecords = tableInfo.getRecordCount(); - int pages = (tableInfo.getRecordCount() / ServiceNowConstants.PAGE_SIZE); - if (tableInfo.getRecordCount() % ServiceNowConstants.PAGE_SIZE > 0) { + int pages = (totalRecords / ServiceNowConstants.PAGE_SIZE); + if (totalRecords % ServiceNowConstants.PAGE_SIZE > 0) { pages++; } int offset = 0; diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index d35cae2a..00f1d448 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -20,9 +20,12 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; -import io.cdap.plugin.servicenow.util.SchemaBuilder; +import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowConstants; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.oltu.oauth2.common.exception.OAuthProblemException; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,14 +37,25 @@ * Record reader that reads the entire contents of a ServiceNow table. */ public class ServiceNowMultiRecordReader extends ServiceNowBaseRecordReader { - private static final Logger LOG = LoggerFactory.getLogger(ServiceNowMultiRecordReader.class); + private final ServiceNowMultiSourceConfig multiSourcePluginConf; + private ServiceNowTableAPIClientImpl restApi; ServiceNowMultiRecordReader(ServiceNowMultiSourceConfig multiSourcePluginConf) { super(); this.multiSourcePluginConf = multiSourcePluginConf; } + @Override + public void initialize(InputSplit split, TaskAttemptContext context) { + this.split = (ServiceNowInputSplit) split; + this.pos = 0; + restApi = new ServiceNowTableAPIClientImpl(multiSourcePluginConf.getConnection()); + tableName = ((ServiceNowInputSplit) split).getTableName(); + tableNameField = multiSourcePluginConf.getTableNameField(); + fetchSchema(restApi); + } + @Override public boolean nextKeyValue() throws IOException { try { @@ -70,8 +84,8 @@ public StructuredRecord getCurrentValue() throws IOException { try { for (Schema.Field field : tableFields) { String fieldName = field.getName(); - Object fieldValue = multiSourcePluginConf.getConnection().convertToValue(fieldName, field.getSchema(), row); - recordBuilder.set(fieldName, fieldValue); + ServiceNowRecordConverter.convertToValue(fieldName, field.getSchema(), row, + recordBuilder); } } catch (Exception e) { throw new IOException("Error decoding row from table " + tableName, e); @@ -81,39 +95,26 @@ public StructuredRecord getCurrentValue() throws IOException { @VisibleForTesting void fetchData() throws IOException { - tableName = split.getTableName(); - tableNameField = multiSourcePluginConf.getTableNameField(); - - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(multiSourcePluginConf.getConnection()); - // Get the table data results = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(), multiSourcePluginConf.getStartDate(), multiSourcePluginConf.getEndDate(), split.getOffset(), ServiceNowConstants.PAGE_SIZE); - LOG.debug("Results size={}", results.size()); - if (!results.isEmpty()) { - fetchSchema(restApi); - } iterator = results.iterator(); } private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { - // Fetch the column definition - ServiceNowTableDataResponse response = restApi.fetchTableSchema(tableName, multiSourcePluginConf.getValueType(), - null, null, false); - if (response == null) { - return; + // Fetch the schema + try { + Schema tempSchema = restApi.fetchTableSchema(tableName); + tableFields = tempSchema.getFields(); + List schemaFields = new ArrayList<>(tableFields); + schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); + schema = Schema.recordOf(tableName, schemaFields); + } catch (OAuthProblemException | OAuthSystemException e) { + throw new RuntimeException(e); } - - // Build schema - Schema tempSchema = SchemaBuilder.constructSchema(tableName, response.getColumns()); - tableFields = tempSchema.getFields(); - List schemaFields = new ArrayList<>(tableFields); - schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); - - schema = Schema.recordOf(tableName, schemaFields); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSource.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSource.java index 7b70e64e..3d03b174 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSource.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSource.java @@ -34,7 +34,6 @@ import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.slf4j.Logger; @@ -92,7 +91,7 @@ public void prepareRun(BatchSourceContext context) { } context.setInput(Input.of(conf.getReferenceName(), - new SourceInputFormatProvider(ServiceNowMultiInputFormat.class, hConf))); + new SourceInputFormatProvider(ServiceNowMultiInputFormat.class, hConf))); } @Override @@ -109,8 +108,8 @@ private void recordLineage(BatchSourceContext context, ServiceNowTableInfo table List fields = Objects.requireNonNull(schema).getFields(); if (fields != null && !fields.isEmpty()) { lineageRecorder.recordRead("Read", - String.format("Read from '%s' ServiceNow table.", tableName), - fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); + String.format("Read from '%s' ServiceNow table.", tableName), + fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); } } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java index cfb34d47..12572495 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java @@ -40,22 +40,22 @@ public class ServiceNowMultiSourceConfig extends ServiceNowBaseSourceConfig { /** * Constructor for ServiceNowSourceConfig object. * - * @param referenceName The reference name - * @param tableNameField The field name to hold the table name value - * @param clientId The Client Id for ServiceNow - * @param clientSecret The Client Secret for ServiceNow + * @param referenceName The reference name + * @param tableNameField The field name to hold the table name value + * @param clientId The Client Id for ServiceNow + * @param clientSecret The Client Secret for ServiceNow * @param restApiEndpoint The rest API endpoint for ServiceNow - * @param user The user id for ServiceNow - * @param password The password for ServiceNow - * @param valueType The value type - * @param startDate The start date - * @param endDate The end date + * @param user The user id for ServiceNow + * @param password The password for ServiceNow + * @param valueType The value type + * @param startDate The start date + * @param endDate The end date */ public ServiceNowMultiSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint, String user, String password, String tableNameField, String valueType, @Nullable String startDate, @Nullable String endDate, String tableNames) { super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate, - endDate); + endDate); this.tableNames = tableNames; } @@ -100,7 +100,7 @@ void validateTableNames(FailureCollector collector) { } else { Set tableNames = ServiceNowMultiInputFormat.getList(getTableNames()); for (String tableName : tableNames) { - validateTable(tableName, getValueType(), collector); + validateTable(tableName, getValueType(), collector, ServiceNowConstants.PROPERTY_TABLE_NAMES); } } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 052ceb5c..8e0fe137 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -19,10 +19,13 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; -import io.cdap.plugin.servicenow.util.SchemaBuilder; +import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceQueryMode; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.oltu.oauth2.common.exception.OAuthProblemException; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,12 +39,23 @@ public class ServiceNowRecordReader extends ServiceNowBaseRecordReader { private static final Logger LOG = LoggerFactory.getLogger(ServiceNowRecordReader.class); private final ServiceNowSourceConfig pluginConf; + private ServiceNowTableAPIClientImpl restApi; ServiceNowRecordReader(ServiceNowSourceConfig pluginConf) { super(); this.pluginConf = pluginConf; } + @Override + public void initialize(InputSplit split, TaskAttemptContext context) { + this.split = (ServiceNowInputSplit) split; + this.pos = 0; + restApi = new ServiceNowTableAPIClientImpl(pluginConf.getConnection()); + tableName = ((ServiceNowInputSplit) split).getTableName(); + tableNameField = pluginConf.getTableNameField(); + fetchSchema(restApi); + } + @Override public boolean nextKeyValue() throws IOException { try { @@ -74,8 +88,7 @@ public StructuredRecord getCurrentValue() throws IOException { try { for (Schema.Field field : tableFields) { String fieldName = field.getName(); - Object fieldValue = pluginConf.getConnection().convertToValue(fieldName, field.getSchema(), row); - recordBuilder.set(fieldName, fieldValue); + ServiceNowRecordConverter.convertToValue(fieldName, field.getSchema(), row, recordBuilder); } } catch (Exception e) { LOG.error("Error decoding row from table " + tableName, e); @@ -85,41 +98,29 @@ public StructuredRecord getCurrentValue() throws IOException { } private void fetchData() throws IOException { - tableName = split.getTableName(); - tableNameField = pluginConf.getTableNameField(); - - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(pluginConf.getConnection()); - // Get the table data results = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), ServiceNowConstants.PAGE_SIZE); LOG.debug("Results size={}", results.size()); - if (!results.isEmpty()) { - fetchSchema(restApi); - } iterator = results.iterator(); } private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { - // Fetch the column definition - ServiceNowTableDataResponse response = restApi.fetchTableSchema(tableName, pluginConf.getValueType(), null, - null, false); - if (response == null) { - return; - } + try { + Schema tempSchema = restApi.fetchTableSchema(tableName); + tableFields = tempSchema.getFields(); + List schemaFields = new ArrayList<>(tableFields); - // Build schema - Schema tempSchema = SchemaBuilder.constructSchema(tableName, response.getColumns()); - tableFields = tempSchema.getFields(); - List schemaFields = new ArrayList<>(tableFields); + if (pluginConf.getQueryMode() == SourceQueryMode.REPORTING) { + schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); + } - if (pluginConf.getQueryMode() == SourceQueryMode.REPORTING) { - schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); + schema = Schema.recordOf(tableName, schemaFields); + } catch (OAuthProblemException | OAuthSystemException e) { + throw new RuntimeException(e); } - - schema = Schema.recordOf(tableName, schemaFields); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java index 9d5f7022..ec9babda 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java @@ -35,11 +35,9 @@ import io.cdap.cdap.etl.api.connector.Connector; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.SourceInputFormatProvider; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import io.cdap.plugin.servicenow.util.SourceQueryMode; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.slf4j.Logger; @@ -84,10 +82,7 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { List tableInfo = ServiceNowInputFormat.fetchTableInfo(conf.getQueryMode(collector), conf.getConnection(), conf.getTableName(), - conf.getApplicationName(), - conf.getValueType(), - conf.getStartDate(), - conf.getEndDate()); + conf.getApplicationName()); stageConfigurer.setOutputSchema(tableInfo.stream().findFirst().get().getSchema()); } else if (conf.getQueryMode() == SourceQueryMode.REPORTING) { stageConfigurer.setOutputSchema(null); @@ -111,7 +106,7 @@ public void prepareRun(BatchSourceContext context) { } context.setInput(Input.of(conf.getReferenceName(), - new SourceInputFormatProvider(ServiceNowInputFormat.class, hConf))); + new SourceInputFormatProvider(ServiceNowInputFormat.class, hConf))); } @Override @@ -128,8 +123,8 @@ private void recordLineage(BatchSourceContext context, ServiceNowTableInfo table List fields = Objects.requireNonNull(schema).getFields(); if (fields != null && !fields.isEmpty()) { lineageRecorder.recordRead("Read", - String.format("Read from '%s' ServiceNow table.", tableName), - fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); + String.format("Read from '%s' ServiceNow table.", tableName), + fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); } } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java index 553d9ff2..b7a80ee0 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java @@ -24,8 +24,8 @@ import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; import io.cdap.plugin.servicenow.util.Util; -import java.util.Optional; +import java.util.Optional; import javax.annotation.Nullable; /** @@ -61,26 +61,26 @@ public class ServiceNowSourceConfig extends ServiceNowBaseSourceConfig { /** * Constructor for ServiceNowSourceConfig object. * - * @param referenceName The reference name - * @param queryMode The query mode + * @param referenceName The reference name + * @param queryMode The query mode * @param applicationName The application name - * @param tableNameField The field name to hold the table name value - * @param tableName The table name - * @param clientId The Client Id for ServiceNow - * @param clientSecret The Client Secret for ServiceNow + * @param tableNameField The field name to hold the table name value + * @param tableName The table name + * @param clientId The Client Id for ServiceNow + * @param clientSecret The Client Secret for ServiceNow * @param restApiEndpoint The rest API endpoint for ServiceNow - * @param user The user id for ServiceNow - * @param password The password for ServiceNow - * @param valueType The value type - * @param startDate The start date - * @param endDate The end date + * @param user The user id for ServiceNow + * @param password The password for ServiceNow + * @param valueType The value type + * @param startDate The start date + * @param endDate The end date */ public ServiceNowSourceConfig(String referenceName, String queryMode, @Nullable String applicationName, @Nullable String tableNameField, @Nullable String tableName, String clientId, String clientSecret, String restApiEndpoint, String user, String password, String valueType, @Nullable String startDate, @Nullable String endDate) { super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate, - endDate); + endDate); this.referenceName = referenceName; this.queryMode = queryMode; this.applicationName = applicationName; @@ -100,7 +100,7 @@ public SourceQueryMode getQueryMode(FailureCollector collector) { } collector.addFailure("Unsupported query mode value: " + queryMode, - String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) + String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) .withConfigProperty(ServiceNowConstants.PROPERTY_QUERY_MODE); collector.getOrThrowException(); return null; @@ -130,7 +130,7 @@ public SourceApplication getApplicationName(FailureCollector collector) { } collector.addFailure("Unsupported application name value: " + applicationName, - String.format("Supported applications are: %s", SourceApplication.getSupportedApplications())) + String.format("Supported applications are: %s", SourceApplication.getSupportedApplications())) .withConfigProperty(ServiceNowConstants.PROPERTY_APPLICATION_NAME); collector.getOrThrowException(); return null; @@ -205,7 +205,7 @@ private void validateTableQueryMode(FailureCollector collector) { collector.addFailure("Table name must be specified.", null) .withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME); } else { - validateTable(tableName, getValueType(), collector); + validateTable(tableName, getValueType(), collector, ServiceNowConstants.PROPERTY_TABLE_NAME); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java index 6af6b5b1..2b3701da 100644 --- a/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java @@ -31,7 +31,7 @@ public class SchemaBuilder { * Constructs Schema object using input parameters. * * @param tableName The table name to be used in Schema object - * @param columns The list of ServiceNowColumn objects that will be added as Schema.Field + * @param columns The list of ServiceNowColumn objects that will be added as Schema.Field * @return The instance of Schema object */ public static Schema constructSchema(String tableName, List columns) { @@ -67,7 +67,7 @@ private Schema.Field transformToField(ServiceNowColumn column) { private Schema createSchema(ServiceNowColumn column) { switch (column.getTypeName().toLowerCase()) { case "decimal": - return Schema.decimalOf(ServiceNowConstants.DEFAULT_PRECISION, ServiceNowConstants.DEFAULT_SCALE); + return Schema.of(Schema.Type.DOUBLE); case "integer": return Schema.of(Schema.Type.INT); case "boolean": diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index 6bf1f105..bfa2ec73 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -118,8 +118,8 @@ public void testGenerateSpec() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -163,8 +163,7 @@ public void testGenerateSpec() throws Exception { SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Mockito.when(ServiceNowInputFormat.fetchTableInfo(mode, serviceNowSourceConfig.getConnection(), serviceNowSourceConfig.getTableName(), - null, valueType, - null, null)).thenReturn(list); + null)).thenReturn(list); ConnectorSpec connectorSpec = serviceNowConnector.generateSpec(new MockConnectorContext (new MockConnectorConfigurer()), diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index 8dc51e1f..3350767a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -93,8 +93,8 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_INTERNAL_SERVER_ERROR; @@ -140,8 +140,8 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -191,8 +191,8 @@ public void testWriteWithUnservicedRequests() throws Exception { ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index a729aacc..76c0c43a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -21,13 +21,13 @@ import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; -import io.cdap.plugin.servicenow.ServiceNowBaseConfig; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIRequest; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.sink.model.SchemaResponse; +import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -47,7 +47,6 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -293,11 +292,11 @@ public void testValidateSchema() throws Exception { " }\n" + " ]\n" + "}"; - SchemaResponse schemaResponse = new SchemaResponse(); - schemaResponse.setLabel("Class"); - schemaResponse.setName("sys_class_name"); - schemaResponse.setInternalType("sys_class_name"); - schemaResponse.setExampleValue(""); + ServiceNowSchemaField schemaField = new ServiceNowSchemaField("Class", "sys_class_name", + "sys_class_name", "sys_class_name"); + List schemaFields = new ArrayList<>(); + schemaFields.add(schemaField); + SchemaResponse schemaResponse = new SchemaResponse(schemaFields); RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -317,17 +316,16 @@ public void testValidateSchema() throws Exception { Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); Mockito.when(restApi.executeGet(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.fetchServiceNowTableSchema(Mockito.anyString(), Mockito.any())).thenReturn(schema); + Mockito.when(restApi.fetchTableSchema(Mockito.anyString(), Mockito.any())).thenReturn(schema); Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getResponseBody())) - .thenReturn(Collections.singletonList(schemaResponse)); + .thenReturn(schemaResponse); try { config.validateSchema(schema, collector); collector.getOrThrowException(); Assert.fail("Exception is not thrown if apiResponse is successful"); - } catch (ValidationException e) { - //Exception as the apiResponse is unsuccessful, it will not be able to fetch the schema of the table. - Assert.assertEquals("Errors were encountered during validation. Unable to fetch schema for table " + - "tableName", e.getMessage()); + } catch (RuntimeException e) { + //Exception as the apiResponse is unsuccessful, it will not be able to fetch the schema of the table. + Assert.assertEquals(1, collector.getValidationFailures().size()); } } @@ -379,7 +377,7 @@ public void testValidateSchemaWithOperation() throws Exception { Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); Mockito.when(restApi.executeGet(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.fetchServiceNowTableSchema("tableName", collector)). + Mockito.when(restApi.fetchTableSchema("tableName", collector)). thenReturn(schema); config.validateSchema(schema, collector); Assert.assertEquals(0, collector.getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index 75c33ee7..cd0400d4 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -88,7 +88,7 @@ public void testConfigurePipeline() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); + List> result = new ArrayList<>(); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -114,8 +114,8 @@ public void testPrepareRun() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index ed3e8fd9..657d128a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -22,7 +22,6 @@ import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; -import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -49,7 +48,7 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({ServiceNowTableAPIClientImpl.class, ServiceNowBaseSourceConfig.class, ServiceNowMultiSource.class, - HttpClientBuilder.class, RestAPIResponse.class}) + HttpClientBuilder.class, RestAPIResponse.class, ServiceNowInputFormat.class}) public class ServiceNowInputFormatTest { private static final String CLIENT_ID = "clientId"; @@ -72,8 +71,8 @@ public void testFetchTableInfo() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -162,10 +161,8 @@ public void testFetchTableInfo() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; - SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Assert.assertEquals(1, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType, "start", "end") - .size()); + application).size()); } @Test @@ -174,8 +171,8 @@ public void testFetchTableInfoReportingMode() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -264,10 +261,8 @@ public void testFetchTableInfoReportingMode() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; - SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Assert.assertEquals(4, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType, "start", - "end").size()); + application).size()); } @Test @@ -276,8 +271,8 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -289,6 +284,7 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); + PowerMockito.mockStatic(ServiceNowInputFormat.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); @@ -307,9 +303,7 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; - SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Assert.assertTrue(ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType, "start", - "end").isEmpty()); + application).isEmpty()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java index 568f55d8..1a9d4724 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java @@ -70,19 +70,16 @@ public void testFetchTablesInfo() { serviceNowTableInfos.add(serviceNowTableInfo); PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; - PowerMockito.when(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "table", - valueType, "start", "end")). + PowerMockito.when(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "table")). thenReturn(serviceNowTableInfos); Assert.assertEquals(1, ServiceNowMultiInputFormat - .fetchTablesInfo(connectorConfig, "table", - valueType, "start", "end") + .fetchTablesInfo(connectorConfig, "table") .size()); } @Test public void testFetchTablesInfoWithEmptyTableNames() { SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; - Assert.assertTrue(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "", - valueType, "start", "end").isEmpty()); + Assert.assertTrue(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "").isEmpty()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index c6e10e16..ff3602db 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -16,11 +16,16 @@ package io.cdap.plugin.servicenow.source; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginProperties; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; +import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowConstants; +import org.apache.oltu.oauth2.common.exception.OAuthProblemException; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -29,6 +34,7 @@ import org.mockito.Mockito; import java.io.IOException; +import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -85,45 +91,32 @@ public void testConstructor() throws IOException { @Test(expected = IllegalStateException.class) public void testConvertToValueInvalidFieldType() { - Schema fieldSchema = Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS); - serviceNowMultiSourceConfig.getConnection().convertToValue("Field Name", fieldSchema, new HashMap<>(1)); + Schema fieldSchema = Schema.recordOf("record", Schema.Field.of("TimeField", + Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS))); + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(fieldSchema); + Map map = new HashMap<>(); + map.put("TimeField", "value"); + ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, map, recordBuilder); } @Test - public void testConvertToValueInvalidRecord() { - Schema fieldSchema = Schema.of(Schema.Type.BOOLEAN); - Assert.assertEquals(Boolean.FALSE, serviceNowMultiSourceConfig.getConnection().convertToValue - ("Field Name", fieldSchema, - new HashMap<>(1))); + public void testConvertToDoubleValue() throws ParseException { + Assert.assertEquals(42.0, ServiceNowRecordConverter.convertToDoubleValue("42"), 0.0); } @Test - public void testConvertToStringValue() { - Assert.assertEquals("Field Value", serviceNowMultiSourceConfig.getConnection(). - convertToStringValue("Field Value")); + public void testConvertToIntegerValue() throws ParseException { + Assert.assertEquals(42, ServiceNowRecordConverter.convertToIntegerValue("42").intValue()); } @Test - public void testConvertToDoubleValue() { - Assert.assertEquals(42.0, serviceNowMultiSourceConfig.getConnection(). - convertToDoubleValue("42").doubleValue(), 0.0); - Assert.assertEquals(42.0, serviceNowMultiSourceConfig.getConnection(). - convertToDoubleValue(42).doubleValue(), 0.0); - Assert.assertNull(serviceNowMultiSourceConfig.getConnection().convertToDoubleValue("")); + public void testConvertToBooleanValue() { + Assert.assertTrue(ServiceNowRecordConverter.convertToBooleanValue("true")); } - @Test - public void testConvertToIntegerValue() { - Assert.assertEquals(42, serviceNowMultiSourceConfig.getConnection().convertToIntegerValue("42").intValue()); - Assert.assertEquals(42, serviceNowMultiSourceConfig.getConnection().convertToIntegerValue(42).intValue()); - Assert.assertNull(serviceNowMultiSourceConfig.getConnection().convertToIntegerValue("")); - } - - @Test - public void testConvertToBooleanValue() { - Assert.assertFalse(serviceNowMultiSourceConfig.getConnection().convertToBooleanValue("Field Value")); - Assert.assertFalse(serviceNowMultiSourceConfig.getConnection().convertToBooleanValue(42)); - Assert.assertNull(serviceNowMultiSourceConfig.getConnection().convertToBooleanValue("")); + @Test(expected = UnexpectedFormatException.class) + public void testConvertToBooleanValueForInvalidFieldValue() { + Assert.assertTrue(ServiceNowRecordConverter.convertToBooleanValue("1")); } @Test @@ -131,8 +124,8 @@ public void testFetchData() throws IOException { String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); + List> results = new ArrayList<>(); + Map map = new HashMap<>(); map.put("calendar_integration", "1"); map.put("country", "India"); map.put("sys_updated_on", "2019-04-05 21:54:45"); @@ -145,16 +138,23 @@ public void testFetchData() throws IOException { ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); - serviceNowMultiRecordReader.initialize(split, null); + ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); + try { + Mockito.when(restApi.fetchTableSchema(tableName)) + .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); + serviceNowMultiRecordReader.initialize(split, null); + } catch (RuntimeException | OAuthProblemException | OAuthSystemException e) { + Assert.assertTrue(e instanceof RuntimeException); + } Mockito.doNothing().when(serviceNowMultiRecordReader).fetchData(); Collections.singletonList(new Object()); - serviceNowMultiRecordReader.iterator = Collections.singletonList(Collections.singletonMap("key", new Object())). + serviceNowMultiRecordReader.iterator = Collections.singletonList(Collections.singletonMap("key", new String())). iterator(); Assert.assertTrue(serviceNowMultiRecordReader.nextKeyValue()); } @Test - public void testFetchDataOnInvalidTable() throws IOException { + public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemException, OAuthSystemException { serviceNowMultiSourceConfig = ServiceNowSourceConfigHelper.newConfigBuilder() .setReferenceName("referenceName") .setRestApiEndpoint(REST_API_ENDPOINT) @@ -172,8 +172,8 @@ public void testFetchDataOnInvalidTable() throws IOException { String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); + List> results = new ArrayList<>(); + Map map = new HashMap<>(); map.put("calendar_integration", "1"); map.put("country", "India"); map.put("sys_updated_on", "2019-04-05 21:54:45"); @@ -190,7 +190,13 @@ public void testFetchDataOnInvalidTable() throws IOException { ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); - serviceNowMultiRecordReader.initialize(split, null); + try { + Mockito.when(restApi.fetchTableSchema(tableName)) + .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); + serviceNowMultiRecordReader.initialize(split, null); + } catch (RuntimeException | OAuthProblemException | OAuthSystemException e) { + Assert.assertTrue(e instanceof RuntimeException); + } Assert.assertFalse(serviceNowMultiRecordReader.nextKeyValue()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index db73e034..d07bb9ed 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -55,7 +55,6 @@ public void testConstructor() { "client_secret", "https://example.com", "user", "password", "tablename", "Actual", "2021-12-30", "2021-12-31", "sys_user"); - Assert.assertEquals("sys_user", serviceNowMultiSourceConfig.getTableNames()); Assert.assertEquals("Actual", serviceNowMultiSourceConfig.getValueType().getValueType()); Assert.assertEquals("2021-12-30", serviceNowMultiSourceConfig.getStartDate()); @@ -113,8 +112,8 @@ public void testValidate() throws Exception { .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); String responseBody = "{\n" + @@ -244,8 +243,8 @@ public void testValidateReferenceName() throws Exception { .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); String responseBody = "{\n" + @@ -321,6 +320,7 @@ public void testValidateReferenceName() throws Exception { Assert.assertEquals("referenceName", mockFailureCollector.getValidationFailures().get(0).getCauses() .get(0).getAttribute("stageConfig")); } + } @Test @@ -346,8 +346,8 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); String responseBody = "{\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index 70fe6aaf..bb096ed8 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.mock.common.MockArguments; @@ -24,6 +25,7 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -43,12 +45,14 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; @RunWith(PowerMockRunner.class) @PrepareForTest({ServiceNowTableAPIClientImpl.class, ServiceNowBaseSourceConfig.class, ServiceNowMultiSource.class, - HttpClientBuilder.class, RestAPIResponse.class}) + HttpClientBuilder.class, RestAPIResponse.class, ServiceNowInputFormat.class, ServiceNowMultiInputFormat.class}) public class ServiceNowMultiSourceTest { private static final String CLIENT_ID = "clientId"; @@ -86,8 +90,8 @@ public void testConfigurePipeline() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -170,7 +174,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); + List> result = new ArrayList<>(); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -190,15 +194,30 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { @Test public void testPrepareRun() throws Exception { MockFailureCollector mockFailureCollector = new MockFailureCollector(); + Set tableInfo = new HashSet<>(); + Schema schema = Schema.recordOf("schema", + Schema.Field.of("IntField", Schema.of(Schema.Type.INT)), + Schema.Field.of("LongField", Schema.of(Schema.Type.LONG)), + Schema.Field.of("DoubleField", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), + Schema.Field.of("BooleanField", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))), + Schema.Field.of("DateField", Schema.of(Schema.LogicalType.DATE)), + Schema.Field.of("TimestampField", + Schema.nullableOf(Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))), + Schema.Field.of("TimeField", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)), + Schema.Field.of("StringField", Schema.of(Schema.Type.STRING)), + Schema.Field.of("ArrayField", Schema.arrayOf(Schema.of(Schema.Type.STRING)))); + ServiceNowTableInfo serviceNowTableInfo = new ServiceNowTableInfo("table", schema, 1); + tableInfo.add(serviceNowTableInfo); MockArguments mockArguments = new MockArguments(); + mockArguments.set("multisink.sys_user", "multisink." + serviceNowMultiSourceConfig.getTableNames()); BatchSourceContext context = Mockito.mock(BatchSourceContext.class); Mockito.when(context.getFailureCollector()).thenReturn(mockFailureCollector); Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -265,6 +284,8 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; + PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); + Mockito.when(ServiceNowMultiInputFormat.setInput(Mockito.any(), Mockito.any())).thenReturn((tableInfo)); RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); @@ -280,6 +301,7 @@ public void testPrepareRun() throws Exception { HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); PowerMockito.mockStatic(HttpClientBuilder.class); PowerMockito.mockStatic(RestAPIResponse.class); + PowerMockito.mockStatic(ServiceNowInputFormat.class); PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index 1caf04f4..c5383e5d 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -16,12 +16,15 @@ package io.cdap.plugin.servicenow.source; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.macro.Macros; import io.cdap.cdap.api.plugin.PluginProperties; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; +import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceQueryMode; @@ -37,6 +40,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; +import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -94,8 +98,8 @@ public void testConstructor2() throws IOException { "https://ven05127." + "service-now.com/", "User", "password", - "Actual", "2021-12-30" - , "2021-12-31"); + "Actual", "2021-12-30", + "2021-12-31"); serviceNowRecordReader.close(); Assert.assertEquals(0, serviceNowRecordReader.pos); @@ -116,37 +120,35 @@ public void testConstructor2() throws IOException { @Test public void testConvertToValue() { - Schema fieldSchema = Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS); - thrown.expect(IllegalStateException.class); - serviceNowSourceConfig.getConnection().convertToValue("Field Name", fieldSchema, new HashMap<>(1)); - } - @Test - public void testConvertToStringValue() { - Assert.assertEquals("Field Value", serviceNowSourceConfig.getConnection().convertToStringValue("Field Value")); + Schema fieldSchema = Schema.recordOf("record", Schema.Field.of("TimeField", + Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS))); + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(fieldSchema); + Map map = new HashMap<>(); + map.put("TimeField", "value"); + thrown.expect(IllegalStateException.class); + ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, map, recordBuilder); } @Test - public void testConvertToDoubleValue() { - Assert.assertEquals(42.0, serviceNowSourceConfig.getConnection().convertToDoubleValue("42").doubleValue(), - 0.0); - Assert.assertEquals(42.0, serviceNowSourceConfig.getConnection().convertToDoubleValue(42).doubleValue(), + public void testConvertToDoubleValue() throws ParseException { + Assert.assertEquals(42.0, ServiceNowRecordConverter.convertToDoubleValue("42"), 0.0); - Assert.assertNull(serviceNowSourceConfig.getConnection().convertToDoubleValue("")); } @Test - public void testConvertToIntegerValue() { - Assert.assertEquals(42, serviceNowSourceConfig.getConnection().convertToIntegerValue("42").intValue()); - Assert.assertEquals(42, serviceNowSourceConfig.getConnection().convertToIntegerValue(42).intValue()); - Assert.assertNull(serviceNowSourceConfig.getConnection().convertToIntegerValue("")); + public void testConvertToIntegerValue() throws ParseException { + Assert.assertEquals(42, ServiceNowRecordConverter.convertToIntegerValue("42").intValue()); } @Test public void testConvertToBooleanValue() { - Assert.assertFalse(serviceNowSourceConfig.getConnection().convertToBooleanValue("Field Value")); - Assert.assertFalse(serviceNowSourceConfig.getConnection().convertToBooleanValue(42)); - Assert.assertNull(serviceNowSourceConfig.getConnection().convertToBooleanValue("")); + Assert.assertTrue(ServiceNowRecordConverter.convertToBooleanValue("true")); + } + + @Test(expected = UnexpectedFormatException.class) + public void testConvertToBooleanValueForInvalidFieldValue() { + Assert.assertTrue(ServiceNowRecordConverter.convertToBooleanValue("1")); } @Test @@ -155,8 +157,8 @@ public void testFetchData() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); + List> results = new ArrayList<>(); + Map map = new HashMap<>(); map.put("calendar_integration", "1"); map.put("country", "India"); map.put("sys_updated_on", "2019-04-05 21:54:45"); @@ -167,8 +169,8 @@ public void testFetchData() throws Exception { map.put("sys_created_on", "2019-04-05 21:09:12"); results.add(map); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); - ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); - ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); + ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); + ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); List columns = new ArrayList<>(); columns.add(column1); columns.add(column2); @@ -181,9 +183,8 @@ public void testFetchData() throws Exception { serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. getEndDate(), split.getOffset(), ServiceNowConstants.PAGE_SIZE)).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType(), null, null, - false)).thenReturn(response); - + Mockito.when(restApi.fetchTableSchema(tableName)) + .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split, null); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); } @@ -210,8 +211,8 @@ public void testFetchDataReportingMode() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); + List> results = new ArrayList<>(); + Map map = new HashMap<>(); map.put("calendar_integration", "1"); map.put("country", "India"); map.put("sys_updated_on", "2019-04-05 21:54:45"); @@ -222,8 +223,8 @@ public void testFetchDataReportingMode() throws Exception { map.put("sys_created_on", "2019-04-05 21:09:12"); results.add(map); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); - ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); - ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); + ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); + ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); List columns = new ArrayList<>(); columns.add(column1); columns.add(column2); @@ -236,9 +237,8 @@ public void testFetchDataReportingMode() throws Exception { serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), ServiceNowConstants.PAGE_SIZE)).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType(), null, null, - false)).thenReturn(response); - + Mockito.when(restApi.fetchTableSchema(tableName)) + .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split, null); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); } @@ -263,7 +263,7 @@ public void testFetchDataOnInvalidTable() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); - List> results = new ArrayList<>(); + List> results = new ArrayList<>(); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(), @@ -272,6 +272,8 @@ public void testFetchDataOnInvalidTable() throws Exception { ServiceNowConstants.PAGE_SIZE)).thenReturn(results); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); + Mockito.when(restApi.fetchTableSchema(tableName)) + .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split, null); Assert.assertFalse(serviceNowRecordReader.nextKeyValue()); } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index 12d58039..521c1b80 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -563,7 +563,8 @@ private ServiceNowSourceConfig withServiceNowValidationMock(ServiceNowSourceConf FailureCollector collector) { ServiceNowSourceConfig spy = Mockito.spy(config); Mockito.doNothing().when(spy).validateServiceNowConnection(collector); - Mockito.doNothing().when(spy).validateTable(config.getTableName(), config.getValueType(), collector); + Mockito.doNothing().when(spy) + .validateTable(config.getTableName(), config.getValueType(), collector, ServiceNowConstants.PROPERTY_TABLE_NAME); return spy; } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 419a6541..6511895c 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -25,6 +25,8 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; + import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -49,7 +51,7 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({ServiceNowTableAPIClientImpl.class, ServiceNowBaseSourceConfig.class, ServiceNowSource.class, - HttpClientBuilder.class, RestAPIResponse.class}) + HttpClientBuilder.class, RestAPIResponse.class, ServiceNowInputFormat.class}) public class ServiceNowSourceTest { private static final String CLIENT_ID = "clientId"; @@ -80,15 +82,28 @@ public void initialize() { @Test public void testConfigurePipeline() throws Exception { + List tableInfo = new ArrayList<>(); + Schema schema = Schema.recordOf("schema", + Schema.Field.of("IntField", Schema.of(Schema.Type.INT)), + Schema.Field.of("LongField", Schema.of(Schema.Type.LONG)), + Schema.Field.of("DoubleField", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), + Schema.Field.of("BooleanField", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))), + Schema.Field.of("DateField", Schema.of(Schema.LogicalType.DATE)), + Schema.Field.of("TimestampField", Schema.nullableOf(Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))), + Schema.Field.of("TimeField", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)), + Schema.Field.of("StringField", Schema.of(Schema.Type.STRING)), + Schema.Field.of("ArrayField", Schema.arrayOf(Schema.of(Schema.Type.STRING)))); + ServiceNowTableInfo serviceNowTableInfo = new ServiceNowTableInfo("table", schema, 1); + tableInfo.add(serviceNowTableInfo); Map plugins = new HashMap<>(); MockFailureCollector mockFailureCollector = new MockFailureCollector(); MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - Mockito.when(restApi.getAccessToken()).thenReturn("token"); + Mockito.when(restApi.getAccessToken()).thenReturn("token1"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); + Map map = new HashMap<>(); + List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -155,6 +170,9 @@ public void testConfigurePipeline() throws Exception { " }\n" + " ]\n" + "}"; + PowerMockito.mockStatic(ServiceNowInputFormat.class); + Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), + Mockito.any())).thenReturn(tableInfo); RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); @@ -189,7 +207,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); + List> result = new ArrayList<>(); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -210,14 +228,15 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { public void testPrepareRun() throws Exception { MockFailureCollector mockFailureCollector = new MockFailureCollector(); MockArguments mockArguments = new MockArguments(); + mockArguments.set("multisink.sys_user", "multisink." + serviceNowSourceConfig.getTableName()); BatchSourceContext context = Mockito.mock(BatchSourceContext.class); Mockito.when(context.getFailureCollector()).thenReturn(mockFailureCollector); Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); + List> result = new ArrayList<>(); + Map map = new HashMap<>(); map.put("key", "value"); result.add(map); int httpStatus = HttpStatus.SC_OK; @@ -285,7 +304,7 @@ public void testPrepareRun() throws Exception { " ]\n" + "}"; RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + PowerMockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -299,6 +318,7 @@ public void testPrepareRun() throws Exception { HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); PowerMockito.mockStatic(HttpClientBuilder.class); PowerMockito.mockStatic(RestAPIResponse.class); + PowerMockito.mockStatic(ServiceNowInputFormat.class); PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); diff --git a/widgets/ServiceNow-batchsource.json b/widgets/ServiceNow-batchsource.json index 1fc52b75..1534c2a1 100644 --- a/widgets/ServiceNow-batchsource.json +++ b/widgets/ServiceNow-batchsource.json @@ -128,6 +128,14 @@ { "label": "Table Mode", "properties": [ + { + "widget-type": "textbox", + "label": "Table Name", + "name": "tableName", + "widget-attributes": { + "placeholder": "ServiceNow table name to fetch data from" + } + }, { "label": "browse", "widget-type": "connection-browser", @@ -136,14 +144,6 @@ "connectionType": " SERVICENOW", "label": "Browse" } - }, - { - "widget-type": "textbox", - "label": "Table Name", - "name": "tableName", - "widget-attributes": { - "placeholder": "ServiceNow table name from which data to be fetched" - } } ] }, @@ -215,6 +215,10 @@ { "type": "property", "name": "tableName" + }, + { + "type": "property", + "label": "browse" } ] },