ServiceNow Batch Source GA Release changes.

data-integrations · Oct 9, 2023 · 74af16a · 74af16a
1 parent ed1e584
commit 74af16a
Show file tree

Hide file tree

Showing 51 changed files with 810 additions and 668 deletions.
diff --git a/docs/ServiceNow-batchsource.md b/docs/ServiceNow-batchsource.md
@@ -67,20 +67,21 @@ ignored if the Mode is set to `Reporting`.
 Data Types Mapping
 ----------
 
-    | ServiceNow Data Type           | CDAP Schema Data Type | Comment                                            |
-    | ------------------------------ | --------------------- | -------------------------------------------------- |
-    | decimal                        | double                |                                                    |
-    | integer                        | int                   |                                                    |
-    | boolean                        | boolean               |                                                    |
-    | reference                      | string                |                                                    |
-    | currency                       | string                |                                                    |
-    | glide_date                     | string                |                                                    |
-    | glide_date_time                | string                |                                                    |
-    | sys_class_name                 | string                |                                                    |
-    | domain_id                      | string                |                                                    |
-    | domain_path                    | string                |                                                    |
-    | guid                           | string                |                                                    |
-    | translated_html                | string                |                                                    |
-    | journal                        | string                |                                                    |
-    | string                         | string                |                                                    |
+    | ServiceNow Data Type           | CDAP Schema Data Type                                         
+    | ------------------------------ | ---------------------
+    | decimal                        | double                                                                    
+    | integer                        | int                                                                       
+    | boolean                        | boolean                                                                   
+    | reference                      | string                                                                    
+    | currency                       | string                                                                    
+    | glide_date                     | date                                                                      
+    | glide_date_time                | datetime                                                                  
+    | glide_time                     | time                                                               
+    | sys_class_name                 | string                                                                    
+    | domain_id                      | string                                                                    
+    | domain_path                    | string                                                                    
+    | guid                           | string                                                                    
+    | translated_html                | string                                                                    
+    | journal                        | string                                                                    
+    | string                         | string                                                                    
 
diff --git a/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature b/src/e2e-test/features/servicenowmultisource/DesignTimeValidation.feature
@@ -26,11 +26,6 @@ Feature: ServiceNow Multi Source - Design time validation scenarios
     And Click on the Validate button
     Then Verify mandatory property error for below listed properties:
       | referenceName   |
-      | clientId        |
-      | clientSecret    |
-      | restApiEndpoint |
-      | user            |
-      | password        |
 
   @TS-SN-MULTI-DSGN-ERROR-02
   Scenario: Verify validation message for invalid table name
@@ -41,7 +36,7 @@ Feature: ServiceNow Multi Source - Design time validation scenarios
       | INVALID_TABLE |
     And fill Credentials section for pipeline user
     And Click on the Validate button
-    Then Verify that the Plugin is displaying an error message: "invalid.property.tablename" on the header
+    Then Verify that the Plugin Property: "tableNames" is displaying an in-line error message: "invalid.property.tablename"
 
   @TS-SN-MULTI-DSGN-ERROR-03
   Scenario: Verify validation message for Start date and End date in invalid format

diff --git a/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature b/src/e2e-test/features/servicenowmultisource/DesignTimeWithMacros.feature
@@ -30,4 +30,4 @@ Feature: ServiceNow Multi Source - Design time scenarios (macro)
     And Click on the Macro button of Property: "restApiEndpoint" and set the value to: "restApiEndpoint"
     And Click on the Macro button of Property: "user" and set the value to: "username"
     And Click on the Macro button of Property: "password" and set the value to: "password"
-    Then Validate "ServiceNow Multi Source" plugin properties
+    Then Validate "ServiceNow Multi Source" plugin properties
diff --git a/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowmultisource/RunTimeWithMacros.feature
@@ -136,4 +136,4 @@ Feature: ServiceNow Multi Source - Run time scenarios (macro)
     And Verify the pipeline status is "Failed"
     Then Open Pipeline logs and verify Log entries having below listed Level and Message:
       | Level | Message                                |
-      | ERROR | invalid.filters.logsmessage            |
+      | ERROR | invalid.filters.logsmessage            |
diff --git a/src/e2e-test/features/servicenowsink/DesignTime.feature b/src/e2e-test/features/servicenowsink/DesignTime.feature
@@ -15,7 +15,7 @@
 @ServiceNow
 @SNSink
 @Smoke
-@Regression
+@RegressionSkip
 Feature: ServiceNow Sink - Design time scenarios
 
   @TS-SN-DSGN-SINK-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE

diff --git a/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature b/src/e2e-test/features/servicenowsink/DesignTimeValidation.feature
@@ -15,7 +15,7 @@
 @ServiceNow
 @SNSink
 @Smoke
-@Regression
+@RegressionSkip
 Feature: ServiceNow Sink - Design time validation scenarios
 
   @TS-SN-DSGN-SINK-ERROR-01
@@ -60,4 +60,4 @@ Feature: ServiceNow Sink - Design time validation scenarios
     And Enter input plugin property: "name" with value: "connection.name"
     And fill Credentials section with invalid credentials
     Then Click on the Test Connection button
-    Then Verify the invalid connection error message: "invalid.testconnection.logmessage" on the footer
+    Then Verify the invalid connection error message: "invalid.testconnection.logmessage" on the footer
diff --git a/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature b/src/e2e-test/features/servicenowsink/DesignTimeWithMacro.feature
@@ -15,7 +15,7 @@
 @ServiceNow
 @SNSink
 @Smoke
-@@Regression
+@@RegressionSkip
 Feature: ServiceNow Sink - Design time validation scenarios (macro)
 
   @TS-SN-DSGN-SINK-MACRO-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE
@@ -69,4 +69,4 @@ Feature: ServiceNow Sink - Design time validation scenarios (macro)
     And fill Credentials section for pipeline user
     And Enter input plugin property: "tableName" with value: "receiving_slip_line"
     And Click on the Macro button of Property: "operation" and set the value to: "operation"
-    Then Validate "ServiceNow" plugin properties
+    Then Validate "ServiceNow" plugin properties
diff --git a/src/e2e-test/features/servicenowsink/RunTime.feature b/src/e2e-test/features/servicenowsink/RunTime.feature
@@ -15,7 +15,7 @@
 @ServiceNow
 @SNSink
 @Smoke
-@Regression
+@RegressionSkip
 Feature: ServiceNow Sink - Run time scenarios
 
   @TS-SN-RNTM-SINK-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE

diff --git a/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowsink/RunTimeWithMacros.feature
@@ -15,7 +15,7 @@
 @ServiceNow
 @SNSink
 @Smoke
-@Regression
+@RegressionSkip
 Feature: ServiceNow Sink - Run time scenarios (macro)
 
   @TS-SN-RNTM-SINK-MACRO-01 @BQ_SOURCE_TEST_RECEIVING_SLIP_LINE @BQ_SINK_CLEANUP
@@ -197,4 +197,3 @@ Feature: ServiceNow Sink - Run time scenarios (macro)
     Then Open Pipeline logs and verify Log entries having below listed Level and Message:
       | Level | Message                                   |
       | ERROR | invalid.credentials.logsmessage           |
-
diff --git a/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature b/src/e2e-test/features/servicenowsource/DesignTimeValidation.feature
@@ -26,11 +26,6 @@ Feature: ServiceNow Source - Design time validation scenarios
     And Click on the Validate button
     Then Verify mandatory property error for below listed properties:
       | referenceName   |
-      | clientId        |
-      | clientSecret    |
-      | restApiEndpoint |
-      | user            |
-      | password        |
 
   @TS-SN-DSGN-ERROR-02
   Scenario: Verify invalid credentials validation messages

diff --git a/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature b/src/e2e-test/features/servicenowsource/DesignTimeWithMacros.feature
@@ -46,4 +46,4 @@ Feature: ServiceNow Source - Design time scenarios (macro)
     And Click on the Macro button of Property: "restApiEndpoint" and set the value to: "restApiEndpoint"
     And Click on the Macro button of Property: "user" and set the value to: "username"
     And Click on the Macro button of Property: "password" and set the value to: "password"
-    Then Validate "ServiceNow" plugin properties
+    Then Validate "ServiceNow" plugin properties
diff --git a/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature b/src/e2e-test/features/servicenowsource/RunTimeWithMacros.feature
@@ -184,4 +184,4 @@ Feature: ServiceNow Source - Run time scenarios (macro)
     And Verify the pipeline status is "Failed"
     Then Open Pipeline logs and verify Log entries having below listed Level and Message:
       | Level | Message                                |
-      | ERROR | invalid.filters.logsmessage            |
+      | ERROR | invalid.filters.logsmessage            |
diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryCommonSteps.java
@@ -44,7 +44,7 @@ public void configureBqSinkPlugin() {
   public void configureBqMultiTableSinkPlugin() {
     String referenceName = "Test" + RandomStringUtils.randomAlphanumeric(10);
     CdfBigQueryPropertiesActions.enterBigQueryReferenceName(referenceName);
-    CdfBigQueryPropertiesActions.enterBigQueryDataset(PluginPropertyUtils.pluginProp("bq.target.dataset"));
+    CdfBigQueryPropertiesActions.enterBigQueryDataset(PluginPropertyUtils.pluginProp("bq.target.dataset2"));
   }
 
   @Then("Verify count of no of records transferred to the target BigQuery Table")

diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties
@@ -4,7 +4,7 @@ validationSuccessMessage=No errors found.
 #Invalid value
 invalid.property.tablename=Bad Request. Table:
 invalid.property.startdate=Invalid format for Start date. Correct Format: yyyy-MM-dd
-invalid.property.enddate=Invalid format for End date. Correct Format:yyyy-MM-dd
+invalid.property.enddate=Invalid format for End date. Correct Format: yyyy-MM-dd
 invalid.property.credentials=Unable to connect to ServiceNow Instance. Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password are correct.
 
 #Logs error message

diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties
@@ -10,6 +10,7 @@ pipeline.user.password = SERVICE_NOW_PASSWORD
 
 #Tables
 receiving_slip_line=proc_rec_slip_item
+asset_covered=clm_m2m_contract_asset
 agent_assist_recommendation = agent_assist_recommendation
 vendor_catalog_item = pc_vendor_cat_item
 service_offering = service_offering
@@ -38,26 +39,27 @@ connection.name = dummy
 projectId=cdf-athena
 datasetprojectId=cdf-athena
 bq.target.dataset=SN_test_automation
+bq.target.dataset2=SN_Test_atm
 
 ##ServiceNowSink
 INSERT=insert
 UPDATE=update
 pagesize=200
 
 ##ExpectedSchemaJSONs
-schema.table.receiving.slip.line=[{"key":"cost","value":"string"},{"key":"quantity","value":"string"},\
-  {"key":"purchase_line","value":"string"},{"key":"sys_mod_count","value":"string"},\
-  {"key":"received","value":"string"},{"key":"requested_for","value":"string"},\
-  {"key":"sys_updated_on","value":"string"},{"key":"sys_tags","value":"string"},{"key":"number","value":"string"},\
+schema.table.receiving.slip.line=[{"key":"quantity","value":"int"},{"key":"cost","value":"string"},\
+  {"key":"purchase_line","value":"string"},{"key":"sys_mod_count","value":"int"},\
+  {"key":"received","value":"datetime"},{"key":"requested_for","value":"string"},\
+  {"key":"sys_updated_on","value":"datetime"},{"key":"sys_tags","value":"string"},{"key":"number","value":"string"},\
   {"key":"sys_id","value":"string"},{"key":"received_by","value":"string"},{"key":"sys_updated_by","value":"string"},\
-  {"key":"receiving_slip","value":"string"},{"key":"sys_created_on","value":"string"},\
+  {"key":"receiving_slip","value":"string"},{"key":"sys_created_on","value":"datetime"},\
   {"key":"sys_domain","value":"string"},{"key":"sys_created_by","value":"string"}]
 
-schema.table.asset.covered=[{"key":"added","value":"string"},{"key":"contract","value":"string"},\
-  {"key":"sys_mod_count","value":"string"},{"key":"sys_updated_on","value":"string"},\
+schema.table.asset.covered=[{"key":"added","value":"date"},{"key":"contract","value":"string"},\
+  {"key":"sys_mod_count","value":"int"},{"key":"sys_updated_on","value":"datetime"},\
   {"key":"sys_domain_path","value":"string"},{"key":"sys_tags","value":"string"},\
-  {"key":"sys_id","value":"string"},{"key":"sys_updated_by","value":"string"},{"key":"removed","value":"string"},\
-  {"key":"sys_created_on","value":"string"},{"key":"sys_domain","value":"string"},{"key":"asset","value":"string"},\
+  {"key":"sys_id","value":"string"},{"key":"sys_updated_by","value":"string"},{"key":"removed","value":"date"},\
+  {"key":"sys_created_on","value":"datetime"},{"key":"sys_domain","value":"string"},{"key":"asset","value":"string"},\
   {"key":"sys_created_by","value":"string"}]
 
 schema.table.condition=[{"key":"condition_check","value":"string"},{"key":"sys_mod_count","value":"string"},\

diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java
@@ -23,15 +23,13 @@
 import io.cdap.cdap.api.plugin.PluginConfig;
 import io.cdap.cdap.etl.api.FailureCollector;
 import io.cdap.plugin.common.ConfigUtil;
-import io.cdap.plugin.common.IdUtils;
 import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl;
 import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder;
 import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig;
 import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
 import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.SourceValueType;
-import io.cdap.plugin.servicenow.util.Util;
 import org.apache.http.HttpStatus;
 import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
 import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
@@ -52,18 +50,17 @@ public class ServiceNowBaseConfig extends PluginConfig {
   @Macro
   @Nullable
   @Description("The existing connection to use.")
-  private ServiceNowConnectorConfig connection;
-
-  @Nullable
-  public ServiceNowConnectorConfig getConnection() {
-    return connection;
-  }
+  private final ServiceNowConnectorConfig connection;
 
   public ServiceNowBaseConfig(String clientId, String clientSecret, String restApiEndpoint,
                               String user, String password) {
     this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password);
   }
 
+  @Nullable
+  public ServiceNowConnectorConfig getConnection() {
+    return connection;
+  }
 
   /**
    * Validates {@link ServiceNowSourceConfig} instance.
@@ -81,16 +78,16 @@ public void validateCredentials(FailureCollector collector) {
       validateServiceNowConnection(collector);
     }
   }
-  
+
   @VisibleForTesting
   public void validateServiceNowConnection(FailureCollector collector) {
     try {
       ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection);
       restApi.getAccessToken();
     } catch (Exception e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",
-          "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password " +
-            "are correct.")
+                           "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password " +
+                             "are correct.")
         .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_ID)
         .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_SECRET)
         .withConfigProperty(ServiceNowConstants.PROPERTY_API_ENDPOINT)
@@ -121,7 +118,8 @@ && shouldConnect()
       && !containsMacro(ServiceNowConstants.PROPERTY_VALUE_TYPE);
   }
 
-  public void validateTable(String tableName, SourceValueType valueType, FailureCollector collector) {
+  public void validateTable(String tableName, SourceValueType valueType, FailureCollector collector,
+                            String tableField) {
     // Call API to fetch first record from the table
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
       connection.getRestApiEndpoint(), tableName, false)
@@ -142,11 +140,11 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
       if (!apiResponse.isSuccess()) {
         if (apiResponse.getHttpStatus() == HttpStatus.SC_BAD_REQUEST) {
           collector.addFailure("Bad Request. Table: " + tableName + " is invalid.", "")
-            .withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME);
+            .withConfigProperty(tableField);
         }
       } else if (serviceNowTableAPIClient.parseResponseToResultListOfMap(apiResponse.getResponseBody()).isEmpty()) {
-        collector.addFailure("Table: " + tableName + " is empty.", "")
-          .withConfigProperty(ServiceNowConstants.PROPERTY_TABLE_NAME);
+        // Removed config property as in case of MultiSource, only first table error was populating.
+        collector.addFailure("Table: " + tableName + " is empty.", "");
       }
     } catch (OAuthSystemException | OAuthProblemException e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",