[Java BQ] Storage API streaming load test (#28264)

* more accurate regex for table spec; add toTableSpec * storage api streaming load test * use io performance test utilities * only fail 30% of the time * use periodicimpulse stopAfter * use custom bq service with crashing behavior
apache · Oct 3, 2023 · 2e05211 · 2e05211
1 parent c7d7896
commit 2e05211
Show file tree

Hide file tree

Showing 8 changed files with 723 additions and 9 deletions.
diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
@@ -65,7 +65,7 @@ jobs:
       matrix:
         job_name: ["beam_PostCommit_Java_IO_Performance_Tests"]
         job_phrase: ["Run Java PostCommit IO Performance Tests"]
-        test_case: ["GCSPerformanceTest", "BigTablePerformanceTest"]
+        test_case: ["GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"]
     steps:
     - uses: actions/checkout@v4
     - name: Setup repository

diff --git a/it/build.gradle b/it/build.gradle
@@ -30,4 +30,8 @@ tasks.register('GCSPerformanceTest') {
 
 tasks.register('BigTablePerformanceTest') {
     dependsOn(":it:google-cloud-platform:BigTablePerformanceTest")
+}
+
+tasks.register('BigQueryStorageApiStreamingPerformanceTest') {
+    dependsOn(":it:google-cloud-platform:BigQueryStorageApiStreamingPerformanceTest")
 }
diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import groovy.json.JsonOutput
 import org.apache.beam.gradle.IoPerformanceTestUtilities
 
 plugins { id 'org.apache.beam.module' }
@@ -73,9 +74,11 @@ dependencies {
     testImplementation project(path: ":sdks:java:io:google-cloud-platform")
     testImplementation project(path: ":sdks:java:io:synthetic")
     testImplementation library.java.mockito_inline
+    testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration")
     testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest")
     testRuntimeOnly library.java.slf4j_simple
 }
 
 tasks.register("GCSPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'FileBasedIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp'])
-tasks.register("BigTablePerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp'])
+tasks.register("BigTablePerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp'])
+tasks.register("BigQueryStorageApiStreamingPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryStreamingLT', ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp'])