Skip to content

Commit

Permalink
Use apache-beam-testing.samples (#28064)
Browse files Browse the repository at this point in the history
* Use apache-beam-testing.samples

* run ./gradlew :examples:java:spotlessApply
  • Loading branch information
liferoad authored Aug 20, 2023
1 parent 26b7772 commit 44aea4b
Show file tree
Hide file tree
Showing 26 changed files with 53 additions and 53 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,15 @@
*
* See examples/java/README.md for instructions about how to configure different runners.
*
* <p>The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations}
* and can be overridden with {@code --input}.
* <p>The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations} and
* can be overridden with {@code --input}.
*/
public class BigQueryTornadoes {
private static final Logger LOG = LoggerFactory.getLogger(BigQueryTornadoes.class);

// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";

/**
* Examines each row in the input table. If a tornado was recorded in that sample, the month in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,13 @@
*
* See examples/java/README.md for instructions about how to configure different runners.
*
* <p>The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations}
* and can be overridden with {@code --input}.
* <p>The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations} and
* can be overridden with {@code --input}.
*/
public class FilterExamples {
// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";
static final Logger LOG = Logger.getLogger(FilterExamples.class.getName());
static final int MONTH_TO_FILTER = 7;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
public class JoinExamples {

// A 1000-row sample of the GDELT data here: gdelt-bq:full.events.
private static final String GDELT_EVENTS_TABLE = "clouddataflow-readonly:samples.gdelt_sample";
private static final String GDELT_EVENTS_TABLE = "apache-beam-testing.samples.gdelt_sample";
// A table that maps country codes to country names.
private static final String COUNTRY_CODES = "gdelt-bq:full.crosswalk_geocountrycodetohuman";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,13 @@
*
* See examples/java/README.md for instructions about how to configure different runners.
*
* <p>The BigQuery input table defaults to {@code clouddataflow-readonly:samples.weather_stations }
* and can be overridden with {@code --input}.
* <p>The BigQuery input table defaults to {@code apache-beam-testing.samples.weather_stations } and
* can be overridden with {@code --input}.
*/
public class MaxPerKeyExamples {
// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";

/**
* Examines each row (weather reading) in the input table. Output the month of the reading, and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,14 @@
*
* <p>Concepts: Reading/writing BigQuery; counting a PCollection; user-defined PTransforms
*
* <p>The BigQuery input is taken from {@code clouddataflow-readonly:samples.weather_stations}
* <p>The BigQuery input is taken from {@code apache-beam-testing.samples.weather_stations}
*/
public class MinimalBigQueryTornadoes {
private static final Logger LOG = LoggerFactory.getLogger(MinimalBigQueryTornadoes.class);

// Use a 1000 row subset of the public weather station table publicdata:samples.gsod.
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";

/**
* Examines each row in the input table. If a tornado was recorded in that sample, the month in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public static void modelBigQueryIO(
Pipeline p, String writeProject, String writeDataset, String writeTable) {
{
// [START BigQueryTableSpec]
String tableSpec = "clouddataflow-readonly:samples.weather_stations";
String tableSpec = "apache-beam-testing.samples.weather_stations";
// [END BigQueryTableSpec]
}

Expand Down Expand Up @@ -212,7 +212,7 @@ public static void modelBigQueryIO(
}

{
String tableSpec = "clouddataflow-readonly:samples.weather_stations";
String tableSpec = "apache-beam-testing.samples.weather_stations";
// [START BigQueryReadTable]
PCollection<Double> maxTemperatures =
p.apply(BigQueryIO.readTableRows().from(tableSpec))
Expand All @@ -224,7 +224,7 @@ public static void modelBigQueryIO(
}

{
String tableSpec = "clouddataflow-readonly:samples.weather_stations";
String tableSpec = "apache-beam-testing.samples.weather_stations";
// [START BigQueryReadFunction]
PCollection<Double> maxTemperatures =
p.apply(
Expand All @@ -242,7 +242,7 @@ public static void modelBigQueryIO(
BigQueryIO.read(
(SchemaAndRecord elem) -> (Double) elem.getRecord().get("max_temperature"))
.fromQuery(
"SELECT max_temperature FROM [clouddataflow-readonly:samples.weather_stations]")
"SELECT max_temperature FROM [apache-beam-testing.samples.weather_stations]")
.withCoder(DoubleCoder.of()));
// [END BigQueryReadQuery]
}
Expand Down Expand Up @@ -280,7 +280,7 @@ public static void modelBigQueryIO(
// [END BigQuerySchemaJson]

{
String tableSpec = "clouddataflow-readonly:samples.weather_stations";
String tableSpec = "apache-beam-testing.samples.weather_stations";
if (!writeProject.isEmpty() && !writeDataset.isEmpty() && !writeTable.isEmpty()) {
tableSpec = writeProject + ":" + writeDataset + "." + writeTable;
}
Expand Down Expand Up @@ -403,7 +403,7 @@ public WeatherData(long year, long month, long day, double maxTemp) {
})
.fromQuery(
"SELECT year, month, day, max_temperature "
+ "FROM [clouddataflow-readonly:samples.weather_stations] "
+ "FROM [apache-beam-testing.samples.weather_stations] "
+ "WHERE year BETWEEN 2007 AND 2009")
.withCoder(AvroCoder.of(WeatherData.class)));

Expand Down Expand Up @@ -461,7 +461,7 @@ public TableSchema getSchema(Long destination) {
.withWriteDisposition(WriteDisposition.WRITE_TRUNCATE));
// [END BigQueryWriteDynamicDestinations]

String tableSpec = "clouddataflow-readonly:samples.weather_stations";
String tableSpec = "apache-beam-testing.samples.weather_stations";
if (!writeProject.isEmpty() && !writeDataset.isEmpty() && !writeTable.isEmpty()) {
tableSpec = writeProject + ":" + writeDataset + "." + writeTable + "_partitioning";
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists
* See examples/java/README.md for instructions about how to configure different runners.
*
*
* The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations`
* The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations`
* and can be overridden with `--input`.
*/
object BigQueryTornadoes {
// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations"
private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations"

/**
* Examines each row in the input table. If a tornado was recorded in that sample, the month in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ import java.util.logging.Logger
* See examples/kotlin/README.md for instructions about how to configure different runners.
*
*
* The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations`
* The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations`
* and can be overridden with `--input`.
*/
object FilterExamples {
// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations"
private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations"
internal val LOG = Logger.getLogger(FilterExamples::class.java.name)
internal const val MONTH_TO_FILTER = 7

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ import org.apache.beam.sdk.values.TupleTag
object JoinExamples {

// A 1000-row sample of the GDELT data here: gdelt-bq:full.events.
private const val GDELT_EVENTS_TABLE = "clouddataflow-readonly:samples.gdelt_sample"
private const val GDELT_EVENTS_TABLE = "apache-beam-testing.samples.gdelt_sample"
// A table that maps country codes to country names.
private const val COUNTRY_CODES = "gdelt-bq:full.crosswalk_geocountrycodetohuman"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ import java.util.ArrayList
* See examples/java/README.md for instructions about how to configure different runners.
*
*
* The BigQuery input table defaults to `clouddataflow-readonly:samples.weather_stations `
* The BigQuery input table defaults to `apache-beam-testing.samples.weather_stations `
* and can be overridden with `--input`.
*/
object MaxPerKeyExamples {
// Default to using a 1000 row subset of the public weather station table publicdata:samples.gsod.
private const val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations"
private const val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations"

/**
* Examines each row (weather reading) in the input table. Output the month of the reading, and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ object Snippets {
pipeline: Pipeline, writeProject: String = "", writeDataset: String = "", writeTable: String = "") {
run {
// [START BigQueryTableSpec]
val tableSpec = "clouddataflow-readonly:samples.weather_stations"
val tableSpec = "apache-beam-testing.samples.weather_stations"
// [END BigQueryTableSpec]
}

Expand All @@ -104,7 +104,7 @@ object Snippets {
}

run {
val tableSpec = "clouddataflow-readonly:samples.weather_stations"
val tableSpec = "apache-beam-testing.samples.weather_stations"
// [START BigQueryReadTable]
val maxTemperatures = pipeline.apply(BigQueryIO.readTableRows().from(tableSpec))
// Each row is of type TableRow
Expand All @@ -118,7 +118,7 @@ object Snippets {
}

run {
val tableSpec = "clouddataflow-readonly:samples.weather_stations"
val tableSpec = "apache-beam-testing.samples.weather_stations"
// [START BigQueryReadFunction]
val maxTemperatures = pipeline.apply(
BigQueryIO.read { it.record["max_temperature"] as Double? }
Expand All @@ -132,7 +132,7 @@ object Snippets {
val maxTemperatures = pipeline.apply(
BigQueryIO.read { it.record["max_temperature"] as Double? }
.fromQuery(
"SELECT max_temperature FROM [clouddataflow-readonly:samples.weather_stations]")
"SELECT max_temperature FROM [apache-beam-testing.samples.weather_stations]")
.withCoder(DoubleCoder.of()))
// [END BigQueryReadQuery]
}
Expand Down Expand Up @@ -167,7 +167,7 @@ object Snippets {
// [END BigQuerySchemaJson]

run {
var tableSpec = "clouddataflow-readonly:samples.weather_stations"
var tableSpec = "apache-beam-testing.samples.weather_stations"
if (writeProject.isNotEmpty() && writeDataset.isNotEmpty() && writeTable.isNotEmpty()) {
tableSpec = "$writeProject:$writeDataset.$writeTable"
}
Expand Down Expand Up @@ -259,7 +259,7 @@ object Snippets {
}
.fromQuery("""
SELECT year, month, day, max_temperature
FROM [clouddataflow-readonly:samples.weather_stations]
FROM [apache-beam-testing.samples.weather_stations]
WHERE year BETWEEN 2007 AND 2009
""".trimIndent())
.withCoder(AvroCoder.of(WeatherData::class.java)))
Expand Down Expand Up @@ -297,7 +297,7 @@ object Snippets {
.withWriteDisposition(WriteDisposition.WRITE_TRUNCATE))
// [END BigQueryWriteDynamicDestinations]

var tableSpec = "clouddataflow-readonly:samples.weather_stations"
var tableSpec = "apache-beam-testing.samples.weather_stations"
if (writeProject.isNotEmpty() && writeDataset.isNotEmpty() && writeTable.isNotEmpty()) {
tableSpec = "$writeProject:$writeDataset.${writeTable}_partitioning"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ The `logOutput` struct is defined as a custom `DoFn` that implements the Process
{{if (eq .Sdk "java")}}
```
PCollection<TableRow> pCollection = pipeline
.apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ))
.apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ))
```

The `BigQueryIO.readTableRows()` method is called to create a `BigQueryIO.Read` transform that will read data from a `BigQuery` table.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public static void main(String[] args) {
*/

PCollection<TableRow> pCollection = pipeline
.apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ));
.apply("ReadFromBigQuery", BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations").withMethod(TypedRead.Method.DIRECT_READ));

final PTransform<PCollection<TableRow>, PCollection<Iterable<TableRow>>> sample = Sample.fixedSizeGlobally(5);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ PCollection<WeatherData> weatherData =
})
.fromQuery(
"SELECT year, month, day, max_temperature "
+ "FROM [clouddataflow-readonly:samples.weather_stations] "
+ "FROM [apache-beam-testing.samples.weather_stations] "
+ "WHERE year BETWEEN 2007 AND 2009")
.withCoder(AvroCoder.of(WeatherData.class)));
Expand Down
4 changes: 2 additions & 2 deletions playground/backend/internal/fs_tool/ExampleData.scala
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ object ExampleData {
"gs://apache-beam-samples/traffic_sensor/Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv"
val GAMING = "gs://apache-beam-samples/game/gaming_data*.csv"

val WEATHER_SAMPLES_TABLE = "clouddataflow-readonly:samples.weather_stations"
val WEATHER_SAMPLES_TABLE = "apache-beam-testing.samples.weather_stations"
val SHAKESPEARE_TABLE = "bigquery-public-data:samples.shakespeare"
val EVENT_TABLE = "clouddataflow-readonly:samples.gdelt_sample"
val EVENT_TABLE = "apache-beam-testing.samples.gdelt_sample"
val COUNTRY_TABLE = "gdelt-bq:full.crosswalk_geocountrycodetohuman"
}
2 changes: 1 addition & 1 deletion sdks/go/examples/cookbook/filter/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
)

var (
input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "Weather data BQ table.")
input = flag.String("input", "apache-beam-testing.samples.weather_stations", "Weather data BQ table.")
output = flag.String("output", "", "Output BQ table.")
month = flag.Int("month_filter", 7, "Numerical month to analyze")
)
Expand Down
2 changes: 1 addition & 1 deletion sdks/go/examples/cookbook/join/join.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
// See: https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java

const (
gdeltEventsTable = "clouddataflow-readonly:samples.gdelt_sample"
gdeltEventsTable = "apache-beam-testing.samples.gdelt_sample"
countryCodesTable = "gdelt-bq:full.crosswalk_geocountrycodetohuman"
)

Expand Down
2 changes: 1 addition & 1 deletion sdks/go/examples/cookbook/max/max.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
)

var (
input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "Weather data BQ table.")
input = flag.String("input", "apache-beam-testing.samples.weather_stations", "Weather data BQ table.")
output = flag.String("output", "", "Output BQ table.")
)

Expand Down
4 changes: 2 additions & 2 deletions sdks/go/examples/cookbook/tornadoes/tornadoes.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
//
// --output=YOUR_PROJECT_ID:DATASET_ID.TABLE_ID
//
// The BigQuery input table defaults to clouddataflow-readonly:samples.weather_stations
// The BigQuery input table defaults to apache-beam-testing.samples.weather_stations
// and can be overridden with {@code --input}.
package main

Expand All @@ -48,7 +48,7 @@ import (
)

var (
input = flag.String("input", "clouddataflow-readonly:samples.weather_stations", "BigQuery table with weather data to read from, specified as <project_id>:<dataset_id>.<table_id>")
input = flag.String("input", "apache-beam-testing.samples.weather_stations", "BigQuery table with weather data to read from, specified as <project_id>:<dataset_id>.<table_id>")
output = flag.String("output", "", "BigQuery table to write to, specified as <project_id>:<dataset_id>.<table_id>. The dataset must already exist")
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
*
* <pre>{@code
* PCollection<TableRow> inputData = pipeline.apply(
* BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
* BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations"));
* }</pre>
*
* and {@code Write} transforms that persist PCollections to external storage:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@
*
* <pre>{@code
* PCollection<TableRow> weatherData = pipeline.apply(
* BigQueryIO.readTableRows().from("clouddataflow-readonly:samples.weather_stations"));
* BigQueryIO.readTableRows().from("apache-beam-testing.samples.weather_stations"));
* }</pre>
*
* <b>Example: Reading rows of a table and parsing them into a custom type.</b>
Expand All @@ -281,7 +281,7 @@
* return new WeatherRecord(...);
* }
* })
* .from("clouddataflow-readonly:samples.weather_stations"))
* .from("apache-beam-testing.samples.weather_stations"))
* .withCoder(SerializableCoder.of(WeatherRecord.class));
* }</pre>
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class BigQueryClusteringIT {
private static final Long EXPECTED_BYTES = 16000L;
private static final BigInteger EXPECTED_ROWS = new BigInteger("1000");
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";
private static final String DATASET_NAME = "BigQueryClusteringIT";
private static final Clustering CLUSTERING =
new Clustering().setFields(Arrays.asList("station_number"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
@RunWith(JUnit4.class)
public class BigQueryTimePartitioningClusteringIT {
private static final String WEATHER_SAMPLES_TABLE =
"clouddataflow-readonly:samples.weather_stations";
"apache-beam-testing.samples.weather_stations";
private static final String DATASET_NAME = "BigQueryTimePartitioningIT";
private static final TimePartitioning TIME_PARTITIONING =
new TimePartitioning().setField("date").setType("DAY");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def run(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument(
'--input',
default='clouddataflow-readonly:samples.weather_stations',
default='apache-beam-testing.samples.weather_stations',
help=(
'Input BigQuery table to process specified as: '
'PROJECT:DATASET.TABLE or DATASET.TABLE.'))
Expand Down
Loading

0 comments on commit 44aea4b

Please sign in to comment.