-
Notifications
You must be signed in to change notification settings - Fork 4.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add GoogleAdsIO for reading from Google Ads (#27681)
* Add GoogleAdsIO for reading from Google Ads * Remove trailing whitespace * Add link to OAuth2 documentation in the Google Ads API * Document retry configuration * Capitalize long literal suffix in documentation * Make GoogleAdsV14.Read consume customer IDs from a PCollection * Change non-empty to non-null in test assertions * Use Long.toString in transform construction example * Use EnsuresNonNull, RequiresNonNull instead of checkStateNotNull * Remove default rate limit policy * Refactor DefaultRateLimitPolicy to SimpleRateLimitPolicy * Synchronize on ReadAllFn to safely modify static variable sleeper * Add additional links to clarify API limits, quotas, and policies * Fix Guava imports * Update CHANGES.md * Move change to upcoming release
- Loading branch information
1 parent
4122e96
commit aee0eb7
Showing
18 changed files
with
1,681 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * License); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an AS IS BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| plugins { id 'org.apache.beam.module' } | ||
| applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.googleads') | ||
|
|
||
| description = "Apache Beam :: SDKs :: Java :: IO :: Google Ads" | ||
| ext.summary = "IO to read from Google Ads" | ||
|
|
||
| dependencies { | ||
| implementation project(path: ":sdks:java:core", configuration: "shadow") | ||
| implementation project(path: ":sdks:java:extensions:google-cloud-platform-core") | ||
| implementation library.java.jackson_annotations | ||
| implementation library.java.gax | ||
| implementation library.java.google_ads | ||
| implementation library.java.google_auth_library_credentials | ||
| implementation library.java.google_auth_library_oauth2_http | ||
| implementation library.java.protobuf_java | ||
| implementation library.java.protobuf_java_util | ||
| implementation library.java.google_ads | ||
| implementation library.java.google_ads_stubs_v14 | ||
| implementation library.java.joda_time | ||
| implementation library.java.vendored_guava_32_1_2_jre | ||
| testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") | ||
| testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") | ||
| testImplementation library.java.mockito_core | ||
| testImplementation library.java.junit | ||
| testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") | ||
| testRuntimeOnly library.java.slf4j_jdk14 | ||
| } |
67 changes: 67 additions & 0 deletions
67
...gle-ads/src/main/java/org/apache/beam/sdk/io/googleads/DefaultGoogleAdsClientFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.beam.sdk.io.googleads; | ||
|
|
||
| import com.google.ads.googleads.lib.GoogleAdsClient; | ||
| import com.google.auth.Credentials; | ||
| import org.checkerframework.checker.nullness.qual.Nullable; | ||
|
|
||
| /** The default way to construct a {@link GoogleAdsClient}. */ | ||
| public class DefaultGoogleAdsClientFactory implements GoogleAdsClientFactory { | ||
| private static final DefaultGoogleAdsClientFactory INSTANCE = new DefaultGoogleAdsClientFactory(); | ||
|
|
||
| public static DefaultGoogleAdsClientFactory getInstance() { | ||
| return INSTANCE; | ||
| } | ||
|
|
||
| @Override | ||
| public GoogleAdsClient newGoogleAdsClient( | ||
| GoogleAdsOptions options, | ||
| @Nullable String developerToken, | ||
| @Nullable Long linkedCustomerId, | ||
| @Nullable Long loginCustomerId) { | ||
|
|
||
| GoogleAdsClient.Builder builder = GoogleAdsClient.newBuilder(); | ||
|
|
||
| Credentials credentials = options.getGoogleAdsCredential(); | ||
| if (credentials != null) { | ||
| builder.setCredentials(credentials); | ||
| } | ||
|
|
||
| if (options.getGoogleAdsEndpoint() != null) { | ||
| builder.setEndpoint(options.getGoogleAdsEndpoint()); | ||
| } | ||
|
|
||
| String developerTokenFromOptions = options.getGoogleAdsDeveloperToken(); | ||
| if (developerToken != null) { | ||
| builder.setDeveloperToken(developerToken); | ||
| } else if (developerTokenFromOptions != null) { | ||
| builder.setDeveloperToken(developerTokenFromOptions); | ||
| } | ||
|
|
||
| if (linkedCustomerId != null) { | ||
| builder.setLinkedCustomerId(linkedCustomerId); | ||
| } | ||
|
|
||
| if (loginCustomerId != null) { | ||
| builder.setLoginCustomerId(loginCustomerId); | ||
| } | ||
|
|
||
| return builder.build(); | ||
| } | ||
| } |
31 changes: 31 additions & 0 deletions
31
.../io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsClientFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.beam.sdk.io.googleads; | ||
|
|
||
| import com.google.ads.googleads.lib.GoogleAdsClient; | ||
| import java.io.Serializable; | ||
| import org.checkerframework.checker.nullness.qual.Nullable; | ||
|
|
||
| /** Defines how to construct a {@link GoogleAdsClient}. */ | ||
| public interface GoogleAdsClientFactory extends Serializable { | ||
| GoogleAdsClient newGoogleAdsClient( | ||
| GoogleAdsOptions options, | ||
| @Nullable String developerToken, | ||
| @Nullable Long linkedCustomerId, | ||
| @Nullable Long loginCustomerId); | ||
| } |
33 changes: 33 additions & 0 deletions
33
sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsIO.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.beam.sdk.io.googleads; | ||
|
|
||
| /** | ||
| * {@link GoogleAdsIO} provides an API for reading from the <a | ||
| * href="https://developers.google.com/google-ads/api/docs/start">Google Ads API</a> over different | ||
| * versions of the Google Ads client libraries. | ||
| * | ||
| * @see GoogleAdsV14 | ||
| */ | ||
| public class GoogleAdsIO { | ||
| private GoogleAdsIO() {} | ||
|
|
||
| public static GoogleAdsV14 v14() { | ||
| return GoogleAdsV14.INSTANCE; | ||
| } | ||
| } |
132 changes: 132 additions & 0 deletions
132
sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/GoogleAdsOptions.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.beam.sdk.io.googleads; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonIgnore; | ||
| import com.google.auth.Credentials; | ||
| import java.io.IOException; | ||
| import java.security.GeneralSecurityException; | ||
| import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory; | ||
| import org.apache.beam.sdk.options.Default; | ||
| import org.apache.beam.sdk.options.DefaultValueFactory; | ||
| import org.apache.beam.sdk.options.Description; | ||
| import org.apache.beam.sdk.options.PipelineOptions; | ||
| import org.apache.beam.sdk.util.InstanceBuilder; | ||
| import org.checkerframework.checker.nullness.qual.Nullable; | ||
|
|
||
| /** Options used to configure Google Ads API specific options. */ | ||
| public interface GoogleAdsOptions extends PipelineOptions { | ||
| /** Host endpoint to use for connections to the Google Ads API. */ | ||
| @Description("Host endpoint to use for connections to the Google Ads API.") | ||
| @Default.String("googleads.googleapis.com:443") | ||
| String getGoogleAdsEndpoint(); | ||
|
|
||
| void setGoogleAdsEndpoint(String endpoint); | ||
|
|
||
| /** | ||
| * OAuth 2.0 Client ID identifying the application. | ||
| * | ||
| * @see https://developers.google.com/google-ads/api/docs/oauth/overview | ||
| * @see https://developers.google.com/identity/protocols/oauth2 | ||
| */ | ||
| @Description("OAuth 2.0 Client ID identifying the application.") | ||
| String getGoogleAdsClientId(); | ||
|
|
||
| void setGoogleAdsClientId(String clientId); | ||
|
|
||
| /** | ||
| * OAuth 2.0 Client Secret for the specified Client ID. | ||
| * | ||
| * @see https://developers.google.com/google-ads/api/docs/oauth/overview | ||
| * @see https://developers.google.com/identity/protocols/oauth2 | ||
| */ | ||
| @Description("OAuth 2.0 Client Secret for the specified Client ID.") | ||
| String getGoogleAdsClientSecret(); | ||
|
|
||
| void setGoogleAdsClientSecret(String clientSecret); | ||
|
|
||
| /** | ||
| * OAuth 2.0 Refresh Token for the user connecting to the Google Ads API. | ||
| * | ||
| * @see https://developers.google.com/google-ads/api/docs/oauth/overview | ||
| * @see https://developers.google.com/identity/protocols/oauth2 | ||
| */ | ||
| @Description("OAuth 2.0 Refresh Token for the user connecting to the Google Ads API.") | ||
| String getGoogleAdsRefreshToken(); | ||
|
|
||
| void setGoogleAdsRefreshToken(String refreshToken); | ||
|
|
||
| /** Google Ads developer token for the user connecting to the Google Ads API. */ | ||
| @Description("Google Ads developer token for the user connecting to the Google Ads API.") | ||
| @Nullable | ||
| String getGoogleAdsDeveloperToken(); | ||
|
|
||
| void setGoogleAdsDeveloperToken(String developerToken); | ||
|
|
||
| /** | ||
| * The class of the credential factory to create credentials if none have been explicitly set. | ||
| * | ||
| * @see #getGoogleAdsCredential() | ||
| */ | ||
| @Description( | ||
| "The class of the credential factory to create credentials if none have been explicitly set.") | ||
| @Default.Class(GoogleAdsUserCredentialFactory.class) | ||
| Class<? extends CredentialFactory> getGoogleAdsCredentialFactoryClass(); | ||
|
|
||
| void setGoogleAdsCredentialFactoryClass( | ||
| Class<? extends CredentialFactory> credentialFactoryClass); | ||
|
|
||
| /** | ||
| * The credential instance that should be used to authenticate against the Google Ads API. | ||
| * Defaults to a credential instance constructed by the credential factory. | ||
| * | ||
| * @see #getGoogleAdsCredential() | ||
| * @see https://github.com/googleapis/google-auth-library-java | ||
| */ | ||
| @JsonIgnore | ||
| @Description( | ||
| "The credential instance that should be used to authenticate against the Google Ads API. " | ||
| + "Defaults to a credential instance constructed by the credential factory.") | ||
| @Default.InstanceFactory(GoogleAdsCredentialsFactory.class) | ||
| @Nullable | ||
| Credentials getGoogleAdsCredential(); | ||
|
|
||
| void setGoogleAdsCredential(Credentials credential); | ||
|
|
||
| /** | ||
| * Attempts to load the Google Ads credentials. See {@link CredentialFactory#getCredential()} for | ||
| * more details. | ||
| */ | ||
| class GoogleAdsCredentialsFactory implements DefaultValueFactory<@Nullable Credentials> { | ||
| @Override | ||
| public @Nullable Credentials create(PipelineOptions options) { | ||
| GoogleAdsOptions googleAdsOptions = options.as(GoogleAdsOptions.class); | ||
| try { | ||
| CredentialFactory factory = | ||
| InstanceBuilder.ofType(CredentialFactory.class) | ||
| .fromClass(googleAdsOptions.getGoogleAdsCredentialFactoryClass()) | ||
| .fromFactoryMethod("fromOptions") | ||
| .withArg(PipelineOptions.class, options) | ||
| .build(); | ||
| return factory.getCredential(); | ||
| } catch (IOException | GeneralSecurityException e) { | ||
| throw new RuntimeException("Unable to obtain credential", e); | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.