forked from datahub-project/datahub
-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'datahub-project:master' into master
- Loading branch information
Showing
26 changed files
with
1,420 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| name: PR Comment | ||
|
|
||
| on: | ||
| pull_request: | ||
| types: [opened] | ||
|
|
||
| permissions: | ||
| pull-requests: write | ||
|
|
||
| jobs: | ||
| post-pr-opened-comment: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
|
|
||
| - name: Get and Format Username (PR only) | ||
| if: github.event_name == 'pull_request' | ||
| run: | | ||
| formatted_username=$(echo "${{ github.event.pull_request.user.login }}" | tr '[:upper:]' '[:lower:]' | sed 's/ /-/g') | ||
| echo "FORMATTED_USERNAME=$formatted_username" >> $GITHUB_ENV | ||
| - name: Create Comment (PR only) | ||
| if: github.event_name == 'pull_request' | ||
| uses: actions/github-script@v6 | ||
| with: | ||
| script: | | ||
| if (context.payload.pull_request) { | ||
| const prUser = process.env.FORMATTED_USERNAME; | ||
| const url = `https://contributors.datahubproject.io/${prUser}`; | ||
| const body = `Hello @${prUser} :smile: \n\n Thank you so much for opening a pull request!\n\n\nYou can check out your contributor card and see all your past stats [here](${url})!`; | ||
| // Create a comment on the PR | ||
| await github.rest.issues.createComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: context.payload.pull_request.number, | ||
| body: body | ||
| }); | ||
| } else { | ||
| console.log('Not a pull request event.'); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
116 changes: 116 additions & 0 deletions
116
...ty-registry/src/main/java/com/linkedin/metadata/aspect/validation/FieldPathValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| package com.linkedin.metadata.aspect.validation; | ||
|
|
||
| import static com.linkedin.metadata.Constants.*; | ||
|
|
||
| import com.linkedin.metadata.aspect.RetrieverContext; | ||
| import com.linkedin.metadata.aspect.batch.BatchItem; | ||
| import com.linkedin.metadata.aspect.batch.ChangeMCP; | ||
| import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; | ||
| import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; | ||
| import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; | ||
| import com.linkedin.metadata.aspect.plugins.validation.ValidationExceptionCollection; | ||
| import com.linkedin.schema.EditableSchemaFieldInfo; | ||
| import com.linkedin.schema.EditableSchemaMetadata; | ||
| import com.linkedin.schema.SchemaField; | ||
| import com.linkedin.schema.SchemaMetadata; | ||
| import java.util.Collection; | ||
| import java.util.Optional; | ||
| import java.util.stream.Stream; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import lombok.experimental.Accessors; | ||
|
|
||
| /** | ||
| * 1. Validates the Schema Field Path specification, specifically that all field IDs must be unique | ||
| * across all fields within a schema. 2. Validates that the field path id is not empty. | ||
| * | ||
| * @see <a href="https://datahubproject.io/docs/advanced/field-path-spec-v2/#requirements">Field | ||
| * Path V2 docs</a> | ||
| */ | ||
| @Setter | ||
| @Getter | ||
| @Accessors(chain = true) | ||
| public class FieldPathValidator extends AspectPayloadValidator { | ||
| @Nonnull private AspectPluginConfig config; | ||
|
|
||
| /** Prevent any MCP for SchemaMetadata where field ids are duplicated. */ | ||
| @Override | ||
| protected Stream<AspectValidationException> validateProposedAspects( | ||
| @Nonnull Collection<? extends BatchItem> mcpItems, | ||
| @Nonnull RetrieverContext retrieverContext) { | ||
|
|
||
| ValidationExceptionCollection exceptions = ValidationExceptionCollection.newCollection(); | ||
|
|
||
| mcpItems.forEach( | ||
| i -> { | ||
| if (i.getAspectName().equals(SCHEMA_METADATA_ASPECT_NAME)) { | ||
| processSchemaMetadataAspect(i, exceptions); | ||
| } else { | ||
| processEditableSchemaMetadataAspect(i, exceptions); | ||
| } | ||
| }); | ||
|
|
||
| return exceptions.streamAllExceptions(); | ||
| } | ||
|
|
||
| @Override | ||
| protected Stream<AspectValidationException> validatePreCommitAspects( | ||
| @Nonnull Collection<ChangeMCP> changeMCPs, @Nonnull RetrieverContext retrieverContext) { | ||
| return Stream.of(); | ||
| } | ||
|
|
||
| private static void processEditableSchemaMetadataAspect( | ||
| BatchItem i, ValidationExceptionCollection exceptions) { | ||
| final EditableSchemaMetadata schemaMetadata = i.getAspect(EditableSchemaMetadata.class); | ||
| final long uniquePaths = | ||
| validateAndCount( | ||
| i, | ||
| schemaMetadata.getEditableSchemaFieldInfo().stream() | ||
| .map(EditableSchemaFieldInfo::getFieldPath), | ||
| exceptions); | ||
|
|
||
| if (uniquePaths != schemaMetadata.getEditableSchemaFieldInfo().size()) { | ||
| exceptions.addException( | ||
| i, | ||
| String.format( | ||
| "Cannot perform %s action on proposal. EditableSchemaMetadata aspect has duplicated field paths", | ||
| i.getChangeType())); | ||
| } | ||
| } | ||
|
|
||
| private static void processSchemaMetadataAspect( | ||
| BatchItem i, ValidationExceptionCollection exceptions) { | ||
| final SchemaMetadata schemaMetadata = i.getAspect(SchemaMetadata.class); | ||
| final long uniquePaths = | ||
| validateAndCount( | ||
| i, schemaMetadata.getFields().stream().map(SchemaField::getFieldPath), exceptions); | ||
|
|
||
| if (uniquePaths != schemaMetadata.getFields().size()) { | ||
| exceptions.addException( | ||
| i, | ||
| String.format( | ||
| "Cannot perform %s action on proposal. SchemaMetadata aspect has duplicated field paths", | ||
| i.getChangeType())); | ||
| } | ||
| } | ||
|
|
||
| private static long validateAndCount( | ||
| BatchItem i, Stream<String> fieldPaths, ValidationExceptionCollection exceptions) { | ||
| return fieldPaths | ||
| .distinct() | ||
| // inspect the stream of fieldPath validation errors since we're already iterating | ||
| .peek( | ||
| fieldPath -> | ||
| validateFieldPath(fieldPath) | ||
| .ifPresent(message -> exceptions.addException(i, message))) | ||
| .count(); | ||
| } | ||
|
|
||
| private static Optional<String> validateFieldPath(String fieldPath) { | ||
| if (fieldPath == null || fieldPath.isEmpty()) { | ||
| return Optional.of("SchemaMetadata aspect has empty field path."); | ||
| } | ||
| return Optional.empty(); | ||
| } | ||
| } |
Oops, something went wrong.