-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Run all tests against S3 and S3 Express (#90)
Enhance Github CI actions and variables to run all tests against both S3 and S3 Express. Address test flakiness by creating per test assets using BucketPrefixFixture, instead of static assets. Remove hypothesis setup from test_mountpoint_client_pickles() as it was flaky. We will add this back when we align on the limits of the parameters.
- Loading branch information
Showing
7 changed files
with
251 additions
and
296 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
104 changes: 87 additions & 17 deletions
104
s3torchconnectorclient/python/tst/integration/conftest.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,39 +1,109 @@ | ||
| # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| # // SPDX-License-Identifier: BSD | ||
|
|
||
| import io | ||
| import os | ||
| import random | ||
|
|
||
| import boto3 | ||
| import numpy as np | ||
| from PIL import Image | ||
| import pytest | ||
|
|
||
| def getenv(var: str) -> str: | ||
|
|
||
| def getenv(var: str, optional: bool = False) -> str: | ||
| v = os.getenv(var) | ||
| if v is None: | ||
| raise Exception(f"required environment variable {var} is not set") | ||
| if v is None and not optional: | ||
| raise Exception(f"Required environment variable {var} is not set") | ||
| return v | ||
|
|
||
|
|
||
| # TODO: Update with test fixtures | ||
| class TestConfig(object): | ||
| """Config object fixture for CI, wrapping region, S3 bucket, S3 express bucket and prefix configuration.""" | ||
| class BucketPrefixFixture(object): | ||
| """An S3 bucket/prefix and its contents for use in a single unit test. The prefix will be unique | ||
| to this instance, so other concurrent tests won't affect its state.""" | ||
|
|
||
| region: str | ||
| bucket: str | ||
| express_bucket: str | ||
| express_region: str | ||
| prefix: str | ||
| storage_class: str = None | ||
|
|
||
| def __init__( | ||
| self, region: str, bucket: str, express_bucket: str, express_region: str | ||
| self, region: str, bucket: str, prefix: str, storage_class: str = None | ||
| ): | ||
| self.region = region | ||
| self.bucket = bucket | ||
| self.express_bucket = express_bucket | ||
| self.express_region = express_region | ||
| self.prefix = prefix | ||
| self.region = region | ||
| self.storage_class = storage_class | ||
| self.contents = {} | ||
| session = boto3.Session(region_name=region) | ||
| self.s3 = session.client("s3") | ||
|
|
||
| @property | ||
| def s3_uri(self): | ||
| return f"s3://{self.bucket}/{self.prefix}" | ||
|
|
||
| def add(self, key: str, contents: bytes, **kwargs): | ||
| """Upload an S3 object to this prefix of the bucket.""" | ||
| full_key = f"{self.prefix}{key}" | ||
| self.s3.put_object(Bucket=self.bucket, Key=full_key, Body=contents, **kwargs) | ||
| self.contents[full_key] = contents | ||
|
|
||
| def __getitem__(self, index): | ||
| return self.contents[index] | ||
|
|
||
| def get_test_config() -> TestConfig: | ||
| def __iter__(self): | ||
| return iter(self.contents) | ||
|
|
||
|
|
||
| def get_test_bucket_prefix(name: str) -> BucketPrefixFixture: | ||
| """Create a new bucket/prefix fixture for the given test name.""" | ||
| region = getenv("CI_REGION") | ||
| bucket = getenv("CI_BUCKET") | ||
| express_bucket = getenv("CI_EXPRESS_BUCKET") | ||
| express_region = getenv("CI_EXPRESS_REGION") | ||
| prefix = getenv("CI_PREFIX") | ||
| region = getenv("CI_REGION") | ||
| storage_class = getenv("CI_STORAGE_CLASS", optional=True) | ||
| assert prefix == "" or prefix.endswith("/") | ||
|
|
||
| nonce = random.randrange(2**64) | ||
| prefix = f"{prefix}{name}/{nonce}/" | ||
|
|
||
| return BucketPrefixFixture(region, bucket, prefix, storage_class) | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def image_directory(request) -> BucketPrefixFixture: | ||
| """Create a bucket/prefix fixture that contains a directory of random JPG image files.""" | ||
| NUM_IMAGES = 10 | ||
| IMAGE_SIZE = 100 | ||
| fixture = get_test_bucket_prefix(f"{request.node.name}/image_directory") | ||
| for i in range(NUM_IMAGES): | ||
| data = np.random.randint(0, 256, IMAGE_SIZE * IMAGE_SIZE * 3, np.uint8) | ||
| data = data.reshape(IMAGE_SIZE, IMAGE_SIZE, 3) | ||
| image = Image.fromarray(data, "RGB") | ||
| image_bytes = io.BytesIO() | ||
| image.save(image_bytes, "jpeg") | ||
| image_bytes.seek(0) | ||
| image_bytes = image_bytes.read() | ||
|
|
||
| key = f"img{i:03d}.jpg" | ||
| fixture.add(key, image_bytes) | ||
|
|
||
| return fixture | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def sample_directory(request) -> BucketPrefixFixture: | ||
| fixture = get_test_bucket_prefix(f"{request.node.name}/sample_files") | ||
| fixture.add("hello_world.txt", b"Hello, World!\n") | ||
| return fixture | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def put_object_tests_directory(request) -> BucketPrefixFixture: | ||
| fixture = get_test_bucket_prefix(f"{request.node.name}/put_integration_tests") | ||
| fixture.add("to_overwrite.txt", b"before") | ||
| return fixture | ||
|
|
||
|
|
||
| return TestConfig(region, bucket, express_bucket, express_region) | ||
| @pytest.fixture | ||
| def checkpoint_directory(request) -> BucketPrefixFixture: | ||
| return get_test_bucket_prefix(f"{request.node.name}/checkpoint_directory") |
Oops, something went wrong.