-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Expose function to enable debug logging
Draft function to enable CRT logs for debug purposes
- Loading branch information
dnnanuti
committed
Jan 18, 2024
1 parent
29db5c9
commit 7f6d5cc
Showing
6 changed files
with
70 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
# // SPDX-License-Identifier: BSD | ||
from s3torchconnector import S3MapDataset, S3IterableDataset, enable_debug_logging | ||
from s3torchconnectorclient import LOG_TRACE | ||
import logging | ||
|
||
enable_debug_logging() | ||
logging.basicConfig( | ||
format="%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s" | ||
) | ||
logging.getLogger().setLevel(LOG_TRACE) | ||
|
||
# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/" | ||
DATASET_URI="s3://s3torchconnector-customer-issues/300mb/" | ||
REGION = "eu-north-1" | ||
|
||
map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION) | ||
iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION) | ||
|
||
# Randomly access to an item in map_dataset. | ||
object = map_dataset[0] | ||
|
||
# Learn about bucket, key, and content of the object | ||
bucket = object.bucket | ||
key = object.key | ||
content = object.read() | ||
len(content) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
# // SPDX-License-Identifier: BSD | ||
from s3torchconnector import S3MapDataset, S3IterableDataset | ||
|
||
# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/" | ||
DATASET_URI="s3://s3torchconnector-customer-issues/300mb/" | ||
REGION = "eu-north-1" | ||
|
||
map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION) | ||
iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION) | ||
|
||
# Randomly access to an item in map_dataset. | ||
object = map_dataset[0] | ||
|
||
# Learn about bucket, key, and content of the object | ||
bucket = object.bucket | ||
key = object.key | ||
content = object.read() | ||
len(content) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
# // SPDX-License-Identifier: BSD | ||
|
||
import random | ||
|
||
filename = input("Enter the file name: ") | ||
filesize = int(input("Enter the file size in mb: ")) * 1024 * 1024 | ||
with open(filename, "wb") as f: | ||
for i in range(filesize): | ||
#convert int to 8bit value | ||
value = bytes([random.randint(0, 255)]) | ||
f.write(value) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters