From 7f6d5cc60ee82c2f9a9bc42b1fb085fefc76d85d Mon Sep 17 00:00:00 2001 From: dnnanuti Date: Thu, 18 Jan 2024 12:14:20 +0000 Subject: [PATCH] Expose function to enable debug logging Draft function to enable CRT logs for debug purposes --- .../src/s3torchconnector/__init__.py | 3 ++- s3torchconnector/tst/e2e/customer_issue.py | 27 +++++++++++++++++++ s3torchconnector/tst/e2e/customer_issue2.py | 19 +++++++++++++ s3torchconnector/tst/e2e/generate_tars.py | 12 +++++++++ .../src/s3torchconnectorclient/__init__.py | 4 +-- s3torchconnectorclient/rust/src/lib.rs | 8 ++++++ 6 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 s3torchconnector/tst/e2e/customer_issue.py create mode 100644 s3torchconnector/tst/e2e/customer_issue2.py create mode 100644 s3torchconnector/tst/e2e/generate_tars.py diff --git a/s3torchconnector/src/s3torchconnector/__init__.py b/s3torchconnector/src/s3torchconnector/__init__.py index fec26aaf..9cf095e3 100644 --- a/s3torchconnector/src/s3torchconnector/__init__.py +++ b/s3torchconnector/src/s3torchconnector/__init__.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # // SPDX-License-Identifier: BSD -from s3torchconnectorclient import S3Exception +from s3torchconnectorclient import S3Exception, enable_debug_logging # The order of these imports is the same in which they will be rendered # in the API docs generated with Sphinx. @@ -20,4 +20,5 @@ "S3Writer", "S3Exception", "__version__", + "enable_debug_logging" ] diff --git a/s3torchconnector/tst/e2e/customer_issue.py b/s3torchconnector/tst/e2e/customer_issue.py new file mode 100644 index 00000000..bbc529b5 --- /dev/null +++ b/s3torchconnector/tst/e2e/customer_issue.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# // SPDX-License-Identifier: BSD +from s3torchconnector import S3MapDataset, S3IterableDataset, enable_debug_logging +from s3torchconnectorclient import LOG_TRACE +import logging + +enable_debug_logging() +logging.basicConfig( + format="%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s" +) +logging.getLogger().setLevel(LOG_TRACE) + +# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/" +DATASET_URI="s3://s3torchconnector-customer-issues/300mb/" +REGION = "eu-north-1" + +map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION) +iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION) + +# Randomly access to an item in map_dataset. +object = map_dataset[0] + +# Learn about bucket, key, and content of the object +bucket = object.bucket +key = object.key +content = object.read() +len(content) \ No newline at end of file diff --git a/s3torchconnector/tst/e2e/customer_issue2.py b/s3torchconnector/tst/e2e/customer_issue2.py new file mode 100644 index 00000000..8e3cf3de --- /dev/null +++ b/s3torchconnector/tst/e2e/customer_issue2.py @@ -0,0 +1,19 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# // SPDX-License-Identifier: BSD +from s3torchconnector import S3MapDataset, S3IterableDataset + +# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/" +DATASET_URI="s3://s3torchconnector-customer-issues/300mb/" +REGION = "eu-north-1" + +map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION) +iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION) + +# Randomly access to an item in map_dataset. +object = map_dataset[0] + +# Learn about bucket, key, and content of the object +bucket = object.bucket +key = object.key +content = object.read() +len(content) \ No newline at end of file diff --git a/s3torchconnector/tst/e2e/generate_tars.py b/s3torchconnector/tst/e2e/generate_tars.py new file mode 100644 index 00000000..bb89ac58 --- /dev/null +++ b/s3torchconnector/tst/e2e/generate_tars.py @@ -0,0 +1,12 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# // SPDX-License-Identifier: BSD + +import random + +filename = input("Enter the file name: ") +filesize = int(input("Enter the file size in mb: ")) * 1024 * 1024 +with open(filename, "wb") as f: + for i in range(filesize): + #convert int to 8bit value + value = bytes([random.randint(0, 255)]) + f.write(value) \ No newline at end of file diff --git a/s3torchconnectorclient/python/src/s3torchconnectorclient/__init__.py b/s3torchconnectorclient/python/src/s3torchconnectorclient/__init__.py index d4ff3d15..c61d03a1 100644 --- a/s3torchconnectorclient/python/src/s3torchconnectorclient/__init__.py +++ b/s3torchconnectorclient/python/src/s3torchconnectorclient/__init__.py @@ -5,7 +5,7 @@ from ._logger_patch import TRACE as LOG_TRACE from ._logger_patch import _install_trace_logging -from ._mountpoint_s3_client import S3Exception, __version__ +from ._mountpoint_s3_client import S3Exception, __version__, enable_debug_logging _install_trace_logging() @@ -16,4 +16,4 @@ def _s3exception_reduce(exc: S3Exception): copyreg.pickle(S3Exception, _s3exception_reduce) -__all__ = ["LOG_TRACE", "S3Exception", "__version__"] +__all__ = ["LOG_TRACE", "S3Exception", "__version__", "enable_debug_logging"] diff --git a/s3torchconnectorclient/rust/src/lib.rs b/s3torchconnectorclient/rust/src/lib.rs index 11622c2e..20fc3379 100644 --- a/s3torchconnectorclient/rust/src/lib.rs +++ b/s3torchconnectorclient/rust/src/lib.rs @@ -4,6 +4,7 @@ */ use log::LevelFilter; +use mountpoint_s3_crt::common::rust_log_adapter::RustLogAdapter; use pyo3::prelude::*; use pyo3_log::Logger; @@ -27,6 +28,12 @@ mod put_object_stream; mod python_structs; mod build_info; +#[pyfunction] +#[pyo3(name = "enable_debug_logging")] +fn enable_debug_logging() { + let _ = RustLogAdapter::try_init().map_err(python_exception); +} + #[pymodule] #[pyo3(name = "_mountpoint_s3_client")] fn make_lib(py: Python, mountpoint_s3_client: &PyModule) -> PyResult<()> { @@ -49,5 +56,6 @@ fn make_lib(py: Python, mountpoint_s3_client: &PyModule) -> PyResult<()> { mountpoint_s3_client.add_class::()?; mountpoint_s3_client.add("S3Exception", py.get_type::())?; mountpoint_s3_client.add("__version__", build_info::FULL_VERSION)?; + mountpoint_s3_client.add_function(wrap_pyfunction!(enable_debug_logging, mountpoint_s3_client)?)?; Ok(()) }