Expose function to enable debug logging

Draft function to enable CRT logs for debug purposes

s3torchconnector/src/s3torchconnector/__init__.py

@@ -1,6 +1,6 @@
 #  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  // SPDX-License-Identifier: BSD
-from s3torchconnectorclient import S3Exception
+from s3torchconnectorclient import S3Exception, enable_debug_logging
 
 # The order of these imports is the same in which they will be rendered
 # in the API docs generated with Sphinx.
@@ -20,4 +20,5 @@
     "S3Writer",
     "S3Exception",
     "__version__",
+    "enable_debug_logging"
 ]

s3torchconnector/tst/e2e/customer_issue.py

@@ -0,0 +1,27 @@
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  // SPDX-License-Identifier: BSD
+from s3torchconnector import S3MapDataset, S3IterableDataset, enable_debug_logging
+from s3torchconnectorclient import LOG_TRACE
+import logging
+
+enable_debug_logging()
+logging.basicConfig(
+    format="%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s"
+)
+logging.getLogger().setLevel(LOG_TRACE)
+
+# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/"
+DATASET_URI="s3://s3torchconnector-customer-issues/300mb/"
+REGION = "eu-north-1"
+
+map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION)
+iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION)
+
+# Randomly access to an item in map_dataset.
+object = map_dataset[0]
+
+# Learn about bucket, key, and content of the object
+bucket = object.bucket
+key = object.key
+content = object.read()
+len(content)

s3torchconnector/tst/e2e/customer_issue2.py

@@ -0,0 +1,19 @@
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  // SPDX-License-Identifier: BSD
+from s3torchconnector import S3MapDataset, S3IterableDataset
+
+# DATASET_URI="s3://s3torchconnector-customer-issues/100mb/"
+DATASET_URI="s3://s3torchconnector-customer-issues/300mb/"
+REGION = "eu-north-1"
+
+map_dataset = S3MapDataset.from_prefix(DATASET_URI, region=REGION)
+iterable_dataset = S3IterableDataset.from_prefix(DATASET_URI, region=REGION)
+
+# Randomly access to an item in map_dataset.
+object = map_dataset[0]
+
+# Learn about bucket, key, and content of the object
+bucket = object.bucket
+key = object.key
+content = object.read()
+len(content)

s3torchconnector/tst/e2e/generate_tars.py

@@ -0,0 +1,12 @@
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  // SPDX-License-Identifier: BSD
+
+import random
+
+filename = input("Enter the file name: ")
+filesize = int(input("Enter the file size in mb: ")) * 1024 * 1024
+with open(filename, "wb") as f:
+    for i in range(filesize):
+        #convert int to 8bit value
+        value = bytes([random.randint(0, 255)])
+        f.write(value)

s3torchconnectorclient/python/src/s3torchconnectorclient/__init__.py

@@ -5,7 +5,7 @@
 
 from ._logger_patch import TRACE as LOG_TRACE
 from ._logger_patch import _install_trace_logging
-from ._mountpoint_s3_client import S3Exception, __version__
+from ._mountpoint_s3_client import S3Exception, __version__, enable_debug_logging
 
 _install_trace_logging()
 
@@ -16,4 +16,4 @@ def _s3exception_reduce(exc: S3Exception):
 
 copyreg.pickle(S3Exception, _s3exception_reduce)
 
-__all__ = ["LOG_TRACE", "S3Exception", "__version__"]
+__all__ = ["LOG_TRACE", "S3Exception", "__version__", "enable_debug_logging"]

s3torchconnectorclient/rust/src/lib.rs

@@ -4,6 +4,7 @@
  */
 
 use log::LevelFilter;
+use mountpoint_s3_crt::common::rust_log_adapter::RustLogAdapter;
 use pyo3::prelude::*;
 use pyo3_log::Logger;
 
@@ -27,6 +28,12 @@ mod put_object_stream;
 mod python_structs;
 mod build_info;
 
+#[pyfunction]
+#[pyo3(name = "enable_debug_logging")]
+fn enable_debug_logging() {
+    let _ = RustLogAdapter::try_init().map_err(python_exception);
+}
+
 #[pymodule]
 #[pyo3(name = "_mountpoint_s3_client")]
 fn make_lib(py: Python, mountpoint_s3_client: &PyModule) -> PyResult<()> {
@@ -49,5 +56,6 @@ fn make_lib(py: Python, mountpoint_s3_client: &PyModule) -> PyResult<()> {
     mountpoint_s3_client.add_class::<PyRestoreStatus>()?;
     mountpoint_s3_client.add("S3Exception", py.get_type::<S3Exception>())?;
     mountpoint_s3_client.add("__version__", build_info::FULL_VERSION)?;
+    mountpoint_s3_client.add_function(wrap_pyfunction!(enable_debug_logging, mountpoint_s3_client)?)?;
     Ok(())
 }