-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Unknown
committed
Dec 15, 2023
0 parents
commit 7c7158b
Showing
101 changed files
with
11,658 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
# Sphinx build info version 1 | ||
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. | ||
config: 7a76781112b2385bb6293343f6ebc58b | ||
tags: 645f666f9bcd5a90fca523b33c5a78b7 |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file added
BIN
+15.1 KB
.doctrees/autoapi/s3torchconnector/_s3client/_mock_s3client/index.doctree
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
<!DOCTYPE html> | ||
<html class="writer-html5" lang="en" data-content_root="../"> | ||
<head> | ||
<meta charset="utf-8" /> | ||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
<title>Overview: module code — Amazon S3 Connector for PyTorch 1.1.0 documentation</title> | ||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" /> | ||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" /> | ||
<link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=eafc0fe6" /> | ||
|
||
|
||
<!--[if lt IE 9]> | ||
<script src="../_static/js/html5shiv.min.js"></script> | ||
<![endif]--> | ||
|
||
<script src="../_static/jquery.js?v=5d32c60e"></script> | ||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script> | ||
<script src="../_static/documentation_options.js?v=fc837d61"></script> | ||
<script src="../_static/doctools.js?v=888ff710"></script> | ||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script> | ||
<script src="../_static/js/theme.js"></script> | ||
<link rel="index" title="Index" href="../genindex.html" /> | ||
<link rel="search" title="Search" href="../search.html" /> | ||
</head> | ||
|
||
<body class="wy-body-for-nav"> | ||
<div class="wy-grid-for-nav"> | ||
<nav data-toggle="wy-nav-shift" class="wy-nav-side"> | ||
<div class="wy-side-scroll"> | ||
<div class="wy-side-nav-search" > | ||
|
||
|
||
|
||
<a href="../index.html" class="icon icon-home"> | ||
Amazon S3 Connector for PyTorch | ||
</a> | ||
<div role="search"> | ||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> | ||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" /> | ||
<input type="hidden" name="check_keywords" value="yes" /> | ||
<input type="hidden" name="area" value="default" /> | ||
</form> | ||
</div> | ||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu"> | ||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p> | ||
<ul> | ||
<li class="toctree-l1"><a class="reference internal" href="../autoapi/index.html">API Reference</a></li> | ||
</ul> | ||
|
||
</div> | ||
</div> | ||
</nav> | ||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" > | ||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i> | ||
<a href="../index.html">Amazon S3 Connector for PyTorch</a> | ||
</nav> | ||
|
||
<div class="wy-nav-content"> | ||
<div class="rst-content"> | ||
<div role="navigation" aria-label="Page navigation"> | ||
<ul class="wy-breadcrumbs"> | ||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li> | ||
<li class="breadcrumb-item active">Overview: module code</li> | ||
<li class="wy-breadcrumbs-aside"> | ||
</li> | ||
</ul> | ||
<hr/> | ||
</div> | ||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> | ||
<div itemprop="articleBody"> | ||
|
||
<h1>All modules for which code is available</h1> | ||
<ul><li><a href="s3torchconnector/_s3_bucket_iterable.html">s3torchconnector._s3_bucket_iterable</a></li> | ||
<li><a href="s3torchconnector/_s3bucket_key.html">s3torchconnector._s3bucket_key</a></li> | ||
<li><a href="s3torchconnector/_s3client/_mock_s3client.html">s3torchconnector._s3client._mock_s3client</a></li> | ||
<li><a href="s3torchconnector/_s3client/_s3client.html">s3torchconnector._s3client._s3client</a></li> | ||
<li><a href="s3torchconnector/_s3dataset_common.html">s3torchconnector._s3dataset_common</a></li> | ||
<li><a href="s3torchconnector/_version.html">s3torchconnector._version</a></li> | ||
<li><a href="s3torchconnector/s3checkpoint.html">s3torchconnector.s3checkpoint</a></li> | ||
<li><a href="s3torchconnector/s3iterable_dataset.html">s3torchconnector.s3iterable_dataset</a></li> | ||
<li><a href="s3torchconnector/s3map_dataset.html">s3torchconnector.s3map_dataset</a></li> | ||
<li><a href="s3torchconnector/s3reader.html">s3torchconnector.s3reader</a></li> | ||
<li><a href="s3torchconnector/s3writer.html">s3torchconnector.s3writer</a></li> | ||
</ul> | ||
|
||
</div> | ||
</div> | ||
<footer> | ||
|
||
<hr/> | ||
|
||
<div role="contentinfo"> | ||
<p>© Copyright 2023, Amazon S3.</p> | ||
</div> | ||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a | ||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> | ||
provided by <a href="https://readthedocs.org">Read the Docs</a>. | ||
|
||
|
||
</footer> | ||
</div> | ||
</div> | ||
</section> | ||
</div> | ||
<script> | ||
jQuery(function () { | ||
SphinxRtdTheme.Navigation.enable(true); | ||
}); | ||
</script> | ||
|
||
</body> | ||
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,180 @@ | ||
<!DOCTYPE html> | ||
<html class="writer-html5" lang="en" data-content_root="../../"> | ||
<head> | ||
<meta charset="utf-8" /> | ||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
<title>s3torchconnector._s3_bucket_iterable — Amazon S3 Connector for PyTorch 1.1.0 documentation</title> | ||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" /> | ||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=19f00094" /> | ||
<link rel="stylesheet" type="text/css" href="../../_static/graphviz.css?v=eafc0fe6" /> | ||
|
||
|
||
<!--[if lt IE 9]> | ||
<script src="../../_static/js/html5shiv.min.js"></script> | ||
<![endif]--> | ||
|
||
<script src="../../_static/jquery.js?v=5d32c60e"></script> | ||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script> | ||
<script src="../../_static/documentation_options.js?v=fc837d61"></script> | ||
<script src="../../_static/doctools.js?v=888ff710"></script> | ||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> | ||
<script src="../../_static/js/theme.js"></script> | ||
<link rel="index" title="Index" href="../../genindex.html" /> | ||
<link rel="search" title="Search" href="../../search.html" /> | ||
</head> | ||
|
||
<body class="wy-body-for-nav"> | ||
<div class="wy-grid-for-nav"> | ||
<nav data-toggle="wy-nav-shift" class="wy-nav-side"> | ||
<div class="wy-side-scroll"> | ||
<div class="wy-side-nav-search" > | ||
|
||
|
||
|
||
<a href="../../index.html" class="icon icon-home"> | ||
Amazon S3 Connector for PyTorch | ||
</a> | ||
<div role="search"> | ||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get"> | ||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" /> | ||
<input type="hidden" name="check_keywords" value="yes" /> | ||
<input type="hidden" name="area" value="default" /> | ||
</form> | ||
</div> | ||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu"> | ||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p> | ||
<ul> | ||
<li class="toctree-l1"><a class="reference internal" href="../../autoapi/index.html">API Reference</a></li> | ||
</ul> | ||
|
||
</div> | ||
</div> | ||
</nav> | ||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" > | ||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i> | ||
<a href="../../index.html">Amazon S3 Connector for PyTorch</a> | ||
</nav> | ||
|
||
<div class="wy-nav-content"> | ||
<div class="rst-content"> | ||
<div role="navigation" aria-label="Page navigation"> | ||
<ul class="wy-breadcrumbs"> | ||
<li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li> | ||
<li class="breadcrumb-item"><a href="../index.html">Module code</a></li> | ||
<li class="breadcrumb-item active">s3torchconnector._s3_bucket_iterable</li> | ||
<li class="wy-breadcrumbs-aside"> | ||
</li> | ||
</ul> | ||
<hr/> | ||
</div> | ||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> | ||
<div itemprop="articleBody"> | ||
|
||
<h1>Source code for s3torchconnector._s3_bucket_iterable</h1><div class="highlight"><pre> | ||
<span></span><span class="c1"># Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.</span> | ||
<span class="c1"># // SPDX-License-Identifier: BSD</span> | ||
|
||
<span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="n">partial</span> | ||
<span class="kn">from</span> <span class="nn">itertools</span> <span class="kn">import</span> <span class="n">chain</span> | ||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Iterator</span><span class="p">,</span> <span class="n">List</span> | ||
|
||
<span class="kn">from</span> <span class="nn">s3torchconnectorclient._mountpoint_s3_client</span> <span class="kn">import</span> <span class="p">(</span> | ||
<span class="n">ObjectInfo</span><span class="p">,</span> | ||
<span class="n">ListObjectResult</span><span class="p">,</span> | ||
<span class="n">ListObjectStream</span><span class="p">,</span> | ||
<span class="p">)</span> | ||
|
||
<span class="kn">from</span> <span class="nn">._s3client</span> <span class="kn">import</span> <span class="n">S3Client</span> | ||
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">S3Reader</span> | ||
|
||
|
||
<div class="viewcode-block" id="S3BucketIterable"> | ||
<a class="viewcode-back" href="../../autoapi/s3torchconnector/_s3_bucket_iterable/index.html#s3torchconnector._s3_bucket_iterable.S3BucketIterable">[docs]</a> | ||
<span class="k">class</span> <span class="nc">S3BucketIterable</span><span class="p">:</span> | ||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client</span><span class="p">:</span> <span class="n">S3Client</span><span class="p">,</span> <span class="n">bucket</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_client</span> <span class="o">=</span> <span class="n">client</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span> <span class="o">=</span> <span class="n">bucket</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_prefix</span> <span class="o">=</span> <span class="n">prefix</span> | ||
|
||
<span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> | ||
<span class="c1"># This allows us to iterate multiple times by re-creating the `_list_stream`</span> | ||
<span class="k">return</span> <span class="nb">iter</span><span class="p">(</span><span class="n">S3BucketIterator</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_client</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prefix</span><span class="p">))</span></div> | ||
|
||
|
||
|
||
<div class="viewcode-block" id="S3BucketIterator"> | ||
<a class="viewcode-back" href="../../autoapi/s3torchconnector/_s3_bucket_iterable/index.html#s3torchconnector._s3_bucket_iterable.S3BucketIterator">[docs]</a> | ||
<span class="k">class</span> <span class="nc">S3BucketIterator</span><span class="p">:</span> | ||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client</span><span class="p">:</span> <span class="n">S3Client</span><span class="p">,</span> <span class="n">bucket</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_client</span> <span class="o">=</span> <span class="n">client</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span> <span class="o">=</span> <span class="n">bucket</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span> <span class="o">=</span> <span class="n">_PickleableListObjectStream</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span> | ||
|
||
<span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterator</span><span class="p">[</span><span class="n">S3Reader</span><span class="p">]:</span> | ||
<span class="k">return</span> <span class="nb">map</span><span class="p">(</span> | ||
<span class="n">partial</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_client</span><span class="o">.</span><span class="n">from_bucket_and_object_info</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span><span class="p">),</span> | ||
<span class="n">chain</span><span class="o">.</span><span class="n">from_iterable</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">_extract_object_info</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="p">)),</span> | ||
<span class="p">)</span></div> | ||
|
||
|
||
|
||
<span class="k">class</span> <span class="nc">_PickleableListObjectStream</span><span class="p">:</span> | ||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client</span><span class="p">:</span> <span class="n">S3Client</span><span class="p">,</span> <span class="n">bucket</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_client</span> <span class="o">=</span> <span class="n">client</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">client</span><span class="o">.</span><span class="n">list_objects</span><span class="p">(</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="p">))</span> | ||
|
||
<span class="k">def</span> <span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> | ||
<span class="k">return</span> <span class="bp">self</span> | ||
|
||
<span class="k">def</span> <span class="fm">__next__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ListObjectResult</span><span class="p">:</span> | ||
<span class="k">return</span> <span class="nb">next</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="p">)</span> | ||
|
||
<span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> | ||
<span class="k">return</span> <span class="p">{</span> | ||
<span class="s2">"client"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_client</span><span class="p">,</span> | ||
<span class="s2">"bucket"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> | ||
<span class="s2">"prefix"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span> | ||
<span class="s2">"delimiter"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span> | ||
<span class="s2">"max_keys"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">max_keys</span><span class="p">,</span> | ||
<span class="s2">"continuation_token"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">continuation_token</span><span class="p">,</span> | ||
<span class="s2">"complete"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span><span class="o">.</span><span class="n">complete</span><span class="p">,</span> | ||
<span class="p">}</span> | ||
|
||
<span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_client</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s2">"client"</span><span class="p">]</span> | ||
<span class="bp">self</span><span class="o">.</span><span class="n">_list_stream</span> <span class="o">=</span> <span class="n">ListObjectStream</span><span class="o">.</span><span class="n">_from_state</span><span class="p">(</span><span class="o">**</span><span class="n">state</span><span class="p">)</span> | ||
|
||
|
||
<span class="k">def</span> <span class="nf">_extract_object_info</span><span class="p">(</span><span class="n">list_result</span><span class="p">:</span> <span class="n">ListObjectResult</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">ObjectInfo</span><span class="p">]:</span> | ||
<span class="k">return</span> <span class="n">list_result</span><span class="o">.</span><span class="n">object_info</span> | ||
</pre></div> | ||
|
||
</div> | ||
</div> | ||
<footer> | ||
|
||
<hr/> | ||
|
||
<div role="contentinfo"> | ||
<p>© Copyright 2023, Amazon S3.</p> | ||
</div> | ||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a | ||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> | ||
provided by <a href="https://readthedocs.org">Read the Docs</a>. | ||
|
||
|
||
</footer> | ||
</div> | ||
</div> | ||
</section> | ||
</div> | ||
<script> | ||
jQuery(function () { | ||
SphinxRtdTheme.Navigation.enable(true); | ||
}); | ||
</script> | ||
|
||
</body> | ||
</html> |
Oops, something went wrong.