Merge branch 'develop' into SYNPY-1322-OOP-POC

.gitignore

@@ -22,3 +22,4 @@ build/*
 /venv
 
 .vscode/
+CONFIGFILE

Dockerfile

@@ -3,14 +3,18 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio
 
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
-       python3 \
-       python3-setuptools \
-       python3-pip \
-       python3-pandas \
+    python3 \
+    python3-setuptools \
+    python3-pip \
+    python3-pandas \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install --upgrade pip
 
-COPY . /synapsePythonClient
 WORKDIR /synapsePythonClient
-RUN python3 setup.py install
+COPY . .
+
+RUN pip install --no-cache-dir .
+
+
+LABEL org.opencontainers.image.source='https://github.com/Sage-Bionetworks/synapsePythonClient'

docs/articles/benchmarking.rst

@@ -12,11 +12,53 @@ Results
 ===================
 
 
-11/14/2023
-==========================
+12/06/2023: Uploading files to Synapse, Varying thread count, 5 annotations per file
+====================================================================================
 The results were created on a `t3a.micro` EC2 instance with a 200GB disk size running in us-east-1.
 The script that was run can be found in `docs/scripts`. The time to create the files on disk is not included.
 
+This test includes adding 5 annotations to each file, a Text, Integer, Floating Point, Boolean, and Date.
+
+S3 was not benchmarked again.
+
+As a result of these tests the sweet spot for thread count is around 50 threads. It is not reccomended to
+go over 50 threads as it resulted in signficant instability in the client.
+
++---------------------------+--------------+-------------------+---------------------+---------------+
+| Test                      | Thread Count | Synapseutils Sync | os.walk + syn.store | Per file size |
++===========================+==============+===================+=====================+===============+
+| 25 Files 1MB total size   | 6            | 10.75s            | 10.96s              | 40KB          |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 25 Files 1MB total size   | 25           | 6.79s             | 11.31s              | 40KB          |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 25 Files 1MB total size   | 50           | 6.05s             | 10.90s              | 40KB          |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 25 Files 1MB total size   | 100          | 6.14s             | 10.89s              | 40KB          |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 775 Files 10MB total size | 6            | 268.33s           | 298.12s             | 12.9KB        |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 775 Files 10MB total size | 25           | 162.63s           | 305.93s             | 12.9KB        |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 775 Files 10MB total size | 50           | 86.46s            | 304.40s             | 12.9KB        |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 775 Files 10MB total size | 100          | 85.55s            | 304.71s             | 12.9KB        |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 10 Files 1GB total size   | 6            | 27.17s            | 36.25s              | 100MB         |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 10 Files 1GB total size   | 25           | 22.26s            | 12.77s              | 100MB         |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 10 Files 1GB total size   | 50           | 22.24s            | 12.26s              | 100MB         |
++---------------------------+--------------+-------------------+---------------------+---------------+
+| 10 Files 1GB total size   | 100          | Wouldn't complete | Wouldn't complete   | 100MB         |
++---------------------------+--------------+-------------------+---------------------+---------------+
+
+
+11/14/2023: Uploading files to Synapse, Default thread count
+============================================================
+The results were created on a `t3a.micro` EC2 instance with a 200GB disk size running in us-east-1.
+The script that was run can be found in `docs/scripts`. The time to create the files on disk is not included.
+
+This test uses the default number of threads in the client: `multiprocessing.cpu_count() + 4`
 
 +---------------------------+-------------------+---------------------+---------+---------------+
 | Test                      | Synapseutils Sync | os.walk + syn.store | S3 Sync | Per file size |

docs/build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 90d422e4b09fda20663a5d255dc16438
+config: b4567f79d237ed79859c36a1e24179ab
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/build/html/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '3.1.1',
+    VERSION: '3.2.0',
     LANGUAGE: 'None',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',
@@ -11,4 +11,4 @@ var DOCUMENTATION_OPTIONS = {
     NAVIGATION_WITH_KEYS: false,
     SHOW_SEARCH_SUMMARY: true,
     ENABLE_SEARCH_SHORTCUTS: true,
-};
+};

docs/build/html/_static/pygments.css

@@ -72,4 +72,4 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
 .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */
 .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */
 .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */
-.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */
+.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */

docs/build/html/api/Activity.html

@@ -4,15 +4,15 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Provenance &mdash; Synapse Python Client 3.1.1 documentation</title>
+  <title>Provenance &mdash; Synapse Python Client 3.2.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
     <link rel="shortcut icon" href="../_static/favicon.png"/>
   <!--[if lt IE 9]>
     <script src="../_static/js/html5shiv.min.js"></script>
   <![endif]-->
-
+  
         <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
         <script src="../_static/jquery.js"></script>
         <script src="../_static/underscore.js"></script>
@@ -21,23 +21,23 @@
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
     <link rel="next" title="Annotations" href="Annotations.html" />
-    <link rel="prev" title="Evaluations" href="Evaluation.html" />
+    <link rel="prev" title="Evaluations" href="Evaluation.html" /> 
 </head>
 
-<body class="wy-body-for-nav">
+<body class="wy-body-for-nav"> 
   <div class="wy-grid-for-nav">
     <nav data-toggle="wy-nav-shift" class="wy-nav-side">
       <div class="wy-side-scroll">
         <div class="wy-side-nav-search" >
 
-
-
+          
+          
           <a href="../index.html">
-
+            
               <img src="../_static/synapse_logo.png" class="logo" alt="Logo"/>
           </a>
               <div class="version">
-                3.1.1
+                3.2.0
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -110,7 +110,7 @@
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
-
+             
   <span class="target" id="module-synapseclient.activity"></span><section id="provenance">
 <h1>Provenance<a class="headerlink" href="#provenance" title="Permalink to this headline"></a></h1>
 <p>The Activity object represents the source of a data set or the data processing steps used to produce it. Using
@@ -247,14 +247,14 @@ <h2>Activity<a class="headerlink" href="#activity" title="Permalink to this head
 
   <div role="contentinfo">
     <p>&#169; Copyright 2012-2023, Sage Bionetworks.
-      <span class="lastupdated">Last updated on Oct 30, 2023.
+      <span class="lastupdated">Last updated on Nov 27, 2023.
       </span></p>
   </div>
 
   Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
     <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
     provided by <a href="https://readthedocs.org">Read the Docs</a>.
-
+   
 
 </footer>
         </div>
@@ -268,7 +268,7 @@ <h2>Activity<a class="headerlink" href="#activity" title="Permalink to this head
   </script>
     <div class="footer">
     &copy; <a href="http://sagebase.org/">Sage Bionetworks</a>.
-      Last updated on Oct 30, 2023.
+      Last updated on Nov 27, 2023.
       Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.5.0.
     </div>
     <script type="text/javascript">
@@ -281,4 +281,4 @@ <h2>Activity<a class="headerlink" href="#activity" title="Permalink to this head
     </script>
 
 </body>
-</html>
+</html>