Default to setting properties in one request, add fallback option for broken proxies

README.md

@@ -8,6 +8,9 @@ It can also be set up to transparently proxy an upstream Galaxy server, storing
 # Acknowledgements
 This project is _heavily_ inspired by [amanda](https://github.com/sivel/amanda/).
 
+# Artifactory compatibility
+For the time being some features may require an Artifactory Pro license. Work is being done to workaround the (many) limitations that JFrog has placed on API calls. [These limitations have also complicated the ability to run integration tests](https://github.com/briantist/galactory/issues/6). As a result, the test of whether all Pro license dependent API calls have been rooted out, will be whether test coverage exists that can run against Artifactory OSS.
+
 # How to use
 There isn't any proper documentation yet. The help output is below.
 
@@ -87,6 +90,11 @@ optional arguments:
                         Populate the upstream cache in Artifactory. Should be false when no API key is
                         provided or the key has no permission to write.
                         [env var: GALACTORY_CACHE_WRITE]
+  --use-property-fallback
+                        Set properties of an uploaded collection in a separate request after publshinng.
+                        Requires a Pro license of Artifactory. This feature is a workaround for an
+                        Artifactory proxy configuration error and may be removed in a future version.
+                        [env var: GALACTORY_USE_PROPERTY_FALLBACK]
 
 Args that start with '--' (eg. --listen-addr) can also be set in a config file
 (/etc/galactory.d/*.conf or ~/.galactory/*.conf or specified via -c). Config file syntax allows:

changelogs/fragments/19-property-fallback.yml

@@ -0,0 +1,3 @@
+---
+breaking_changes:
+  - property setting - by default properties are now set on the initial upload of a collection to Artifactory. This removes an additional roundtrip to the server, and removes another API call that requires a Pro license of Artifactory. However, some reverse proxy configurations will not work with this. A new parameter ``USE_PROPERTY_FALLBACK`` has been added which will use the old behavior of setting properties in a second request,  but this will still require a Pro license to use. This option may be removed in a future version. See (https://github.com/briantist/galactory/issues/19).

galactory/__main__.py

@@ -61,6 +61,7 @@ def __call__(self, parser, namespace, values, option_string=None):
     parser.add_argument('--cache-minutes', default=60, type=int, env_var='GALACTORY_CACHE_MINUTES', help='The time period that a cache entry should be considered valid.')
     parser.add_argument('--cache-read', action=_StrBool, default=True, env_var='GALACTORY_CACHE_READ', help='Look for upsteam caches and use their values.')
     parser.add_argument('--cache-write', action=_StrBool, default=True, env_var='GALACTORY_CACHE_WRITE', help='Populate the upstream cache in Artifactory. Should be false when no API key is provided or the key has no permission to write.')
+    parser.add_argument('--use-property-fallback', action='store_true', env_var='GALACTORY_USE_PROPERTY_FALLBACK', help='Set properties of an uploaded collection in a separate request after publshinng. Requires a Pro license of Artifactory. This feature is a workaround for an Artifactory proxy configuration error and may be removed in a future version.')
     args = parser.parse_args()
 
     logging.basicConfig(filename=args.log_file, level=args.log_level)
@@ -79,6 +80,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         CACHE_MINUTES=args.cache_minutes,
         CACHE_READ=args.cache_read,
         CACHE_WRITE=args.cache_write,
+        USE_PROPERTY_FALLBACK=args.use_property_fallback,
     )
 
     app.run(args.listen_addr, args.listen_port, threaded=True)

galactory/api/v2/collections.py

@@ -162,13 +162,14 @@ def publish():
     sha256 = request.form['sha256']
     file = request.files['file']
     skip_configured_key = current_app.config['PUBLISH_SKIP_CONFIGURED_KEY']
+    property_fallback = current_app.config.get('USE_PROPERTY_FALLBACK', False)
 
     target = authorize(request, current_app.config['ARTIFACTORY_PATH'] / file.filename, skip_configured_key=skip_configured_key)
 
     with _chunk_to_temp(Base64IO(file)) as tmp:
         if tmp.sha256 != sha256:
             abort(Response(f"Hash mismatch: uploaded=='{sha256}', calculated=='{tmp.sha256}'", C.HTTP_INTERNAL_SERVER_ERROR))
 
-        upload_collection_from_hashed_tempfile(target, tmp)
+        upload_collection_from_hashed_tempfile(target, tmp, property_fallback=property_fallback)
 
     return jsonify(task=url_for('api.v2.import_singleton', _external=True))

galactory/download/download.py

@@ -17,6 +17,7 @@ def download(filename):
     cache_minutes = current_app.config['CACHE_MINUTES']
     cache_read = current_app.config['CACHE_READ']
     cache_write = current_app.config['CACHE_WRITE']
+    property_fallback = current_app.config.get('USE_PROPERTY_FALLBACK', False)
 
     try:
         stat = artifact.stat()
@@ -34,7 +35,7 @@ def download(filename):
             if not cache_write:
                 return send_file(tmp.handle, as_attachment=True, download_name=filename, etag=False)
 
-            upload_collection_from_hashed_tempfile(artifact, tmp)
+            upload_collection_from_hashed_tempfile(artifact, tmp, property_fallback=property_fallback)
 
         stat = artifact.stat()
 

galactory/utilities.py

@@ -225,7 +225,7 @@ def _chunk_to_temp(fsrc, iterator=None, spool_size=5*1024*1024, seek_to_zero=Tru
     return HashedTempFile(tmp, md5sum.hexdigest(), sha1sum.hexdigest(),  sha256sum.hexdigest(), close=close)
 
 
-def upload_collection_from_hashed_tempfile(artifact: ArtifactoryPath, tmpfile: HashedTempFile) -> Dict[str, Any]:
+def upload_collection_from_hashed_tempfile(artifact: ArtifactoryPath, tmpfile: HashedTempFile, property_fallback: bool = False) -> Dict[str, Any]:
     try:
         manifest = load_manifest_from_archive(tmpfile.handle)
     except Exception:
@@ -240,16 +240,18 @@ def upload_collection_from_hashed_tempfile(artifact: ArtifactoryPath, tmpfile: H
             'fqcn': f"{ci['namespace']}.{ci['name']}"
         }
 
+    params = props
+    if property_fallback:
+        params = {}
+
     try:
-        artifact.deploy(tmpfile.handle, md5=tmpfile.md5, sha1=tmpfile.sha1, sha256=tmpfile.sha256)  # parameters=props
-        # we can't use parameters=props here because Artifactory rejects quote characters
-        # in their matrix params, and that's not compatible with collection_info because
-        # it's JSON, so we'll still have to set the properties as a separate request :(
+        artifact.deploy(tmpfile.handle, md5=tmpfile.md5, sha1=tmpfile.sha1, sha256=tmpfile.sha256, parameters=params)
     except ArtifactoryException as exc:
         cause = exc.__cause__
         current_app.logger.debug(cause)
         abort(Response(cause.response.text, cause.response.status_code))
     else:
-        artifact.properties = props
+        if property_fallback:
+            artifact.properties = props
 
     return props