Test and fix the cache API

app_cache.py

@@ -18,7 +18,7 @@
     do_forward_resource_to_cache,
     do_get_resource_from_cache,
     do_are_resources_in_cache,
-    reset_cache,
+    do_reset_cache,
 )
 from latexonhttp.caching.store import get_cache_metadata
 
@@ -55,6 +55,7 @@
     "get_resource_from_cache": {"fn": do_get_resource_from_cache, "mode": "sync"},
     "get_cache_metadata": {"fn": get_cache_metadata, "mode": "sync"},
     "are_resources_in_cache": {"fn": do_are_resources_in_cache, "mode": "sync"},
+    "reset_cache": {"fn": do_reset_cache, "mode": "sync"},
 }
 
 # Other implementation ideas:
@@ -72,7 +73,7 @@
     logger.info("Preparing cache...")
     # Reset cache.
     # (Flush cache on disk and init metadata).
-    reset_cache()
+    do_reset_cache()
     logger.info("Cache init process, done...")
     rep_socket = context.socket(zmq.REP)
     rep_socket.bind("tcp://*:10000")

latexonhttp/api/caches.py

@@ -9,10 +9,11 @@
     :license: AGPL, see LICENSE for more details.
 """
 import logging
-from flask import Blueprint, request, jsonify, Response
+from flask import Blueprint, request, jsonify
 from latexonhttp.caching.resources import (
     get_cache_metadata_snapshot,
     are_resources_in_cache,
+    reset_cache,
 )
 
 from pprint import pformat
@@ -21,6 +22,12 @@
 
 caches_app = Blueprint("caches", __name__)
 
+# TODO Name the cache?
+#    /caches/<cache name>/resources
+# For eg.
+#    /caches/input/resources
+#    /caches/output/resources
+
 
 @caches_app.route("/resources", methods=["GET"])
 def resources_metadata():
@@ -30,6 +37,14 @@ def resources_metadata():
     return (jsonify(map_cache_metadata_for_public(cache_response)), 200)
 
 
+@caches_app.route("/resources", methods=["DELETE"])
+def resources_reset_cache():
+    is_ok, cache_response = reset_cache()
+    if not is_ok:
+        return (jsonify(cache_response), 500)
+    return '', 204
+
+
 @caches_app.route("/resources/check_cached", methods=["POST"])
 def resources_check_cached():
     payload = request.get_json()

latexonhttp/caching/bridge.py

@@ -12,9 +12,12 @@
 import msgpack
 import zmq
 
+RECV_TIMEOUT = 1500
+
 logger = logging.getLogger(__name__)
 
 context = zmq.Context()
+context.setsockopt(zmq.SocketOption.RCVTIMEO, RECV_TIMEOUT)
 req_socket = None
 dealer_socket = None
 
@@ -67,17 +70,29 @@ def request_cache_process_sync(message):
     socket = get_cache_process_sync_socket()
     if not socket:
         return False, {"error": "No cache process host defined"}
-    socket.send(serialize_message(message))
+    try:
+        socket.send(serialize_message(message), zmq.NOBLOCK)
+    except zmq.ZMQError as ze:
+        return False, {"error": "ZMQ socket failure", "message": str(ze)}
     # Get reply.
-    # TODO Manage timeout and server error.
-    return True, deserialize_message(socket.recv())
+    # TODO Use polling to handle timeouts
+    # and allow to choose timeout specific for the request.
+    # https://stackoverflow.com/questions/7538988/zeromq-how-to-prevent-infinite-wait
+    try:
+        response = socket.recv()
+        return True, deserialize_message(response)
+    except zmq.ZMQError as ze:
+        return False, {"error": "ZMQ socket failure", "message": str(ze)}
 
 
 def request_cache_process_async(message):
     socket = get_cache_process_async_socket()
     if not socket:
         return False, {"error": "No cache process host defined"}
-    socket.send(serialize_message(message))
+    try:
+        socket.send(serialize_message(message), zmq.NOBLOCK)
+    except zmq.ZMQError as ze:
+        return False, {"error": "ZMQ socket failure", "message": str(ze)}
     return True, None
     # We do not expect a response.
     # We could have an async mode with responses,

latexonhttp/caching/resources.hy

@@ -28,8 +28,6 @@
 ; External API.
 ; --------------------------------
 
-; TODO What to do if cache process not available / error?
-
 (defn get-cache-metadata-snapshot []
   (request-cache-process-sync
     {
@@ -65,11 +63,18 @@
       }
     }))
 
+(defn reset-cache []
+  (request-cache-process-sync
+    {
+      "action" "reset_cache"
+      "args" {}
+    }))
+
 ; --------------------------------
 ; Implementation.
 ; --------------------------------
 
-(defn reset-cache []
+(defn do-reset-cache []
   (setv action {
     "name" "reset_cache"
   })
@@ -160,7 +165,7 @@
   })
 
 (defn free-space-from-first-rec [resources size-to-free actions]
-  (if (pos? size-to-free )
+  (if (> size-to-free 0)
     (free-space-from-first-rec
       (list (drop 1 resources))
       (- size-to-free (get (first resources) "size"))
@@ -170,7 +175,7 @@
 (defn free-space-from-old-entries [cache-metadata size-to-free]
   (setv ordered-resources (fun-sort (.values (get cache-metadata "cached_resources")) (fn [resource] (get resource "added_at"))))
   ; (logger.debug "Ordered resources: %s" ordered-resources)
-  (if (pos? size-to-free)
+  (if (> size-to-free 0)
     ; Order resources by timestamp.
     ; Remove until we have freed the specified size
     (free-space-from-first-rec
@@ -219,6 +224,7 @@
         "total_size" 0
         "free_remaining_size" MAX-RESOURCES-CACHE-SIZE
         "cached_resources" {}
+        "hashing_algorighm" "sha256"
     })
 
 (defn process-cache-total-size [resources]

tests/test_api_caches.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+    tests.test_cache
+    ~~~~~~~~~~~~~~~~~~~~~
+    Test the LaTeX-On-HTTP file fetchers.
+
+    :copyright: (c) 2019 Yoan Tournade.
+    :license: AGPL, see LICENSE for more details.
+"""
+import pytest
+import requests
+
+
+def test_resource_flush_and_get_cached_files_empty(latex_on_http_api_url):
+    # Flush.
+    r = requests.delete(
+        latex_on_http_api_url + "/caches/resources",
+    )
+    assert r.status_code == 204
+    r = requests.get(
+        latex_on_http_api_url + "/caches/resources",
+    )
+    assert r.status_code == 200
+    response_payload = r.json()
+    assert response_payload["cached_resources"] == []
+    assert response_payload["total_size"] == 0
+
+
+# TODO Check cached resources.
+def test_resource_cache_file(latex_on_http_api_url):
+    r = requests.post(
+        latex_on_http_api_url + "/builds/sync",
+        json={
+            "resources": [
+                {
+                    "content": "\\documentclass{article}\n\\begin{document}\nHello World\n\\end{document}",
+                    "main": True,
+                },
+                {
+                    "url": "https://www.ytotech.com/images/ytotech_logo.png",
+                    "path": "logo.png",
+                },
+            ],
+        },
+    )
+    assert r.status_code == 201
+    # Then we can check the cache.
+    r = requests.get(
+        latex_on_http_api_url + "/caches/resources",
+    )
+    assert r.status_code == 200
+    response_payload = r.json()
+    assert response_payload["cached_resources"] == [{"size": 6783}]
+    assert response_payload["total_size"] == 6783
+    # Check cached resources.
+    r = requests.post(
+        latex_on_http_api_url + "/caches/resources/check_cached",
+        json={
+            "resources": [
+                {
+                    "hash": "sha256:683d205d5044f5822c01424189a96e710512f79fe322bbcd8a83a79c8d27cf70",
+                },
+                {
+                    "hash": "sha256:b9797e795d0c45e23671d6037fc77f81a0b4783b",
+                },
+            ],
+        },
+    )
+    assert r.status_code == 200
+    response_payload = r.json()
+    print(response_payload)
+    assert response_payload["resources"] == {
+        "sha256:683d205d5044f5822c01424189a96e710512f79fe322bbcd8a83a79c8d27cf70": {'hit': True},
+        "sha256:b9797e795d0c45e23671d6037fc77f81a0b4783b": {'hit': False},
+    }