Add script for load testing & load tuning

Makefile

@@ -33,7 +33,7 @@ else
 	POFILES_TAR_ARGS+=$(TAR)
 endif
 
-pysrcdirs = internetnl tests interface checks integration_tests
+pysrcdirs = internetnl tests interface checks integration_tests load_tests
 pysrc = $(shell find ${pysrcdirs} -name \*.py)
 
 bin = .venv/bin
@@ -657,3 +657,19 @@ batch-api-add-user docker-compose-batch-api-add-user:
 
 test-%: env=test
 test-up test-down test-build test-stop: test-%: %
+
+locust=docker run --interactive --volume=$$PWD/load_tests:/load_tests --workdir=/load_tests --rm locustio/locust
+locust-file=locustfile.py
+load-test-runtime=10m
+load-test-users=60
+load-test-host=https://dev-docker.internet.nl
+load-test-classes=InternetnlVisitor
+
+load-test:
+	${locust} --locustfile=${locust-file} --headless --host=${load-test-host} --run-time=${load-test-runtime} --users=${load-test-users} ${load-test-classes}
+
+load-test-workers=5
+
+load-test-distributed:
+	for i in $$(seq ${load-test-workers});do ${locust} --locustfile ${locust-file} --worker --master-host=127.0.0.1& done
+	${locust} --locustfile=${locust-file} --headless --host=${load-test-host} --run-time=${load-test-runtime} --users=${load-test-users} ${load-test-classes} --master

docker/defaults.env

@@ -129,7 +129,7 @@ PUBLIC_SUFFIX_LIST_URL=
 INTEGRATION_TESTS=False
 
 # Amount of concurrent worker (green)threads
-WORKER_CONCURRENCY=100
+WORKER_CONCURRENCY=50
 
 # Generate secret key instead of using specified one
 GENERATE_SECRET_KEY=True
@@ -178,3 +178,6 @@ IPV4_IP_MOCK_RESOLVER_PUBLIC=172.42.0.114
 IPV6_IP_MOCK_RESOLVER_PUBLIC=fd00:42:1::114
 
 LOGGING_DRIVER=journald
+
+# limit worker memory so it won't grow into swap
+WORKER_MEMORY_LIMIT=1G

docker/docker-compose.yml

@@ -197,6 +197,10 @@ services:
       dockerfile: docker/Dockerfile
       target: app
     restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: $WORKER_MEMORY_LIMIT
     logging:
       driver: $LOGGING_DRIVER
       options:

docker/host-dist.env

@@ -26,3 +26,8 @@ UNBOUND_PORT_TCP=$IPV4_IP_PUBLIC:53:53/tcp
 UNBOUND_PORT_UDP=$IPV4_IP_PUBLIC:53:53/udp
 UNBOUND_PORT_IPV6_TCP=[$IPV6_IP_PUBLIC]:53:53/tcp
 UNBOUND_PORT_IPV6_UDP=[$IPV6_IP_PUBLIC]:53:53/udp
+
+# modify worker memory limit to allow more concurrent processes
+# WORKER_MEMORY_LIMIT=2G
+# increase concurrent amount of workers
+# WORKER_CONCURRENCY=100

load_tests/locustfile.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+This is a configuration file for Locust load tests.
+
+It provides a balance of different types of visits and tests being performed on the Internet.nl website.
+
+Run it using the following command:
+    locust --headless --users 50 --spawn-rate 50 --run-time 10m --host https://docker.internet.nl
+
+Where in this case it runs headless for 10 minutes simulating 50 simultaneous users.
+"""
+
+import time
+from locust import FastHttpUser, run_single_user, task
+import random
+
+MAX_TEST_DURATION_S = 200
+PROBES_INTERVAL_S = 3
+
+
+class InternetnlStaticVisitor(FastHttpUser):
+    """This class defines an average Internet.nl visitor which always enters through
+    the frontpage and visits some of the subpages."""
+
+    @task(10)
+    def about(self):
+        self.client.get("/")
+        self.client.get("/about/")
+
+    @task(10)
+    def faqs(self):
+        self.client.get("/")
+        self.client.get("/faqs/")
+
+    @task(10)
+    def news(self):
+        self.client.get("/")
+        self.client.get("/news/")
+
+    @task(10)
+    def halloffame(self):
+        self.client.get("/")
+        self.client.get("/halloffame/")
+
+
+class InternetnlTestingVisitor(FastHttpUser):
+    """This class defines an Internet.nl visitor which enters through the frontpage
+    and performs a test against a website."""
+
+    @task(100)
+    def start_test(self):
+        """Run majority of tests against known working targets."""
+
+        test_domain = random.choice(
+            [
+                # randomize url's to prevent server cached results
+                *[f"a{random.randrange(1,9999999)}.test-ns-signed.dev.internet.nl"]
+                * 30,
+            ]
+        )
+
+        self.client.get("/")
+
+        # start test
+        self.client.post("/site/", {"url": test_domain}, name="/site/ (start test)")
+
+        # wait for test to finish
+        timeout = MAX_TEST_DURATION_S
+        while True:
+            if timeout < 0:
+                break
+
+            time.sleep(PROBES_INTERVAL_S)
+            with self.client.get(
+                f"/site/probes/{test_domain}?{time.time()}",
+                catch_response=True,
+                name="/site/probes/[test_domain] (wait for test finish)",
+            ) as probe_response:
+                try:
+                    json = probe_response.json()
+                except json.decoder.JSONDecodeError:
+                    probe_response.failure("Failed to decode probe response")
+                    continue
+
+                if json and type(json) == list and all(x.get("done") for x in json):
+                    break
+
+            timeout -= PROBES_INTERVAL_S
+
+        # get test result
+        with self.client.get(
+            f"/site/{test_domain}/results", catch_response=True, name="/site/[test_domain]/results (test results)"
+        ) as result_response:
+            if "data-resultscore" not in result_response.text:
+                result_response.failure("no score/task did not complete in time")
+
+
+class InternetnlTestingInvalidsVisitor(FastHttpUser):
+    """This class defines an Internet.nl visitor which enters through the frontpage
+    and performs tests against websites with issues."""
+
+    @task(50)
+    def website_test_invalids(self):
+        """Test a number of domain's that have known issues. To simulate non-happy flows. These
+        are not expected to succeed."""
+
+        test_domain = random.choice(
+            [
+                "servfail.nl",
+                "forfun.net",
+                "brokendnssec.net",
+                "expired.badssl.com",
+                "wrong.host.badssl.com",
+                "self-signed.badssl.com",
+                "untrusted-root.badssl.com",
+                "revoked.badssl.com",
+                "pinning-test.badssl.com",
+                "invalid.rpki.isbgpsafeyet.com",
+            ]
+        )
+
+        self.client.get("/")
+
+        # start test
+        self.client.post("/site/", {"url": test_domain}, name="/site/ (start test, invalid)")
+
+        # wait for test to finish
+        timeout = MAX_TEST_DURATION_S
+        while True:
+            if timeout < 0:
+                break
+
+            time.sleep(PROBES_INTERVAL_S)
+            with self.client.get(
+                f"/site/probes/{test_domain}?{time.time()}",
+                catch_response=True,
+                name="/site/probes/[test_invalid_domain] (wait for test finish)",
+            ) as probe_response:
+                try:
+                    json = probe_response.json()
+                except json.decoder.JSONDecodeError:
+                    probe_response.failure("Failed to decode probe response")
+                    continue
+
+                if json and type(json) == list and all(x.get("done") for x in json):
+                    break
+
+            timeout -= PROBES_INTERVAL_S
+
+        # get test result, we won't assert on this as we expect some tests to fail
+        self.client.get(
+            f"/site/{test_domain}/results",
+            catch_response=True,
+            name="/site/[test_invalid_domain]/results (test results)",
+        )
+
+
+class InternetnlVisitor(InternetnlStaticVisitor, InternetnlTestingVisitor, InternetnlTestingInvalidsVisitor):
+    pass
+
+
+# run this file directly for debugging, eg: ./locustfile.py --users 100 --run-time 10m --host https://internet.nl
+if __name__ == "__main__":
+    run_single_user(InternetnlTestingVisitor)