parse size string instead of bytes

homcc/server/cache.py

@@ -55,12 +55,12 @@ def _evict_oldest(self):
         Evicts the oldest entry from the cache.
         Note: The caller of this method has to ensure that the cache is locked.
         """
-        oldest_hash, _ = self.cache.popitem(last=False)
-        oldest_path = self._get_cache_file_path(oldest_hash)
+        oldest_hash, oldest_path_str = self.cache.popitem(last=False)
+        oldest_path = Path(oldest_path_str)
         oldest_size = oldest_path.stat().st_size
 
         try:
-            Path.unlink(oldest_path, missing_ok=False)
+            oldest_path.unlink(missing_ok=False)
         except FileNotFoundError:
             logger.error(
                 "Tried to evict cache entry with hash '%s', but corresponding cache file at '%s' did not exist.",

homcc/server/main.py

@@ -45,6 +45,7 @@
     ServerConfig,
     parse_cli_args,
     parse_config,
+    size_string_to_bytes,
 )
 from homcc.server.server import (  # pylint: disable=wrong-import-position
     start_server,
@@ -103,9 +104,9 @@ def main():
     if (address := homccd_args_dict["listen"]) is not None:
         homccd_config.address = address
 
-    # MAX_DEPENDENCY_CACHE_SIZE_BYTES
-    if (max_dependency_cache_size_bytes := homccd_args_dict["max_dependency_cache_size_bytes"]) is not None:
-        homccd_config.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes
+    # MAX_DEPENDENCY_CACHE_SIZE
+    if (max_dependency_cache_size := homccd_args_dict["max_dependency_cache_size"]) is not None:
+        homccd_config.max_dependency_cache_size_bytes = size_string_to_bytes(max_dependency_cache_size)
 
     # provide additional DEBUG information
     logger.debug(

homcc/server/parsing.py

@@ -27,6 +27,23 @@ def mib_to_bytes(mb: int) -> int:
     return mb * 1024**2
 
 
+def gib_to_bytes(gb: int) -> int:
+    return gb * 1024**3
+
+
+def size_string_to_bytes(size_string: str) -> int:
+    """Converts e.g. 100M or 1G to bytes. Only supports M (Mebibyte) and G (Gibibyte)"""
+    unit = size_string[-1]
+    amount = size_string[:-1]
+
+    if unit == "M":
+        return mib_to_bytes(int(amount))
+    elif unit == "G":
+        return gib_to_bytes(int(amount))
+
+    raise ArgumentTypeError(f"Invalid size string: '{size_string}'. Specify either M (Mebibyte) or G (Gibibyte).")
+
+
 HOMCC_SERVER_CONFIG_SECTION: str = "homccd"
 
 DEFAULT_ADDRESS: str = "0.0.0.0"
@@ -36,7 +53,7 @@ def mib_to_bytes(mb: int) -> int:
     or os.cpu_count()  # total number of physical CPUs on the machine
     or -1  # fallback error value
 )
-DEFAULT_MAX_CACHE_SIZE_BYTES: int = mib_to_bytes(10000)
+DEFAULT_MAX_CACHE_SIZE_BYTES: int = gib_to_bytes(10)
 
 
 class ShowVersion(Action):
@@ -80,7 +97,7 @@ class EnvironmentVariables:
         HOMCCD_ADDRESS_ENV_VAR: ClassVar[str] = "HOMCCD_ADDRESS"
         HOMCCD_LOG_LEVEL_ENV_VAR: ClassVar[str] = "HOMCCD_LOG_LEVEL"
         HOMCCD_VERBOSE_ENV_VAR: ClassVar[str] = "HOMCCD_VERBOSE"
-        HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES: ClassVar[str] = "HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES"
+        HOMCCD_MAX_DEPENDENCY_CACHE_SIZE: ClassVar[str] = "HOMCCD_MAX_DEPENDENCY_CACHE_SIZE"
 
         @classmethod
         def __iter__(cls) -> Iterator[str]:
@@ -90,7 +107,7 @@ def __iter__(cls) -> Iterator[str]:
                 cls.HOMCCD_ADDRESS_ENV_VAR,
                 cls.HOMCCD_LOG_LEVEL_ENV_VAR,
                 cls.HOMCCD_VERBOSE_ENV_VAR,
-                cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES,
+                cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE,
             )
 
         @classmethod
@@ -121,9 +138,9 @@ def get_verbose(cls) -> Optional[bool]:
             return None
 
         @classmethod
-        def get_max_dependency_cache_size_bytes(cls) -> Optional[int]:
-            if max_dependency_cache_size_bytes := os.getenv(cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE_BYTES):
-                return int(max_dependency_cache_size_bytes)
+        def get_max_dependency_cache_size(cls) -> Optional[int]:
+            if max_dependency_cache_size := os.getenv(cls.HOMCCD_MAX_DEPENDENCY_CACHE_SIZE):
+                return size_string_to_bytes(max_dependency_cache_size)
 
             return None
 
@@ -157,7 +174,9 @@ def __init__(
         verbose = self.EnvironmentVariables.get_verbose() or verbose
         self.verbose = verbose is not None and verbose
 
-        self.max_dependency_cache_size_bytes = max_dependency_cache_size_bytes
+        self.max_dependency_cache_size_bytes = (
+            self.EnvironmentVariables.get_max_dependency_cache_size() or max_dependency_cache_size_bytes
+        )
 
     @classmethod
     def empty(cls):
@@ -170,7 +189,7 @@ def from_config_section(cls, files: List[str], homccd_config: SectionProxy) -> S
         address: Optional[str] = homccd_config.get("address")
         log_level: Optional[str] = homccd_config.get("log_level")
         verbose: Optional[bool] = homccd_config.getboolean("verbose")
-        max_dependency_cache_size_bytes: Optional[int] = homccd_config.getint("max_dependency_cache_size_bytes")
+        max_dependency_cache_size: Optional[str] = homccd_config.get("max_dependency_cache_size")
 
         return ServerConfig(
             files=files,
@@ -179,7 +198,9 @@ def from_config_section(cls, files: List[str], homccd_config: SectionProxy) -> S
             address=address,
             log_level=log_level,
             verbose=verbose,
-            max_dependency_cache_size_bytes=max_dependency_cache_size_bytes,
+            max_dependency_cache_size_bytes=None
+            if max_dependency_cache_size is None
+            else size_string_to_bytes(max_dependency_cache_size),
         )
 
     def __str__(self):
@@ -210,13 +231,8 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int:
 
         raise ArgumentTypeError(f"LIMIT must be more than {minimum}")
 
-    def max_dependency_cache_size_bytes(value: Union[int, str]) -> int:
-        value = int(value)
-
-        if value <= 0:
-            raise ArgumentTypeError("Maximum dependency cache size must be larger than 0.")
-
-        return value
+    def max_dependency_cache_size_bytes(value: str) -> int:
+        return size_string_to_bytes(value)
 
     general_options_group = parser.add_argument_group("Options")
     networking_group = parser.add_argument_group(" Networking")
@@ -244,11 +260,12 @@ def max_dependency_cache_size_bytes(value: Union[int, str]) -> int:
         help="enforce that only configurations provided via the CLI are used",
     )
     general_options_group.add_argument(
-        "--max-dependency-cache-size-bytes",
+        "--max-dependency-cache-size",
         required=False,
-        metavar="BYTES",
+        metavar="SIZE",
         type=max_dependency_cache_size_bytes,
-        help=f"The maximum cache size for the dependency cache in bytes. Default: {DEFAULT_MAX_CACHE_SIZE_BYTES} bytes",
+        help=f"""The maximum cache size for the dependency cache. Expects a size string, e.g. 100M or 10G.
+         Default: {DEFAULT_MAX_CACHE_SIZE_BYTES} bytes""",
     )
 
     # networking

tests/server/parsing_test.py

@@ -4,6 +4,7 @@
 
 """ Tests for client/compilation.py"""
 import os
+from argparse import ArgumentTypeError
 from pathlib import Path
 from typing import List
 
@@ -12,7 +13,13 @@
 
 from homcc import server
 from homcc.common.parsing import HOMCC_CONFIG_FILENAME
-from homcc.server.parsing import ServerConfig, parse_cli_args, parse_config
+from homcc.server.parsing import (
+    ServerConfig,
+    gib_to_bytes,
+    parse_cli_args,
+    parse_config,
+    size_string_to_bytes,
+)
 
 
 class TestParsingConfig:
@@ -28,17 +35,14 @@ class TestParsingConfig:
         "AdDrEsS=0.0.0.0",
         "LOG_LEVEL=DEBUG",
         "verbose=TRUE",
+        "max_dependency_cache_size=10G",
         # the following configs should be ignored
         "[homcc]",
         "LOG_LEVEL=INFO",
         "verbose=FALSE",
     ]
 
-    config_overwrite: List[str] = [
-        "[homccd]",
-        "LOG_LEVEL=INFO",
-        "verbose=FALSE",
-    ]
+    config_overwrite: List[str] = ["[homccd]", "LOG_LEVEL=INFO", "verbose=FALSE", "max_dependency_cache_size=1G"]
 
     def test_version(self, capfd: CaptureFixture):
         with pytest.raises(SystemExit) as sys_exit:
@@ -54,15 +58,19 @@ def test_parse_config_file(self, tmp_path: Path):
         tmp_config_file: Path = tmp_path / HOMCC_CONFIG_FILENAME
         tmp_config_file.write_text("\n".join(self.config))
 
-        assert parse_config([tmp_config_file]) == ServerConfig(
+        parsed_config = parse_config([tmp_config_file])
+        expected_config = ServerConfig(
             files=[str(tmp_config_file.absolute())],
             limit=42,
             port=3126,
             address="0.0.0.0",
+            max_dependency_cache_size_bytes=gib_to_bytes(10),
             log_level="DEBUG",
             verbose=True,
         )
 
+        assert parsed_config == expected_config
+
     def test_parse_multiple_config_files(self, tmp_path: Path):
         tmp_config_file: Path = tmp_path / HOMCC_CONFIG_FILENAME
         tmp_config_file.write_text("\n".join(self.config))
@@ -75,6 +83,19 @@ def test_parse_multiple_config_files(self, tmp_path: Path):
             limit=42,
             port=3126,
             address="0.0.0.0",
+            max_dependency_cache_size_bytes=gib_to_bytes(1),
             log_level="INFO",
             verbose=False,
         )
+
+    def test_size_string_conversions(self):
+        assert size_string_to_bytes("1M") == 1048576
+        assert size_string_to_bytes("100M") == 104857600
+        assert size_string_to_bytes("555M") == 581959680
+
+        assert size_string_to_bytes("1G") == 1073741824
+        assert size_string_to_bytes("100G") == 107374182400
+        assert size_string_to_bytes("123G") == 132070244352
+
+        with pytest.raises(ArgumentTypeError):
+            size_string_to_bytes("123")