diff --git a/env-manager/src/main/java/io/yupiik/dev/command/List.java b/env-manager/src/main/java/io/yupiik/dev/command/List.java index e526b2ef..d1ca091d 100644 --- a/env-manager/src/main/java/io/yupiik/dev/command/List.java +++ b/env-manager/src/main/java/io/yupiik/dev/command/List.java @@ -19,11 +19,15 @@ import io.yupiik.dev.provider.model.Candidate; import io.yupiik.dev.provider.model.Version; import io.yupiik.fusion.framework.build.api.cli.Command; +import io.yupiik.fusion.framework.build.api.configuration.Property; import io.yupiik.fusion.framework.build.api.configuration.RootConfiguration; import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Logger; +import java.util.stream.Stream; +import static java.util.Locale.ROOT; import static java.util.function.Function.identity; import static java.util.logging.Level.FINEST; import static java.util.stream.Collectors.joining; @@ -32,38 +36,55 @@ @Command(name = "list", description = "List remote (available) distributions.") public class List implements Runnable { private final Logger logger = Logger.getLogger(getClass().getName()); + + private final Conf conf; private final ProviderRegistry registry; public List(final Conf conf, final ProviderRegistry registry) { + this.conf = conf; this.registry = registry; } @Override public void run() { + final var toolFilter = toFilter(conf.tools()); + final var providerFilter = toFilter(conf.providers()); final var collect = registry.providers().stream() + .filter(p -> providerFilter.test(p.name()) || providerFilter.test(p.getClass().getSimpleName().toLowerCase(ROOT))) .map(p -> { try { return p.listTools().stream() - .collect(toMap(identity(), tool -> p.listVersions(tool.tool()))); + .filter(t -> toolFilter.test(t.tool()) || toolFilter.test(t.name())) + .collect(toMap(c -> "[" + p.name() + "] " + c.tool(), tool -> p.listVersions(tool.tool()))); } catch (final RuntimeException re) { logger.log(FINEST, re, re::getMessage); return Map.>of(); } }) .flatMap(m -> m.entrySet().stream()) - .map(e -> "- " + e.getKey().tool() + ":" + (e.getValue().isEmpty() ? - " no version available" : - e.getValue().stream() - .sorted((a, b) -> -a.compareTo(b)) - .map(v -> "-- " + v.version()) - .collect(joining("\n", "\n", "\n")))) + .filter(Predicate.not(m -> m.getValue().isEmpty())) + .map(e -> "- " + e.getKey() + ":" + e.getValue().stream() + .sorted((a, b) -> -a.compareTo(b)) + .map(v -> "-- " + v.version()) + .collect(joining("\n", "\n", "\n"))) .sorted() .collect(joining("\n")); logger.info(() -> collect.isBlank() ? "No distribution available." : collect); } + private Predicate toFilter(final String values) { + return values == null || values.isBlank() ? + t -> true : + Stream.of(values.split(",")) + .map(String::strip) + .filter(Predicate.not(String::isBlank)) + .map(t -> (Predicate) t::equals) + .reduce(t -> false, Predicate::or); + } + @RootConfiguration("list") - public record Conf(/* no option yet */) { + public record Conf(@Property(documentation = "List of tools to list (comma separated).") String tools, + @Property(documentation = "List of providers to use (comma separated).") String providers) { } } diff --git a/env-manager/src/main/java/io/yupiik/dev/provider/zulu/ZuluCdnClient.java b/env-manager/src/main/java/io/yupiik/dev/provider/zulu/ZuluCdnClient.java index 78dbc35f..d8d37b6b 100644 --- a/env-manager/src/main/java/io/yupiik/dev/provider/zulu/ZuluCdnClient.java +++ b/env-manager/src/main/java/io/yupiik/dev/provider/zulu/ZuluCdnClient.java @@ -21,6 +21,7 @@ import io.yupiik.dev.provider.model.Version; import io.yupiik.dev.shared.Archives; import io.yupiik.dev.shared.Os; +import io.yupiik.dev.shared.http.Cache; import io.yupiik.dev.shared.http.YemHttpClient; import io.yupiik.fusion.framework.api.scope.DefaultScoped; @@ -38,6 +39,7 @@ import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; +import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toMap; @DefaultScoped @@ -45,14 +47,17 @@ public class ZuluCdnClient implements Provider { private final String suffix; private final Archives archives; private final YemHttpClient client; + private final Cache cache; private final URI base; private final Path local; private final boolean enabled; private final boolean preferJre; - public ZuluCdnClient(final YemHttpClient client, final ZuluCdnConfiguration configuration, final Os os, final Archives archives) { + public ZuluCdnClient(final YemHttpClient client, final ZuluCdnConfiguration configuration, final Os os, final Archives archives, + final Cache cache) { this.client = client; this.archives = archives; + this.cache = cache; this.base = URI.create(configuration.base()); this.local = Path.of(configuration.local()); this.enabled = configuration.enabled(); @@ -182,9 +187,23 @@ public List listVersions(final String tool) { if (!enabled) { return List.of(); } + + // here the payload is >5M so we can let the client cache it but saving the result will be way more efficient on the JSON side + final var entry = cache.lookup(base.toASCIIString()); + if (entry != null && entry.hit() != null) { + return parseVersions(entry.hit().payload()); + } + final var res = client.send(HttpRequest.newBuilder().GET().uri(base).build()); ensure200(res); - return parseVersions(res.body()); + + final var filtered = parseVersions(res.body()); + if (entry != null) { + cache.save(entry.key(), Map.of(), filtered.stream() + .map(it -> "zulu" + it.identifier() + '-' + suffix + "") + .collect(joining("\n", "", "\n"))); + } + return filtered; } private void ensure200(final HttpResponse res) { diff --git a/env-manager/src/main/java/io/yupiik/dev/shared/http/Cache.java b/env-manager/src/main/java/io/yupiik/dev/shared/http/Cache.java new file mode 100644 index 00000000..4cf0376f --- /dev/null +++ b/env-manager/src/main/java/io/yupiik/dev/shared/http/Cache.java @@ -0,0 +1,96 @@ +package io.yupiik.dev.shared.http; + +import io.yupiik.fusion.framework.api.scope.ApplicationScoped; +import io.yupiik.fusion.framework.build.api.json.JsonModel; +import io.yupiik.fusion.json.JsonMapper; + +import java.io.IOException; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Base64; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.Clock.systemDefaultZone; +import static java.util.stream.Collectors.toMap; + +@ApplicationScoped +public class Cache { + private final Path cache; + private final long cacheValidity; + private final JsonMapper jsonMapper; + private final Clock clock; + + protected Cache() { + this.cache = null; + this.jsonMapper = null; + this.clock = null; + this.cacheValidity = 0L; + } + + public Cache(final HttpConfiguration configuration, final JsonMapper jsonMapper) { + this.jsonMapper = jsonMapper; + try { + this.cache = configuration.isCacheEnabled() ? null : Files.createDirectories(Path.of(configuration.cache())); + } catch (final IOException e) { + throw new IllegalArgumentException("Can't create HTTP cache directory : '" + configuration.cache() + "', adjust --http-cache parameter"); + } + this.cacheValidity = configuration.cacheValidity(); + this.clock = systemDefaultZone(); + } + + public void save(final Path key, final HttpResponse result) { + save( + key, + result.headers().map().entrySet().stream() + .filter(it -> !"content-encoding".equalsIgnoreCase(it.getKey())) + .collect(toMap(Map.Entry::getKey, l -> String.join(",", l.getValue()))), + result.body()); + } + + public void save(final Path cacheLocation, final Map headers, final String body) { + final var cachedData = jsonMapper.toString(new Response(headers, body, clock.instant().plusMillis(cacheValidity).toEpochMilli())); + try { + Files.writeString(cacheLocation, cachedData); + } catch (final IOException e) { + try { + Files.deleteIfExists(cacheLocation); + } catch (final IOException ex) { + // no-op + } + } + } + + public CachedEntry lookup(final HttpRequest request) { + return lookup(request.uri().toASCIIString()); + } + + public CachedEntry lookup(final String key) { + if (cache == null) { + return null; + } + + final var cacheLocation = cache.resolve(Base64.getUrlEncoder().withoutPadding().encodeToString(key.getBytes(UTF_8))); + if (Files.exists(cacheLocation)) { + try { + final var cached = jsonMapper.fromString(Response.class, Files.readString(cacheLocation)); + if (cached.validUntil() > clock.instant().toEpochMilli()) { + return new CachedEntry(cacheLocation, cached); + } + } catch (final IOException e) { + throw new IllegalStateException(e); + } + } + return new CachedEntry(cacheLocation, null); + } + + public record CachedEntry(Path key, Response hit) { + } + + @JsonModel + public record Response(Map headers, String payload, long validUntil) { + } +} diff --git a/env-manager/src/main/java/io/yupiik/dev/shared/http/HttpConfiguration.java b/env-manager/src/main/java/io/yupiik/dev/shared/http/HttpConfiguration.java index a33c29d4..e88ae826 100644 --- a/env-manager/src/main/java/io/yupiik/dev/shared/http/HttpConfiguration.java +++ b/env-manager/src/main/java/io/yupiik/dev/shared/http/HttpConfiguration.java @@ -26,4 +26,7 @@ public record HttpConfiguration( @Property(defaultValue = "86_400_000L", documentation = "Cache validity of requests (1 day by default) in milliseconds. A negative or zero value will disable cache.") long cacheValidity, @Property(defaultValue = "System.getProperty(\"user.home\", \"\") + \"/.yupiik/yem/cache/http\"", documentation = "Where to cache slow updates (version fetching). `none` will disable cache.") String cache ) { + public boolean isCacheEnabled() { + return "none".equals(cache()) || cacheValidity() <= 0; + } } diff --git a/env-manager/src/main/java/io/yupiik/dev/shared/http/YemHttpClient.java b/env-manager/src/main/java/io/yupiik/dev/shared/http/YemHttpClient.java index a04a1bdc..2cf4d235 100644 --- a/env-manager/src/main/java/io/yupiik/dev/shared/http/YemHttpClient.java +++ b/env-manager/src/main/java/io/yupiik/dev/shared/http/YemHttpClient.java @@ -17,7 +17,6 @@ import io.yupiik.dev.provider.Provider; import io.yupiik.fusion.framework.api.scope.ApplicationScoped; -import io.yupiik.fusion.framework.build.api.json.JsonModel; import io.yupiik.fusion.httpclient.core.ExtendedHttpClient; import io.yupiik.fusion.httpclient.core.ExtendedHttpClientConfiguration; import io.yupiik.fusion.httpclient.core.listener.RequestListener; @@ -25,7 +24,6 @@ import io.yupiik.fusion.httpclient.core.listener.impl.ExchangeLogger; import io.yupiik.fusion.httpclient.core.request.UnlockedHttpRequest; import io.yupiik.fusion.httpclient.core.response.StaticHttpResponse; -import io.yupiik.fusion.json.JsonMapper; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -37,10 +35,8 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Clock; import java.time.Duration; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import java.util.Map; import java.util.concurrent.Flow; @@ -58,21 +54,16 @@ @ApplicationScoped public class YemHttpClient implements AutoCloseable { private final Logger logger = Logger.getLogger(getClass().getName()); + private final ExtendedHttpClient client; - private final Path cache; - private final JsonMapper jsonMapper; - private final Clock clock; - private final long cacheValidity; + private final Cache cache; protected YemHttpClient() { // for subclassing proxy this.client = null; this.cache = null; - this.jsonMapper = null; - this.clock = null; - this.cacheValidity = 0L; } - public YemHttpClient(final HttpConfiguration configuration, final JsonMapper jsonMapper) { + public YemHttpClient(final HttpConfiguration configuration, final Cache cache) { final var listeners = new ArrayList>(); if (configuration.log()) { listeners.add((new ExchangeLogger( @@ -111,17 +102,8 @@ public RequestListener.State before(final long count, final HttpRequest re .build()) .setRequestListeners(listeners); - try { - this.cache = "none".equals(configuration.cache()) || configuration.cacheValidity() <= 0 ? - null : - Files.createDirectories(Path.of(configuration.cache())); - } catch (final IOException e) { - throw new IllegalArgumentException("Can't create HTTP cache directory : '" + configuration.cache() + "', adjust --http-cache parameter"); - } - this.cacheValidity = configuration.cacheValidity(); + this.cache = cache; this.client = new ExtendedHttpClient(conf); - this.jsonMapper = jsonMapper; - this.clock = systemDefaultZone(); } @Override @@ -160,27 +142,15 @@ public HttpResponse getFile(final HttpRequest request, final Path target, } public HttpResponse send(final HttpRequest request) { - final Path cacheLocation; - if (cache != null) { - cacheLocation = cache.resolve(Base64.getUrlEncoder().withoutPadding().encodeToString(request.uri().toASCIIString().getBytes(UTF_8))); - if (Files.exists(cacheLocation)) { - try { - final var cached = jsonMapper.fromString(Response.class, Files.readString(cacheLocation)); - if (cached.validUntil() > clock.instant().toEpochMilli()) { - return new StaticHttpResponse<>( - request, request.uri(), HTTP_1_1, 200, - HttpHeaders.of( - cached.headers().entrySet().stream() - .collect(toMap(Map.Entry::getKey, e -> List.of(e.getValue()))), - (a, b) -> true), - cached.payload()); - } - } catch (final IOException e) { - throw new IllegalStateException(e); - } - } - } else { - cacheLocation = null; + final var entry = cache.lookup(request); + if (entry != null && entry.hit() != null) { + return new StaticHttpResponse<>( + request, request.uri(), HTTP_1_1, 200, + HttpHeaders.of( + entry.hit().headers().entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> List.of(e.getValue()))), + (a, b) -> true), + entry.hit().payload()); } logger.finest(() -> "Calling " + request); @@ -199,22 +169,8 @@ public HttpResponse send(final HttpRequest request) { result = result != null ? result : new StaticHttpResponse<>( response.request(), response.uri(), response.version(), response.statusCode(), response.headers(), new String(response.body(), UTF_8)); - if (cacheLocation != null && result.statusCode() == 200) { - final var cachedData = jsonMapper.toString(new Response( - result.headers().map().entrySet().stream() - .filter(it -> !"content-encoding".equalsIgnoreCase(it.getKey())) - .collect(toMap(Map.Entry::getKey, l -> String.join(",", l.getValue()))), - result.body(), - clock.instant().plusMillis(cacheValidity).toEpochMilli())); - try { - Files.writeString(cacheLocation, cachedData); - } catch (final IOException e) { - try { - Files.deleteIfExists(cacheLocation); - } catch (final IOException ex) { - // no-op - } - } + if (entry != null && result.statusCode() == 200) { + cache.save(entry.key(), result); } return result; } catch (final InterruptedException var4) { @@ -282,8 +238,4 @@ public void onComplete() { throw new IllegalStateException(var5); } } - - @JsonModel - public record Response(Map headers, String payload, long validUntil) { - } } diff --git a/env-manager/src/test/java/io/yupiik/dev/command/CommandsTest.java b/env-manager/src/test/java/io/yupiik/dev/command/CommandsTest.java index d453f7fd..3692fb8c 100644 --- a/env-manager/src/test/java/io/yupiik/dev/command/CommandsTest.java +++ b/env-manager/src/test/java/io/yupiik/dev/command/CommandsTest.java @@ -70,7 +70,7 @@ void simplifiedOptions(@TempDir final Path work, final URI uri) throws IOExcepti @Test void list(@TempDir final Path work, final URI uri) { assertEquals(""" - - java: + - [zulu] java: -- 21.0.2""", captureOutput(work, uri, "list")); } diff --git a/env-manager/src/test/java/io/yupiik/dev/provider/zulu/ZuluCdnClientTest.java b/env-manager/src/test/java/io/yupiik/dev/provider/zulu/ZuluCdnClientTest.java index de4c4c47..a3724077 100644 --- a/env-manager/src/test/java/io/yupiik/dev/provider/zulu/ZuluCdnClientTest.java +++ b/env-manager/src/test/java/io/yupiik/dev/provider/zulu/ZuluCdnClientTest.java @@ -19,6 +19,8 @@ import io.yupiik.dev.provider.model.Version; import io.yupiik.dev.shared.Archives; import io.yupiik.dev.shared.Os; +import io.yupiik.dev.shared.http.Cache; +import io.yupiik.dev.shared.http.HttpConfiguration; import io.yupiik.dev.shared.http.YemHttpClient; import io.yupiik.dev.test.Mock; import org.junit.jupiter.api.Test; @@ -174,6 +176,9 @@ void delete(final URI uri, @TempDir final Path work, final YemHttpClient client) } private ZuluCdnClient newProvider(final URI uri, final YemHttpClient client, final Path local) { - return new ZuluCdnClient(client, new ZuluCdnConfiguration(true, true, uri.toASCIIString(), "linux_x64.zip", local.toString()), new Os(), new Archives()); + return new ZuluCdnClient( + client, + new ZuluCdnConfiguration(true, true, uri.toASCIIString(), "linux_x64.zip", local.toString()), + new Os(), new Archives(), new Cache(new HttpConfiguration(false, 30_000L, 30_000L, 0L, "none"), null)); } } diff --git a/env-manager/src/test/java/io/yupiik/dev/test/HttpMockExtension.java b/env-manager/src/test/java/io/yupiik/dev/test/HttpMockExtension.java index a5696e26..ac069507 100644 --- a/env-manager/src/test/java/io/yupiik/dev/test/HttpMockExtension.java +++ b/env-manager/src/test/java/io/yupiik/dev/test/HttpMockExtension.java @@ -16,6 +16,7 @@ package io.yupiik.dev.test; import com.sun.net.httpserver.HttpServer; +import io.yupiik.dev.shared.http.Cache; import io.yupiik.dev.shared.http.HttpConfiguration; import io.yupiik.dev.shared.http.YemHttpClient; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -92,7 +93,8 @@ public Object resolveParameter(final ParameterContext parameterContext, final Ex return URI.create("http://localhost:" + context.getStore(NAMESPACE).get(HttpServer.class, HttpServer.class).getAddress().getPort() + "/2/"); } if (YemHttpClient.class == parameterContext.getParameter().getType()) { - return new YemHttpClient(new HttpConfiguration(false, 30_000L, 30_000L, 0, "none"), null); + final var configuration = new HttpConfiguration(false, 30_000L, 30_000L, 0, "none"); + return new YemHttpClient(configuration, new Cache(configuration, null)); } throw new ParameterResolutionException("Can't resolve " + parameterContext.getParameter().getType()); }