Skip to content

Commit

Permalink
[env manager] more efficient zulu cache and add filters to list command
Browse files Browse the repository at this point in the history
  • Loading branch information
rmannibucau committed Feb 6, 2024
1 parent eb222a4 commit 3279782
Show file tree
Hide file tree
Showing 8 changed files with 174 additions and 76 deletions.
37 changes: 29 additions & 8 deletions env-manager/src/main/java/io/yupiik/dev/command/List.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@
import io.yupiik.dev.provider.model.Candidate;
import io.yupiik.dev.provider.model.Version;
import io.yupiik.fusion.framework.build.api.cli.Command;
import io.yupiik.fusion.framework.build.api.configuration.Property;
import io.yupiik.fusion.framework.build.api.configuration.RootConfiguration;

import java.util.Map;
import java.util.function.Predicate;
import java.util.logging.Logger;
import java.util.stream.Stream;

import static java.util.Locale.ROOT;
import static java.util.function.Function.identity;
import static java.util.logging.Level.FINEST;
import static java.util.stream.Collectors.joining;
Expand All @@ -32,38 +36,55 @@
@Command(name = "list", description = "List remote (available) distributions.")
public class List implements Runnable {
private final Logger logger = Logger.getLogger(getClass().getName());

private final Conf conf;
private final ProviderRegistry registry;

public List(final Conf conf,
final ProviderRegistry registry) {
this.conf = conf;
this.registry = registry;
}

@Override
public void run() {
final var toolFilter = toFilter(conf.tools());
final var providerFilter = toFilter(conf.providers());
final var collect = registry.providers().stream()
.filter(p -> providerFilter.test(p.name()) || providerFilter.test(p.getClass().getSimpleName().toLowerCase(ROOT)))
.map(p -> {
try {
return p.listTools().stream()
.collect(toMap(identity(), tool -> p.listVersions(tool.tool())));
.filter(t -> toolFilter.test(t.tool()) || toolFilter.test(t.name()))
.collect(toMap(c -> "[" + p.name() + "] " + c.tool(), tool -> p.listVersions(tool.tool())));
} catch (final RuntimeException re) {
logger.log(FINEST, re, re::getMessage);
return Map.<Candidate, java.util.List<Version>>of();
}
})
.flatMap(m -> m.entrySet().stream())
.map(e -> "- " + e.getKey().tool() + ":" + (e.getValue().isEmpty() ?
" no version available" :
e.getValue().stream()
.sorted((a, b) -> -a.compareTo(b))
.map(v -> "-- " + v.version())
.collect(joining("\n", "\n", "\n"))))
.filter(Predicate.not(m -> m.getValue().isEmpty()))
.map(e -> "- " + e.getKey() + ":" + e.getValue().stream()
.sorted((a, b) -> -a.compareTo(b))
.map(v -> "-- " + v.version())
.collect(joining("\n", "\n", "\n")))
.sorted()
.collect(joining("\n"));
logger.info(() -> collect.isBlank() ? "No distribution available." : collect);
}

private Predicate<String> toFilter(final String values) {
return values == null || values.isBlank() ?
t -> true :
Stream.of(values.split(","))
.map(String::strip)
.filter(Predicate.not(String::isBlank))
.map(t -> (Predicate<String>) t::equals)
.reduce(t -> false, Predicate::or);
}

@RootConfiguration("list")
public record Conf(/* no option yet */) {
public record Conf(@Property(documentation = "List of tools to list (comma separated).") String tools,
@Property(documentation = "List of providers to use (comma separated).") String providers) {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import io.yupiik.dev.provider.model.Version;
import io.yupiik.dev.shared.Archives;
import io.yupiik.dev.shared.Os;
import io.yupiik.dev.shared.http.Cache;
import io.yupiik.dev.shared.http.YemHttpClient;
import io.yupiik.fusion.framework.api.scope.DefaultScoped;

Expand All @@ -38,21 +39,25 @@

import static java.util.Optional.ofNullable;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toMap;

@DefaultScoped
public class ZuluCdnClient implements Provider {
private final String suffix;
private final Archives archives;
private final YemHttpClient client;
private final Cache cache;
private final URI base;
private final Path local;
private final boolean enabled;
private final boolean preferJre;

public ZuluCdnClient(final YemHttpClient client, final ZuluCdnConfiguration configuration, final Os os, final Archives archives) {
public ZuluCdnClient(final YemHttpClient client, final ZuluCdnConfiguration configuration, final Os os, final Archives archives,
final Cache cache) {
this.client = client;
this.archives = archives;
this.cache = cache;
this.base = URI.create(configuration.base());
this.local = Path.of(configuration.local());
this.enabled = configuration.enabled();
Expand Down Expand Up @@ -182,9 +187,23 @@ public List<Version> listVersions(final String tool) {
if (!enabled) {
return List.of();
}

// here the payload is >5M so we can let the client cache it but saving the result will be way more efficient on the JSON side
final var entry = cache.lookup(base.toASCIIString());
if (entry != null && entry.hit() != null) {
return parseVersions(entry.hit().payload());
}

final var res = client.send(HttpRequest.newBuilder().GET().uri(base).build());
ensure200(res);
return parseVersions(res.body());

final var filtered = parseVersions(res.body());
if (entry != null) {
cache.save(entry.key(), Map.of(), filtered.stream()
.map(it -> "<a href=\"/zulu/bin/zulu" + it.identifier() + '-' + suffix + "\">zulu" + it.identifier() + '-' + suffix + "</a>")
.collect(joining("\n", "", "\n")));
}
return filtered;
}

private void ensure200(final HttpResponse<?> res) {
Expand Down
96 changes: 96 additions & 0 deletions env-manager/src/main/java/io/yupiik/dev/shared/http/Cache.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package io.yupiik.dev.shared.http;

import io.yupiik.fusion.framework.api.scope.ApplicationScoped;
import io.yupiik.fusion.framework.build.api.json.JsonModel;
import io.yupiik.fusion.json.JsonMapper;

import java.io.IOException;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Clock;
import java.util.Base64;
import java.util.Map;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.time.Clock.systemDefaultZone;
import static java.util.stream.Collectors.toMap;

@ApplicationScoped
public class Cache {
private final Path cache;
private final long cacheValidity;
private final JsonMapper jsonMapper;
private final Clock clock;

protected Cache() {
this.cache = null;
this.jsonMapper = null;
this.clock = null;
this.cacheValidity = 0L;
}

public Cache(final HttpConfiguration configuration, final JsonMapper jsonMapper) {
this.jsonMapper = jsonMapper;
try {
this.cache = configuration.isCacheEnabled() ? null : Files.createDirectories(Path.of(configuration.cache()));
} catch (final IOException e) {
throw new IllegalArgumentException("Can't create HTTP cache directory : '" + configuration.cache() + "', adjust --http-cache parameter");
}
this.cacheValidity = configuration.cacheValidity();
this.clock = systemDefaultZone();
}

public void save(final Path key, final HttpResponse<String> result) {
save(
key,
result.headers().map().entrySet().stream()
.filter(it -> !"content-encoding".equalsIgnoreCase(it.getKey()))
.collect(toMap(Map.Entry::getKey, l -> String.join(",", l.getValue()))),
result.body());
}

public void save(final Path cacheLocation, final Map<String, String> headers, final String body) {
final var cachedData = jsonMapper.toString(new Response(headers, body, clock.instant().plusMillis(cacheValidity).toEpochMilli()));
try {
Files.writeString(cacheLocation, cachedData);
} catch (final IOException e) {
try {
Files.deleteIfExists(cacheLocation);
} catch (final IOException ex) {
// no-op
}
}
}

public CachedEntry lookup(final HttpRequest request) {
return lookup(request.uri().toASCIIString());
}

public CachedEntry lookup(final String key) {
if (cache == null) {
return null;
}

final var cacheLocation = cache.resolve(Base64.getUrlEncoder().withoutPadding().encodeToString(key.getBytes(UTF_8)));
if (Files.exists(cacheLocation)) {
try {
final var cached = jsonMapper.fromString(Response.class, Files.readString(cacheLocation));
if (cached.validUntil() > clock.instant().toEpochMilli()) {
return new CachedEntry(cacheLocation, cached);
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
}
return new CachedEntry(cacheLocation, null);
}

public record CachedEntry(Path key, Response hit) {
}

@JsonModel
public record Response(Map<String, String> headers, String payload, long validUntil) {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,7 @@ public record HttpConfiguration(
@Property(defaultValue = "86_400_000L", documentation = "Cache validity of requests (1 day by default) in milliseconds. A negative or zero value will disable cache.") long cacheValidity,
@Property(defaultValue = "System.getProperty(\"user.home\", \"\") + \"/.yupiik/yem/cache/http\"", documentation = "Where to cache slow updates (version fetching). `none` will disable cache.") String cache
) {
public boolean isCacheEnabled() {
return "none".equals(cache()) || cacheValidity() <= 0;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,13 @@

import io.yupiik.dev.provider.Provider;
import io.yupiik.fusion.framework.api.scope.ApplicationScoped;
import io.yupiik.fusion.framework.build.api.json.JsonModel;
import io.yupiik.fusion.httpclient.core.ExtendedHttpClient;
import io.yupiik.fusion.httpclient.core.ExtendedHttpClientConfiguration;
import io.yupiik.fusion.httpclient.core.listener.RequestListener;
import io.yupiik.fusion.httpclient.core.listener.impl.DefaultTimeout;
import io.yupiik.fusion.httpclient.core.listener.impl.ExchangeLogger;
import io.yupiik.fusion.httpclient.core.request.UnlockedHttpRequest;
import io.yupiik.fusion.httpclient.core.response.StaticHttpResponse;
import io.yupiik.fusion.json.JsonMapper;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
Expand All @@ -37,10 +35,8 @@
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Flow;
Expand All @@ -58,21 +54,16 @@
@ApplicationScoped
public class YemHttpClient implements AutoCloseable {
private final Logger logger = Logger.getLogger(getClass().getName());

private final ExtendedHttpClient client;
private final Path cache;
private final JsonMapper jsonMapper;
private final Clock clock;
private final long cacheValidity;
private final Cache cache;

protected YemHttpClient() { // for subclassing proxy
this.client = null;
this.cache = null;
this.jsonMapper = null;
this.clock = null;
this.cacheValidity = 0L;
}

public YemHttpClient(final HttpConfiguration configuration, final JsonMapper jsonMapper) {
public YemHttpClient(final HttpConfiguration configuration, final Cache cache) {
final var listeners = new ArrayList<RequestListener<?>>();
if (configuration.log()) {
listeners.add((new ExchangeLogger(
Expand Down Expand Up @@ -111,17 +102,8 @@ public RequestListener.State<Void> before(final long count, final HttpRequest re
.build())
.setRequestListeners(listeners);

try {
this.cache = "none".equals(configuration.cache()) || configuration.cacheValidity() <= 0 ?
null :
Files.createDirectories(Path.of(configuration.cache()));
} catch (final IOException e) {
throw new IllegalArgumentException("Can't create HTTP cache directory : '" + configuration.cache() + "', adjust --http-cache parameter");
}
this.cacheValidity = configuration.cacheValidity();
this.cache = cache;
this.client = new ExtendedHttpClient(conf);
this.jsonMapper = jsonMapper;
this.clock = systemDefaultZone();
}

@Override
Expand Down Expand Up @@ -160,27 +142,15 @@ public HttpResponse<Path> getFile(final HttpRequest request, final Path target,
}

public HttpResponse<String> send(final HttpRequest request) {
final Path cacheLocation;
if (cache != null) {
cacheLocation = cache.resolve(Base64.getUrlEncoder().withoutPadding().encodeToString(request.uri().toASCIIString().getBytes(UTF_8)));
if (Files.exists(cacheLocation)) {
try {
final var cached = jsonMapper.fromString(Response.class, Files.readString(cacheLocation));
if (cached.validUntil() > clock.instant().toEpochMilli()) {
return new StaticHttpResponse<>(
request, request.uri(), HTTP_1_1, 200,
HttpHeaders.of(
cached.headers().entrySet().stream()
.collect(toMap(Map.Entry::getKey, e -> List.of(e.getValue()))),
(a, b) -> true),
cached.payload());
}
} catch (final IOException e) {
throw new IllegalStateException(e);
}
}
} else {
cacheLocation = null;
final var entry = cache.lookup(request);
if (entry != null && entry.hit() != null) {
return new StaticHttpResponse<>(
request, request.uri(), HTTP_1_1, 200,
HttpHeaders.of(
entry.hit().headers().entrySet().stream()
.collect(toMap(Map.Entry::getKey, e -> List.of(e.getValue()))),
(a, b) -> true),
entry.hit().payload());
}

logger.finest(() -> "Calling " + request);
Expand All @@ -199,22 +169,8 @@ public HttpResponse<String> send(final HttpRequest request) {
result = result != null ? result : new StaticHttpResponse<>(
response.request(), response.uri(), response.version(), response.statusCode(), response.headers(),
new String(response.body(), UTF_8));
if (cacheLocation != null && result.statusCode() == 200) {
final var cachedData = jsonMapper.toString(new Response(
result.headers().map().entrySet().stream()
.filter(it -> !"content-encoding".equalsIgnoreCase(it.getKey()))
.collect(toMap(Map.Entry::getKey, l -> String.join(",", l.getValue()))),
result.body(),
clock.instant().plusMillis(cacheValidity).toEpochMilli()));
try {
Files.writeString(cacheLocation, cachedData);
} catch (final IOException e) {
try {
Files.deleteIfExists(cacheLocation);
} catch (final IOException ex) {
// no-op
}
}
if (entry != null && result.statusCode() == 200) {
cache.save(entry.key(), result);
}
return result;
} catch (final InterruptedException var4) {
Expand Down Expand Up @@ -282,8 +238,4 @@ public void onComplete() {
throw new IllegalStateException(var5);
}
}

@JsonModel
public record Response(Map<String, String> headers, String payload, long validUntil) {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ void simplifiedOptions(@TempDir final Path work, final URI uri) throws IOExcepti
@Test
void list(@TempDir final Path work, final URI uri) {
assertEquals("""
- java:
- [zulu] java:
-- 21.0.2""", captureOutput(work, uri, "list"));
}

Expand Down
Loading

0 comments on commit 3279782

Please sign in to comment.