Skip to content

Commit

Permalink
Implement v2 of executor SPI
Browse files Browse the repository at this point in the history 
This allows to support certificate files without introducing breaking changes.

- undo breaking changes to spi.v1
- play it safe and keep spi.v2 totally independent of spi.v1
  (no `ExecutorSpiOptions2 extends ExecutorSpiOptions` etc.)
- run executor tests against both SPI versions
- switch to `getPlatformClassLoader()`
  as indicated in "once we move to JDK9+" comments
  • Loading branch information
@odenix
odenix committed Feb 27, 2024
1 parent 09be2f4 commit 8d78d28
Show file tree
Hide file tree
Showing 9 changed files with 449 additions and 181 deletions.
130 changes: 95 additions & 35 deletions pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.nio.file.Path;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
Expand All @@ -36,8 +37,11 @@
import org.pkl.executor.spi.v1.ExecutorSpi;
import org.pkl.executor.spi.v1.ExecutorSpiException;
import org.pkl.executor.spi.v1.ExecutorSpiOptions;
import org.pkl.executor.spi.v2.ExecutorSpi2;
import org.pkl.executor.spi.v2.ExecutorSpiException2;
import org.pkl.executor.spi.v2.ExecutorSpiOptions2;

public class ExecutorSpiImpl implements ExecutorSpi {
public class ExecutorSpiImpl implements ExecutorSpi, ExecutorSpi2 {
private static final int MAX_HTTP_CLIENTS = 3;

// Don't create a new HTTP client for every executor request.
Expand Down Expand Up @@ -66,33 +70,94 @@ public String getPklVersion() {

@Override
public String evaluatePath(Path modulePath, ExecutorSpiOptions options) {
var allowedModules =
options.getAllowedModules().stream().map(Pattern::compile).collect(Collectors.toList());
var builder =
createEvaluatorBuilder(
options.getAllowedModules(),
options.getAllowedResources(),
options.getRootDir(),
options.getModulePath(),
options.getEnvironmentVariables(),
options.getExternalProperties(),
options.getTimeout(),
options.getOutputFormat(),
options.getModuleCacheDir(),
options.getProjectDir(),
List.of(),
List.of());

var allowedResources =
options.getAllowedResources().stream().map(Pattern::compile).collect(Collectors.toList());
try (var evaluator = builder.build()) {
return evaluator.evaluateOutputText(ModuleSource.path(modulePath));
} catch (PklException e) {
throw new ExecutorSpiException(e.getMessage(), e.getCause());
} finally {
ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories());
}
}

@Override
public String evaluatePath(Path modulePath, ExecutorSpiOptions2 options) {
var builder =
createEvaluatorBuilder(
options.getAllowedModules(),
options.getAllowedResources(),
options.getRootDir(),
options.getModulePath(),
options.getEnvironmentVariables(),
options.getExternalProperties(),
options.getTimeout(),
options.getOutputFormat(),
options.getModuleCacheDir(),
options.getProjectDir(),
options.getCertificateFiles(),
options.getCertificateUris());

try (var evaluator = builder.build()) {
return evaluator.evaluateOutputText(ModuleSource.path(modulePath));
} catch (PklException e) {
throw new ExecutorSpiException2(e.getMessage(), e.getCause());
} finally {
ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories());
}
}

private EvaluatorBuilder createEvaluatorBuilder(
List<String> allowedModules,
List<String> allowedResources,
Path rootDir,
List<Path> modulePath,
Map<String, String> environmentVariables,
Map<String, String> externalProperties,
java.time.Duration timeout,
String outputFormat,
Path moduleCacheDir,
Path projectDir,
List<Path> certificateFiles,
List<URI> certificateUris) {
var allowedModulePatterns =
allowedModules.stream().map(Pattern::compile).collect(Collectors.toList());

var allowedResourcePatterns =
allowedResources.stream().map(Pattern::compile).collect(Collectors.toList());

var securityManager =
SecurityManagers.standard(
allowedModules,
allowedResources,
allowedModulePatterns,
allowedResourcePatterns,
SecurityManagers.defaultTrustLevels,
options.getRootDir());
rootDir);

var transformer = StackFrameTransformers.defaultTransformer;
if (options.getRootDir() != null) {
if (rootDir != null) {
transformer =
transformer.andThen(
StackFrameTransformers.relativizeModuleUri(options.getRootDir().toUri()));
transformer.andThen(StackFrameTransformers.relativizeModuleUri(rootDir.toUri()));
}

var resolver = new ModulePathResolver(options.getModulePath());

var resolver = new ModulePathResolver(modulePath);
var builder =
EvaluatorBuilder.unconfigured()
.setStackFrameTransformer(transformer)
.setSecurityManager(securityManager)
.setHttpClient(getOrCreateHttpClient(options))
.setHttpClient(getOrCreateHttpClient(certificateFiles, certificateUris))
.addResourceReader(ResourceReaders.environmentVariable())
.addResourceReader(ResourceReaders.externalProperty())
.addResourceReader(ResourceReaders.modulePath(resolver))
Expand All @@ -108,27 +173,22 @@ public String evaluatePath(Path modulePath, ExecutorSpiOptions options) {
.addModuleKeyFactory(ModuleKeyFactories.projectpackage)
.addModuleKeyFactory(ModuleKeyFactories.file)
.addModuleKeyFactory(ModuleKeyFactories.genericUrl)
.setEnvironmentVariables(options.getEnvironmentVariables())
.setExternalProperties(options.getExternalProperties())
.setTimeout(options.getTimeout())
.setOutputFormat(options.getOutputFormat())
.setModuleCacheDir(options.getModuleCacheDir());
if (options.getProjectDir() != null) {
var project = Project.loadFromPath(options.getProjectDir().resolve(PKL_PROJECT_FILENAME));
.setEnvironmentVariables(environmentVariables)
.setExternalProperties(externalProperties)
.setTimeout(timeout)
.setOutputFormat(outputFormat)
.setModuleCacheDir(moduleCacheDir);

if (projectDir != null) {
var project = Project.loadFromPath(projectDir.resolve(PKL_PROJECT_FILENAME));
builder.setProjectDependencies(project.getDependencies());
}

try (var evaluator = builder.build()) {
return evaluator.evaluateOutputText(ModuleSource.path(modulePath));
} catch (PklException e) {
throw new ExecutorSpiException(e.getMessage(), e.getCause());
} finally {
ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories());
}
return builder;
}

private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) {
var clientKey = new HttpClientKey(options);
private HttpClient getOrCreateHttpClient(List<Path> certificateFiles, List<URI> certificateUris) {
var clientKey = new HttpClientKey(certificateFiles, certificateUris);
return httpClients.computeIfAbsent(
clientKey,
(key) -> {
Expand All @@ -140,7 +200,7 @@ private HttpClient getOrCreateHttpClient(ExecutorSpiOptions options) {
builder.addCertificates(uri);
}
// If the above didn't add any certificates,
// builder will fall back to Pkl's built-in certificates.
// builder will use the JVM's default SSL context.
return builder.build();
});
}
Expand All @@ -149,10 +209,10 @@ private static final class HttpClientKey {
final Set<Path> certificateFiles;
final Set<URI> certificateUris;

HttpClientKey(ExecutorSpiOptions options) {
// make defensive copies
certificateFiles = Set.copyOf(options.getCertificateFiles());
certificateUris = Set.copyOf(options.getCertificateUris());
HttpClientKey(List<Path> certificateFiles, List<URI> certificateUris) {
// also serve as defensive copies
this.certificateFiles = Set.copyOf(certificateFiles);
this.certificateUris = Set.copyOf(certificateUris);
}

@Override
Expand Down
1 change: 1 addition & 0 deletions pkl-core/src/main/resources/META-INF/services/org.pkl.executor.spi.v2.ExecutorSpi2
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.pkl.core.service.ExecutorSpiImpl
102 changes: 71 additions & 31 deletions pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
import org.pkl.executor.spi.v1.ExecutorSpi;
import org.pkl.executor.spi.v1.ExecutorSpiException;
import org.pkl.executor.spi.v1.ExecutorSpiOptions;
import org.pkl.executor.spi.v2.ExecutorSpi2;
import org.pkl.executor.spi.v2.ExecutorSpiException2;
import org.pkl.executor.spi.v2.ExecutorSpiOptions2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -40,12 +43,17 @@ final class EmbeddedExecutor implements Executor {
private final List<PklDistribution> pklDistributions = new ArrayList<>();

/**
* @throws IllegalArgumentException if a Jar file cannot be found or is not a valid PklPkl
* @throws IllegalArgumentException if a Jar file cannot be found or is not a valid Pkl
* distribution
*/
public EmbeddedExecutor(List<Path> pklFatJars) {
this(pklFatJars, 0);
}

// for testing only
EmbeddedExecutor(List<Path> pklFatJars, int spiVersion) {
for (var jarFile : pklFatJars) {
pklDistributions.add(new PklDistribution(jarFile));
pklDistributions.add(new PklDistribution(jarFile, spiVersion));
}
}

Expand Down Expand Up @@ -163,41 +171,61 @@ public void close() throws Exception {
}

private static final class PklDistribution implements AutoCloseable {
final URLClassLoader classLoader;
final ExecutorSpi executorSpi;
final URLClassLoader pklDistributionClassLoader;
final /* @Nullable */ ExecutorSpi executorSpi;
final /* @Nullable */ ExecutorSpi2 executorSpi2;
final Version version;

/**
* @throws IllegalArgumentException if the Jar file does not exist or is not a valid Pkl
* distribution
*/
PklDistribution(Path pklFatJar) {
PklDistribution(Path pklFatJar, /* 0 -> choose highest available */ int spiVersion) {
if (!Files.isRegularFile(pklFatJar)) {
throw new IllegalArgumentException(
String.format("Invalid Pkl distribution: Cannot find Jar file `%s`.", pklFatJar));
}

classLoader = new PklDistributionClassLoader(pklFatJar);
var serviceLoader = ServiceLoader.load(ExecutorSpi.class, classLoader);
pklDistributionClassLoader = new PklDistributionClassLoader(pklFatJar);

try {
executorSpi = serviceLoader.iterator().next();
} catch (NoSuchElementException e) {
throw new IllegalArgumentException(
String.format(
"Invalid Pkl distribution: Cannot find service of type `%s` in Jar file `%s`.",
ExecutorSpi.class.getTypeName(), pklFatJar));

if (spiVersion == 0 || spiVersion == 2) {
var serviceLoader2 = ServiceLoader.load(ExecutorSpi2.class, pklDistributionClassLoader);
executorSpi2 = serviceLoader2.findFirst().orElse(null);
if (executorSpi2 != null) {
executorSpi = null;
// convert to normal to allow running with a dev version
version = Version.parse(executorSpi2.getPklVersion()).toNormal();
return;
}
} else {
executorSpi2 = null;
}
if (spiVersion == 0 || spiVersion == 1) {
var serviceLoader = ServiceLoader.load(ExecutorSpi.class, pklDistributionClassLoader);
executorSpi = serviceLoader.findFirst().orElse(null);
if (executorSpi != null) {
// convert to normal to allow running with a dev version
version = Version.parse(executorSpi.getPklVersion()).toNormal();
return;
}
} else {
executorSpi = null;
}
if (spiVersion == 0) {
throw new IllegalArgumentException(
String.format(
"Invalid Pkl distribution: Cannot find service of type `%s` in Jar file `%s`.",
ExecutorSpi.class.getTypeName(), pklFatJar));
}
throw new IllegalArgumentException("Invalid SPI version: " + spiVersion);
} catch (ServiceConfigurationError e) {
throw new IllegalArgumentException(
String.format(
"Invalid Pkl distribution: Unexpected error loading service of type `%s` from Jar file `%s`.",
ExecutorSpi.class.getTypeName(), pklFatJar),
e);
}

// convert to normal to allow running with a dev version
version = Version.parse(executorSpi.getPklVersion()).toNormal();
}

Version getVersion() {
Expand All @@ -208,10 +236,12 @@ String evaluatePath(Path modulePath, ExecutorOptions options) {
var currentThread = Thread.currentThread();
var prevContextClassLoader = currentThread.getContextClassLoader();
// Truffle loads stuff from context class loader, so set it to our class loader
currentThread.setContextClassLoader(classLoader);
currentThread.setContextClassLoader(pklDistributionClassLoader);
try {
return executorSpi.evaluatePath(modulePath, toEvaluatorOptions(options));
} catch (ExecutorSpiException e) {
return executorSpi2 != null
? executorSpi2.evaluatePath(modulePath, toSpiOptions2(options))
: executorSpi.evaluatePath(modulePath, toSpiOptions(options));
} catch (ExecutorSpiException | ExecutorSpiException2 e) {
throw new ExecutorException(e.getMessage(), e.getCause());
} finally {
currentThread.setContextClassLoader(prevContextClassLoader);
Expand All @@ -220,24 +250,38 @@ String evaluatePath(Path modulePath, ExecutorOptions options) {

@Override
public void close() throws IOException {
classLoader.close();
pklDistributionClassLoader.close();
}

ExecutorSpiOptions toEvaluatorOptions(ExecutorOptions options) {
ExecutorSpiOptions toSpiOptions(ExecutorOptions options) {
return new ExecutorSpiOptions(
options.getAllowedModules(),
options.getAllowedResources(),
options.getEnvironmentVariables(),
options.getExternalProperties(),
options.getModulePath(),
options.getCertificateFiles(),
options.getCertificateUris(),
options.getRootDir(),
options.getTimeout(),
options.getOutputFormat(),
options.getModuleCacheDir(),
options.getProjectDir());
}

ExecutorSpiOptions2 toSpiOptions2(ExecutorOptions options) {
return new ExecutorSpiOptions2(
options.getAllowedModules(),
options.getAllowedResources(),
options.getModulePath(),
options.getRootDir(),
options.getTimeout(),
options.getOutputFormat(),
options.getModuleCacheDir(),
options.getProjectDir(),
options.getEnvironmentVariables(),
options.getExternalProperties(),
options.getCertificateFiles(),
options.getCertificateUris());
}
}

private static final class PklDistributionClassLoader extends URLClassLoader {
Expand Down Expand Up @@ -284,18 +328,14 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE

@Override
public URL getResource(String name) {
// try bootstrap class loader first
// once we move to JDK 9+, should use `getPlatformClassLoader().getResource()` instead of
// `super.getResource()`
var resource = super.getResource(name);
var resource = getPlatformClassLoader().getResource(name);
return resource != null ? resource : findResource(name);
}

@Override
public Enumeration<URL> getResources(String name) throws IOException {
// once we move to JDK 9+, should use `getPlatformClassLoader().getResources()` instead of
// `super.getResources()`
return ConcatenatedEnumeration.create(super.getResources(name), findResources(name));
return ConcatenatedEnumeration.create(
getPlatformClassLoader().getResources(name), findResources(name));
}

static URL[] toUrls(Path pklFatJar) {
Expand Down
Loading

0 comments on commit 8d78d28

Please sign in to comment.