diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt index 51bf01e77..29e51e75a 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt @@ -1457,6 +1457,49 @@ result = someLib.x assertThat(output).isEqualTo("result = 1\n") } + @Test + fun `eval file with non-ASCII name`() { + val dir = tempDir.resolve("🤬").createDirectory() + val file = writePklFile(dir.resolve("日本語.pkl").toString(), """ + 日本語 = "Japanese language" + readDir = read(".").text + readDirFile = read("file:$tempDir/🤬").text + readOne = read("日本語.pkl").text.split("\n").first + readOneFile = read("file:$tempDir/🤬/日本語.pkl").text.split("\n").first + readGlob = read*("./日*.pkl").keys + readGlobFile = read*("file:$tempDir/**/*.pkl").keys.map((it) -> it.replaceAll("$tempDir", "")) + importOne = import("日本語.pkl").readOne + importOneFile = import("file:$tempDir/🤬/日本語.pkl").日本語 + importGlob = import*("./日*.pkl").keys + importGlobFile = import*("file:$tempDir/**/*.pkl").keys.map((it) -> it.replaceAll("$tempDir", "")) + """.trimIndent()) + val output = + evalToConsole( + CliEvaluatorOptions( + CliBaseOptions(sourceModules = listOf(file)), + ) + ) + + assertThat(output).isEqualTo("""日本語 = "Japanese language" +readDir = ""${'"'} + 日本語.pkl + + ""${'"'} +readDirFile = ""${'"'} + 日本語.pkl + + ""${'"'} +readOne = "日本語 = \"Japanese language\"" +readOneFile = "日本語 = \"Japanese language\"" +readGlob = Set("./日本語.pkl") +readGlobFile = Set("file:/🤬/日本語.pkl") +importOne = "日本語 = \"Japanese language\"" +importOneFile = "Japanese language" +importGlob = Set("./日本語.pkl") +importGlobFile = Set("file:/🤬/日本語.pkl") +""") + } + private fun evalModuleThatImportsPackage(certsFile: Path?, testPort: Int = -1) { val moduleUri = writePklFile( diff --git a/pkl-core/src/main/java/org/pkl/core/StackFrameTransformers.java b/pkl-core/src/main/java/org/pkl/core/StackFrameTransformers.java index 2122cfa19..64eb10391 100644 --- a/pkl-core/src/main/java/org/pkl/core/StackFrameTransformers.java +++ b/pkl-core/src/main/java/org/pkl/core/StackFrameTransformers.java @@ -93,7 +93,7 @@ public static StackFrameTransformer convertFilePathToUriScheme(String scheme) { var uri = frame.getModuleUri(); if (!uri.startsWith("file:")) return frame; - return transformUri(frame, Path.of(URI.create(uri)).toString(), scheme); + return transformUri(frame, IoUtils.pathOfURI(URI.create(uri)).toString(), scheme); }; } diff --git a/pkl-core/src/main/java/org/pkl/core/module/FileResolver.java b/pkl-core/src/main/java/org/pkl/core/module/FileResolver.java index 300724757..e3d0dd4b7 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/FileResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/module/FileResolver.java @@ -24,12 +24,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.pkl.core.util.IoUtils; public final class FileResolver { private FileResolver() {} public static List listElements(URI baseUri) throws IOException { - return listElements(Path.of(baseUri)); + return listElements(IoUtils.pathOfURI(baseUri)); } public static List listElements(Path path) throws IOException { @@ -49,7 +50,7 @@ public static List listElements(Path path) throws IOException { } public static boolean hasElement(URI elementUri) { - return Files.exists(Path.of(elementUri)); + return Files.exists(IoUtils.pathOfURI(elementUri)); } public static boolean hasElement(Path path) { diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index 7a798a113..d17031f8c 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -327,18 +327,10 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) throw new FileNotFoundException(); } - // Path.of(URI) throws on non-ASCII characters so the module URI here must be normalized to - // ASCII - // Unfortunately there's no way to go from URI -> ASCII URI directly - // so this must transform URI -> ASCII String -> ASCII URI - try { - var realPath = Path.of(new URI(uri.toASCIIString())).toRealPath(); - var resolvedUri = realPath.toUri(); - securityManager.checkResolveModule(resolvedUri); - return ResolvedModuleKeys.file(this, resolvedUri, realPath); - } catch (URISyntaxException e) { - throw new PklBugException("File module URI could not be normalized to ASCII", e); - } + var realPath = IoUtils.pathOfURI(uri).toRealPath(); + var resolvedUri = realPath.toUri(); + securityManager.checkResolveModule(resolvedUri); + return ResolvedModuleKeys.file(this, resolvedUri, realPath); } @Override diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 96ee80b76..aa2b1c82d 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -39,6 +39,7 @@ import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; +import org.pkl.core.module.ResolvedModuleKeys; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.ReaderBase; import org.pkl.core.runtime.VmContext; @@ -88,6 +89,22 @@ public static URI toUri(String str) throws URISyntaxException { return new URI(null, null, str, null); } + /** + * Converts a URI to a Path, normalizing any non-ASCII characters. + */ + public static Path pathOfURI(URI uri) { + // Path.of(URI) throws on non-ASCII characters so the module URI here must be normalized to + // ASCII + // Unfortunately there's no way to go from URI -> ASCII URI directly + // so this must transform URI -> ASCII String -> ASCII URI + try { + return Path.of(new URI(uri.toASCIIString())); + } catch (URISyntaxException e) { + // impossible to get here; we started from a valid URI to begin with + throw PklBugException.unreachableCode(); + } + } + /** Like {@link #toUri(String)}, except without checked exceptions. */ public static URI createUri(String str) { try { diff --git "a/pkl-core/src/test/files/LanguageSnippetTests/input/modules/\346\227\245\346\234\254\350\252\236_error.pkl" "b/pkl-core/src/test/files/LanguageSnippetTests/input/modules/\346\227\245\346\234\254\350\252\236_error.pkl" new file mode 100644 index 000000000..2af8f6365 --- /dev/null +++ "b/pkl-core/src/test/files/LanguageSnippetTests/input/modules/\346\227\245\346\234\254\350\252\236_error.pkl" @@ -0,0 +1,2 @@ +// covers https://github.com/apple/pkl/issues/653 +日本語 = throw("Error reporting should also handle unicode filenames!") diff --git "a/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236.pcf" "b/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236.pcf" index d43194be7..97feec800 100644 --- "a/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236.pcf" +++ "b/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236.pcf" @@ -8,7 +8,7 @@ readOne = """ importGlob = import*("./日*.pkl").keys """ -readGlob = Set("./日本語.pkl") +readGlob = Set("./日本語.pkl", "./日本語_error.pkl") importOne = """ // covers https://github.com/apple/pkl/issues/653 日本語 = "Japanese language" @@ -18,4 +18,4 @@ importOne = """ importGlob = import*("./日*.pkl").keys """ -importGlob = Set("./日本語.pkl") +importGlob = Set("./日本語.pkl", "./日本語_error.pkl") diff --git "a/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236_error.err" "b/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236_error.err" new file mode 100644 index 000000000..6b4e13aa6 --- /dev/null +++ "b/pkl-core/src/test/files/LanguageSnippetTests/output/modules/\346\227\245\346\234\254\350\252\236_error.err" @@ -0,0 +1,10 @@ +–– Pkl Error –– +Error reporting should also handle unicode filenames! + +x | 日本語 = throw("Error reporting should also handle unicode filenames!") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at 日本語_error#日本語 (file:///$snippetsDir/input/modules/%E6%97%A5%E6%9C%AC%E8%AA%9E_error.pkl) + +xxx | text = renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base)