Follow P2 contract of cached file's extension (eclipse-tycho#2938)

This is a backport from 5.0.0

P2 relies on correct file extensions to parse cached files.
This change prevents cached file name from changing over HTTP redirect.

See
eclipse-equinox/p2#355

Fix HttpServer concurrency bug.
URLs returned from HttpServer.getAccessedUrls() are now stripped of
context prefix. No callers have used these values until now.
Fix concurrency bug in test utility class HttpServer request logging.
Add verification of request logs to ensure that metadata repository does
indeed unjars fresh, non-cached artifact. Ensure that cached XML file
does not produce false negative in tests.
Refactor tests.

p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/SharedHttpCacheStorage.java

@@ -262,8 +262,16 @@ public synchronized File fetchFile(URI uri, HttpTransportFactory transportFactor
 				}
 				updateHeader(response, code);
 				if (isRedirected(code)) {
-					return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger)
+					File cachedFile = SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger)
 							.getCacheFile(transportFactory);
+					// https://github.com/eclipse-tycho/tycho/issues/2938
+					// Redirect may change extension. P2's SimpleMetadataRepositoryFactory relies on
+					// accurate file extension to be cached.
+					// Copying file to accommodate original request and its file extension.
+					// Once https://github.com/eclipse-equinox/p2/issues/355 is fixed, cachedFile
+					// may be returned directly without copying.
+					FileUtils.copyFile(cachedFile, file);
+					return file;
 				}
 				if (exits) {
 					FileUtils.forceDelete(file);

tycho-its/projects/target.content_jar/category.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+   <feature id="issue_2938_reproducer"/>
+</site>

tycho-its/projects/target.content_jar/pom.xml

@@ -0,0 +1,49 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>tycho-its-project.p2Repository.slicerDependencies</groupId>
+    <artifactId>aggregator</artifactId>
+    <version>1.0.0</version>
+    <packaging>eclipse-repository</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.eclipse.tycho</groupId>
+                <artifactId>tycho-maven-plugin</artifactId>
+                <version>${tycho-version}</version>
+                <extensions>true</extensions>
+            </plugin>
+            <plugin>
+                <groupId>org.eclipse.tycho</groupId>
+                <artifactId>target-platform-configuration</artifactId>
+                <version>${tycho-version}</version>
+                <configuration>
+                    <resolver>p2</resolver>
+                    <executionEnvironment>JavaSE-11</executionEnvironment>
+                    <target>
+						<file>targetplatform.target</file>
+                    </target>
+                    <environments>
+                        <environment>
+                            <os>win32</os>
+                            <ws>win32</ws>
+                            <arch>x86_64</arch>
+                        </environment>
+                        <environment>
+                            <os>linux</os>
+                            <ws>gtk</ws>
+                            <arch>x86_64</arch>
+                        </environment>
+                    </environments>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

tycho-its/projects/target.content_jar/targetplatform.target

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?pde?>
+<target name="p2Repository.slicerDependencies" sequenceNumber="1">
+    <locations>
+        <location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="slicer" includeSource="true" type="InstallableUnit">
+            <repository id="repoA" location="url"/>
+
+            <unit id="issue_2938_reproducer.feature.group" version="0.0.0"/>
+        </location>
+    </locations>
+</target>

tycho-its/src/test/java/org/eclipse/tycho/test/target/OfflineModeTest.java

@@ -92,7 +92,7 @@ private void runAndVerifyOnlineBuild(Verifier verifier) throws VerificationExcep
 		verifier.executeGoal("integration-test");
 		verifier.verifyErrorFreeLog();
 		assertFalse(server.getAccessedUrls("test").isEmpty());
-		server.getAccessedUrls("test").clear();
+		server.clear();
 	}
 
 }

tycho-its/src/test/java/org/eclipse/tycho/test/tycho2938/ContentJarTest.java

@@ -0,0 +1,131 @@
+/*******************************************************************************
+ * Copyright (c) 2023, 2023 Sonatype Inc. and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Vasili Gulevich - initial implementation
+ *******************************************************************************/
+package org.eclipse.tycho.test.tycho2938;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.it.Verifier;
+import org.eclipse.tycho.test.AbstractTychoIntegrationTest;
+import org.eclipse.tycho.test.util.HttpServer;
+import org.eclipse.tycho.test.util.ResourceUtil;
+import org.eclipse.tycho.test.util.TargetDefinitionUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.xml.sax.SAXException;
+
+public class ContentJarTest extends AbstractTychoIntegrationTest {
+	private HttpServer server;
+	private static final String TARGET_FEATURE_PATH = "/features/issue_2938_reproducer_1.0.0.202310211419.jar";
+	@Rule
+	public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+	private Verifier verifier;
+	private String mainRepoUrl;
+
+	@Before
+	public void startServer() throws Exception {
+		server = HttpServer.startServer();
+		File repositoryRoot = temporaryFolder.getRoot();
+		this.mainRepoUrl = server.addServer("repoA", repositoryRoot);
+		File originalResource = ResourceUtil.resolveTestResource("repositories/content_jar");
+		FileUtils.copyDirectory(originalResource, repositoryRoot);
+		verifier = getVerifier("target.content_jar", false);
+		verifier.deleteArtifacts("p2.org.eclipse.update.feature", "issue_2938_reproducer", "1.0.0.202310211419");
+	}
+
+	@After
+	public void stopServer() throws Exception {
+		if (server != null) {
+			server.stop();
+		}
+	}
+
+	@Test
+	public void noRedirect() throws Exception {
+		configureRepositoryInTargetDefinition(mainRepoUrl);
+		verifier.executeGoal("package");
+		verifier.verifyErrorFreeLog();
+		assertVisited(TARGET_FEATURE_PATH);
+	}
+
+	@Test
+	public void redirectKeepFilename() throws Exception {
+		String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath);
+		configureRepositoryInTargetDefinition(redirectedUrl);
+		verifier.executeGoal("package");
+		verifier.verifyErrorFreeLog();
+		assertVisited(TARGET_FEATURE_PATH);
+	}
+
+	@Test
+	public void redirectToBadLocation() throws Exception {
+		String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath + "_invalid");
+		configureRepositoryInTargetDefinition(redirectedUrl);
+		Assert.assertThrows(VerificationException.class, () -> verifier.executeGoal("package"));
+		verifier.verifyTextInLog("No repository found at " + redirectedUrl);
+		assertVisited("/content.jar_invalid");
+	}
+
+	@Test
+	public void redirectToMangledLocations() throws Exception {
+		File repositoryRoot = temporaryFolder.getRoot();
+		mangleFileNames(repositoryRoot.toPath());
+
+		// https://github.com/eclipse-tycho/tycho/issues/2938
+		// Redirect may change extension.
+		String redirectedUrl = server.addRedirect("repoB", originalPath -> mainRepoUrl + originalPath + "_invalid");
+
+		configureRepositoryInTargetDefinition(redirectedUrl);
+		verifier.executeGoal("package");
+		verifier.verifyErrorFreeLog();
+		assertVisited("/content.jar_invalid");
+		assertVisited(TARGET_FEATURE_PATH + "_invalid");
+	}
+
+	private void assertVisited(String path) {
+		List<String> accessedUrls = server.getAccessedUrls("repoA");
+		Assert.assertTrue(String.format("Path %s should be visited, %s were visited instead", path, accessedUrls),
+				accessedUrls.contains(path));
+	}
+
+	private void mangleFileNames(Path repositoryRoot) throws IOException {
+		Files.walkFileTree(repositoryRoot, new SimpleFileVisitor<Path>() {
+			@Override
+			public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+				Files.move(file, file.getParent().resolve(file.getFileName() + "_invalid"));
+				return super.visitFile(file, attrs);
+			}
+		});
+	}
+
+	private void configureRepositoryInTargetDefinition(String url)
+			throws IOException, ParserConfigurationException, SAXException {
+		File platformFile = new File(verifier.getBasedir(), "targetplatform.target");
+		TargetDefinitionUtil.setRepositoryURLs(platformFile, "repoA", url);
+	}
+
+}