Skip to content

Commit

Permalink
Add Function Server executable and integrate docker.
Browse files Browse the repository at this point in the history
  • Loading branch information
Joe-Abraham committed Oct 11, 2024
1 parent 59dc194 commit 775acbd
Show file tree
Hide file tree
Showing 7 changed files with 101 additions and 6 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,4 @@ presto-native-execution/deps-install
# Compiled executables used for docker build
/docker/presto-cli-*-executable.jar
/docker/presto-server-*.tar.gz
/docker/presto-remote-function-server-executable.jar
4 changes: 4 additions & 0 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ ENV PRESTO_HOME="/opt/presto-server"

COPY $PRESTO_PKG .
COPY $PRESTO_CLI_JAR /opt/presto-cli
COPY $PRESTO_REMOTE_SERVER_JAR /opt/presto-remote-server


RUN dnf install -y java-11-openjdk less procps python3 \
&& ln -s $(which python3) /usr/bin/python \
Expand All @@ -19,6 +21,8 @@ RUN dnf install -y java-11-openjdk less procps python3 \
&& rm -rf ./presto-server-$PRESTO_VERSION \
&& chmod +x /opt/presto-cli \
&& ln -s /opt/presto-cli /usr/local/bin/ \
&& chmod +x /opt/presto-remote-server \

# clean cache jobs
&& mv /etc/yum/protected.d/systemd.conf /etc/yum/protected.d/systemd.conf.bak \
&& dnf clean all \
Expand Down
20 changes: 20 additions & 0 deletions presto-main/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,26 @@
</ignorePackages>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>presto-remote-function-server-executable</finalName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.facebook.presto.server.FunctionServer</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import com.google.inject.Injector;
import com.google.inject.Module;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.facebook.presto.server.PrestoSystemRequirements.verifyJvmRequirements;
import static com.facebook.presto.server.PrestoSystemRequirements.verifySystemTimeIsReasonable;
Expand All @@ -47,21 +49,26 @@ public void run()
verifyJvmRequirements();
verifySystemTimeIsReasonable();

Logger log = Logger.get(FunctionServer.class);
Map<String, String> configMap = new HashMap<>();
configMap.put("http-server.http.enabled", "true");
configMap.put("http-server.http.port", "8085");

List<Module> modules = ImmutableList.of(
new FunctionServerModule(),
new HttpServerModule(),
new JaxrsModule());

try {
Bootstrap app = new Bootstrap(modules);
Bootstrap app = new Bootstrap(modules)
.setRequiredConfigurationProperties(configMap); // Set the configuration map
Injector injector = app.initialize();

HttpServerInfo serverInfo = injector.getInstance(HttpServerInfo.class);
String httpUrl = serverInfo.getHttpUri().toString(); // This will give you the HTTP URL
System.out.println(httpUrl);
log.info("======== REMOTE FUNCTION SERVER STARTED at: " + serverInfo.getHttpUri() + " =========");

Thread.currentThread().join();
Thread.currentThread().join(); // Keep the server running
}
catch (Throwable e) {
log.error(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
Expand All @@ -58,7 +59,7 @@ public class ContainerQueryRunner
private static final String PRESTO_COORDINATOR_IMAGE = System.getProperty("coordinatorImage", "presto-coordinator:latest");
private static final String PRESTO_WORKER_IMAGE = System.getProperty("workerImage", "presto-worker:latest");
private static final String CONTAINER_TIMEOUT = System.getProperty("containerTimeout", "120");
private static final String CLUSTER_SHUTDOWN_TIMEOUT = System.getProperty("clusterShutDownTimeout", "10");
private static final String CLUSTER_SHUTDOWN_TIMEOUT = System.getProperty("clusterShutDownTimeout", "10000");
private static final String BASE_DIR = System.getProperty("user.dir");
private static final int DEFAULT_COORDINATOR_PORT = 8080;
private static final String TPCH_CATALOG = "tpch";
Expand Down Expand Up @@ -110,6 +111,7 @@ public ContainerQueryRunner(int coordinatorPort, String catalog, String schema,
try {
Connection connection = DriverManager.getConnection(url, "test", null);
statement = connection.createStatement();
statement.execute("set session remote_functions_enabled=true");
}
catch (Exception e) {
e.printStackTrace();
Expand All @@ -131,6 +133,7 @@ private GenericContainer<?> createCoordinator()
ContainerQueryRunnerUtils.createCoordinatorLogProperties();
ContainerQueryRunnerUtils.createCoordinatorNodeProperties();
ContainerQueryRunnerUtils.createCoordinatorEntryPointScript();
ContainerQueryRunnerUtils.createFunctionNamespaceRemoteProperties();

return new GenericContainer<>(PRESTO_COORDINATOR_IMAGE)
.withExposedPorts(coordinatorPort)
Expand Down Expand Up @@ -297,9 +300,9 @@ public Session getDefaultSession()
public MaterializedResult execute(Session session, String sql)
{
try {
ResultSet resultSet = statement.executeQuery(sql);
return ContainerQueryRunnerUtils
.toMaterializedResult(
statement.executeQuery(sql));
.toMaterializedResult(resultSet);
}
catch (SQLException e) {
e.printStackTrace();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ public static void createCoordinatorConfigProperties(int port)
properties.setProperty("http-server.http.port", Integer.toString(port));
properties.setProperty("discovery-server.enabled", "true");
properties.setProperty("discovery.uri", "http://presto-coordinator:" + port);
properties.setProperty("list-built-in-functions-only", "false");
properties.setProperty("native-execution-enabled", "false");

// Get native worker system properties and add them to the coordinator properties
Map<String, String> nativeWorkerProperties = NativeQueryRunnerUtils.getNativeWorkerSystemProperties();
Expand All @@ -97,6 +99,24 @@ public static void createCoordinatorConfigProperties(int port)
createPropertiesFile("testcontainers/coordinator/etc/config.properties", properties);
}

public static void createFunctionNamespaceRemoteProperties()
throws IOException
{
Properties properties = new Properties();
properties.setProperty("function-namespace-manager.name", "rest");
properties.setProperty("supported-function-languages", "Java");
properties.setProperty("function-implementation-type", "REST");
properties.setProperty("rest-based-function-manager.rest.url", "http://localhost:8085");

String directoryPath = "testcontainers/function-namespace";
File directory = new File(directoryPath);
if (!directory.exists()) {
directory.mkdirs();
}

createPropertiesFile("testcontainers/coordinator/etc/function-namespace/remote.properties", properties);
}

public static void createCoordinatorJvmConfig()
throws IOException

Expand Down Expand Up @@ -155,6 +175,7 @@ public static void createCoordinatorEntryPointScript()
{
String scriptContent = "#!/bin/sh\n" +
"set -e\n" +
"java -jar /opt/presto-remote-server >> log1.txt 2>&1 &\n" +
"$PRESTO_HOME/bin/launcher run\n";
createScriptFile("testcontainers/coordinator/entrypoint.sh", scriptContent);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.nativeworker;

import com.facebook.presto.tests.AbstractTestQueryFramework;
import org.testng.annotations.Test;

import static org.testng.Assert.assertEquals;

public class TestPrestoContainerRemoteFunction
extends AbstractTestQueryFramework
{
private ContainerQueryRunner queryRunner;

@Override
protected ContainerQueryRunner createQueryRunner()
throws Exception
{
queryRunner = new ContainerQueryRunner();
return queryRunner;
}

@Test
public void testPresenceAndBasicFunctionality()
{
assertEquals(computeActual("select remote.default.abs(-10)").getMaterializedRows().get(0).getField(0).toString(), "10");
}
}

0 comments on commit 775acbd

Please sign in to comment.