Skip to content

Commit

Permalink
[Optimize] Optimize doc and CI bash, Refactoring OALRuntime
Browse files Browse the repository at this point in the history
  • Loading branch information
yswdqz committed Sep 26, 2023
1 parent 98e747f commit a21905d
Show file tree
Hide file tree
Showing 8 changed files with 172 additions and 183 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/native-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Native Compile
run: |
./mvnw -Pnative,backend clean package -Dmaven.test.skip
./mvnw -Pbackend,native clean package -Dmaven.test.skip
- uses: actions/upload-artifact@v3
name: Upload distribution tar
with:
Expand Down
7 changes: 4 additions & 3 deletions docs/en/setup/backend/backend-graalvm.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ The native-image compilation is not enabled by default. To enable it, we need to

```shell

./mvnw -Pnative,backend clean package -Dmaven.test.skip
./mvnw -backend,naative clean package -Dmaven.test.skip

```

Then, 2 packages are in `distribution/graal/dist`, The package named `apache-skywalking-apm-native-pre-bin.tar.gz` is unnecessary for most users. It is just an intermediate product for generating the final native-image program, used for testing purposes.
The real outcome is the package named `apache-skywalking-apm-native-bin.tar.gz`, and followings are the introduction to its package structure.
Then, 2 packages are in `distribution/graal/dist`,
As this is an experimental feature, a package named `apache-skywalking-apm-native-pre-bin.tar.gz` is a tarball for GraalVMization friendly, including original classes to be compiled by GraalVM.
The package named `apache-skywalking-apm-native-bin.tar.gz` includes the final compiled native binary, relative configuration files, and booting shells. Read `Package Structure` doc for more details.

## Package Structure

Expand Down
7 changes: 0 additions & 7 deletions graal/graal-server-starter/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,6 @@
<artifactId>oal-rt-graal-native</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.graalvm.nativeimage</groupId>
<artifactId>svm</artifactId>
<scope>provided</scope>
<version>22.3.1</version>
</dependency>
</dependencies>

<profiles>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,8 @@
import lombok.extern.slf4j.Slf4j;

import org.apache.skywalking.oal.rt.util.OALClassGenerator;
import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener;
import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.StreamAnnotationListener;
import org.apache.skywalking.oap.server.core.oal.rt.OALDefine;
import org.apache.skywalking.oap.server.core.oal.rt.OALEngine;
import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
import org.apache.skywalking.oap.server.core.storage.StorageException;

import org.apache.skywalking.oap.server.library.module.ModuleStartException;

import java.io.File;
import java.io.IOException;
Expand All @@ -40,70 +33,25 @@
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;

import java.util.Map;
import java.util.Objects;


/**
* Changed the implementation of the OAL kernel due to the Graal limitation on runtime class generation.
* Now, class generation will be advanced to compile time, see (@link org.apache.skywalking.graal.OALGenerator) and loaded at runtime.
*/
@Slf4j
public class OALRuntime implements OALEngine {

private StreamAnnotationListener streamAnnotationListener;
private DispatcherDetectorListener dispatcherDetectorListener;
private final List<Class> metricsClasses;
private final List<Class> dispatcherClasses;
public class OALRuntime extends OALKernel {

// change
private static boolean INITIALED = false;

//change end

public OALRuntime(OALDefine define) {
metricsClasses = new ArrayList<>();
dispatcherClasses = new ArrayList<>();
}

@Override
public void setStreamListener(StreamAnnotationListener listener) throws ModuleStartException {
this.streamAnnotationListener = listener;
}

@Override
public void setDispatcherListener(DispatcherDetectorListener listener) throws ModuleStartException {
dispatcherDetectorListener = listener;
}

@Override
public void setStorageBuilderFactory(final StorageBuilderFactory factory) {

}

@Override
public void notifyAllListeners() throws ModuleStartException {
for (Class metricsClass : metricsClasses) {
try {
streamAnnotationListener.notify(metricsClass);
} catch (StorageException e) {
throw new ModuleStartException(e.getMessage(), e);
}
}
for (Class dispatcherClass : dispatcherClasses) {
try {
dispatcherDetectorListener.addIfAsSourceDispatcher(dispatcherClass);
} catch (Exception e) {
throw new ModuleStartException(e.getMessage(), e);
}
}
super(define);
}

// ------------------------------------ substituted method ------------------------------------
@Override
public void start(ClassLoader currentClassLoader) {
if (INITIALED) {
Expand Down Expand Up @@ -143,7 +91,7 @@ private void startByNativeImage(ClassLoader currentClassLoader) {
if (!aClass.isAnnotationPresent(Stream.class)) {
return;
}
metricsClasses.add(aClass);
getMetricsClasses().add(aClass);
} catch (ClassNotFoundException e) {
// should not reach here
log.error(e.getMessage());
Expand All @@ -161,7 +109,7 @@ private void startByNativeImage(ClassLoader currentClassLoader) {
String name = file.toString().replace(File.separator, ".");
name = name.substring(0, name.length() - ".class".length());
try {
dispatcherClasses.add(Class.forName(name));
getDispatcherClasses().add(Class.forName(name));
} catch (ClassNotFoundException e) {
// should not reach here
log.error(e.getMessage());
Expand All @@ -187,7 +135,7 @@ private void startByJar(ClassLoader currentClassLoader) {
Enumeration<URL> metricsResources = currentClassLoader.getResources(metricsPath);
while (metricsResources.hasMoreElements()) {
URL resource = metricsResources.nextElement();
processResourcePath(resource, metricsClasses, metricsBuilderPath.replace(File.separatorChar, '.'));
processResourcePath(resource, getMetricsClasses(), metricsBuilderPath.replace(File.separatorChar, '.'));
}
} catch (IOException e) {
log.error("Failed to locate resource " + metricsPath + " on classpath");
Expand All @@ -197,7 +145,7 @@ private void startByJar(ClassLoader currentClassLoader) {
Enumeration<URL> dispatcherResources = currentClassLoader.getResources(dispatcherPath);
while (dispatcherResources.hasMoreElements()) {
URL resource = dispatcherResources.nextElement();
processResourcePath(resource, dispatcherClasses, null);
processResourcePath(resource, getDispatcherClasses(), null);
}
} catch (IOException e) {
log.error("Failed to locate resource " + dispatcherPath + " on classpath");
Expand Down
16 changes: 3 additions & 13 deletions graal/skywalking-on-graal-compiling-plugin/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,17 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.9.1</version>
</dependency>

<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.9.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>3.8.1</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>oal-rt-graal-native</artifactId>
Expand All @@ -61,6 +57,7 @@
<artifactId>library-util-graal-native</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>server-starter</artifactId>
Expand All @@ -83,13 +80,6 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>connect-api</artifactId>
<version>2.4.1</version>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>oal-rt</artifactId>
Expand All @@ -103,7 +93,7 @@
<plugins>
<plugin>
<artifactId>maven-plugin-plugin</artifactId>
<version>3.8.1</version>
<version>3.9.0</version>
<configuration>
<skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
</configuration>
Expand Down
21 changes: 21 additions & 0 deletions oap-server-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
<consul.client.version>1.5.3</consul.client.version>
<commons-net.version>3.9.0</commons-net.version>
<maven-core.version>3.9.0</maven-core.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -620,6 +621,26 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>${maven-core.version}</version>
</dependency>

<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>${maven-core.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>${maven-core.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</dependencyManagement>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.apache.skywalking.oal.rt;

import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oal.rt.parser.OALScripts;
import org.apache.skywalking.oal.rt.parser.ScriptParser;
import org.apache.skywalking.oal.rt.util.OALClassGenerator;
import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener;
import org.apache.skywalking.oap.server.core.analysis.StreamAnnotationListener;
import org.apache.skywalking.oap.server.core.oal.rt.OALCompileException;
import org.apache.skywalking.oap.server.core.oal.rt.OALDefine;
import org.apache.skywalking.oap.server.core.oal.rt.OALEngine;
import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
import org.apache.skywalking.oap.server.core.storage.StorageException;

import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;

/**
* OAL Runtime is the class generation engine, which load the generated classes from OAL scrip definitions. This runtime
* is loaded dynamically.
*/
@Slf4j
public class OALKernel implements OALEngine {

private static boolean IS_RT_TEMP_FOLDER_INIT_COMPLETED = false;

private OALClassGenerator oalClassGenerator;

private StreamAnnotationListener streamAnnotationListener;
private DispatcherDetectorListener dispatcherDetectorListener;
private final List<Class> metricsClasses;
private final List<Class> dispatcherClasses;

private final OALDefine oalDefine;

public OALKernel(OALDefine define) {
oalDefine = define;
metricsClasses = new ArrayList<>();
dispatcherClasses = new ArrayList<>();

oalClassGenerator = new OALClassGenerator(define);
}

@Override
public void setStreamListener(StreamAnnotationListener listener) throws ModuleStartException {
this.streamAnnotationListener = listener;
}

@Override
public void setDispatcherListener(DispatcherDetectorListener listener) throws ModuleStartException {
dispatcherDetectorListener = listener;
}

@Override
public void setStorageBuilderFactory(final StorageBuilderFactory factory) {
oalClassGenerator.setStorageBuilderFactory(factory);
}

@Override
public void start(ClassLoader currentClassLoader) throws ModuleStartException, OALCompileException {
if (!IS_RT_TEMP_FOLDER_INIT_COMPLETED) {
oalClassGenerator.prepareRTTempFolder();
IS_RT_TEMP_FOLDER_INIT_COMPLETED = true;
}

Reader read;
oalClassGenerator.setCurrentClassLoader(currentClassLoader);

try {
read = ResourceUtils.read(oalDefine.getConfigFile());
} catch (FileNotFoundException e) {
throw new ModuleStartException("Can't locate " + oalDefine.getConfigFile(), e);
}

OALScripts oalScripts;
try {
ScriptParser scriptParser = ScriptParser.createFromFile(read, oalDefine.getSourcePackage());
oalScripts = scriptParser.parse();
} catch (IOException e) {
throw new ModuleStartException("OAL script parse analysis failure.", e);
}

oalClassGenerator.generateClassAtRuntime(oalScripts, metricsClasses, dispatcherClasses);
}

@Override
public void notifyAllListeners() throws ModuleStartException {
for (Class metricsClass : metricsClasses) {
try {
streamAnnotationListener.notify(metricsClass);
} catch (StorageException e) {
throw new ModuleStartException(e.getMessage(), e);
}
}
for (Class dispatcherClass : dispatcherClasses) {
try {
dispatcherDetectorListener.addIfAsSourceDispatcher(dispatcherClass);
} catch (Exception e) {
throw new ModuleStartException(e.getMessage(), e);
}
}
}

protected List<Class> getMetricsClasses() {
return metricsClasses;
}

protected List<Class> getDispatcherClasses() {
return dispatcherClasses;
}
}
Loading

0 comments on commit a21905d

Please sign in to comment.