Skip to content

Commit

Permalink
Integrate create index with template
Browse files Browse the repository at this point in the history
  • Loading branch information
mgodwan committed Apr 23, 2024
1 parent 3733370 commit f417e4d
Show file tree
Hide file tree
Showing 8 changed files with 217 additions and 3 deletions.
9 changes: 8 additions & 1 deletion server/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,14 @@ dependencies {
api "com.google.protobuf:protobuf-java:${versions.protobuf}"
api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"

implementation "com.hubspot.jinjava:jinjava:2.7.2"
api "com.hubspot.jinjava:jinjava:2.7.2"
runtimeOnly "com.google.guava:guava:31.1-jre"
runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
runtimeOnly "org.apache.commons:commons-lang3:${versions.commonslang}"
runtimeOnly "com.google.re2j:re2j:1.7"


testImplementation(project(":test:framework")) {
// tests use the locally compiled version of server
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.opensearch.action.IndicesRequest;
import org.opensearch.action.admin.indices.alias.Alias;
import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.opensearch.action.admin.indices.template.contextaware.Context;
import org.opensearch.action.support.ActiveShardCount;
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.action.support.master.AcknowledgedRequest;
Expand Down Expand Up @@ -89,6 +90,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
public static final ParseField MAPPINGS = new ParseField("mappings");
public static final ParseField SETTINGS = new ParseField("settings");
public static final ParseField ALIASES = new ParseField("aliases");
public static final ParseField CONTEXT = new ParseField("context");

private String cause = "";

Expand All @@ -100,6 +102,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>

private final Set<Alias> aliases = new HashSet<>();

private Context context;

private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;

public CreateIndexRequest(StreamInput in) throws IOException {
Expand Down Expand Up @@ -383,6 +387,16 @@ public CreateIndexRequest aliases(Map<String, ?> source) {
}
}

public CreateIndexRequest context(Map<String, ?> source) {
try {
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.map(source);
return context(BytesReference.bytes(builder));
} catch (IOException e) {
throw new OpenSearchGenerationException("Failed to generate [" + source + "]", e);
}
}

/**
* Sets the aliases that will be associated with the index when it gets created
*/
Expand Down Expand Up @@ -414,6 +428,25 @@ public CreateIndexRequest aliases(BytesReference source) {
}
}

public CreateIndexRequest context(BytesReference source) {
// EMPTY is safe here because we never call namedObject
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source)) {
return context(Context.fromXContent(parser));
} catch (IOException e) {
throw new OpenSearchParseException("Failed to parse aliases", e);
}
}

public CreateIndexRequest context(Context context) {
this.context = context;
System.out.println(context);
return this;
}

public Context context() {
return context;
}

/**
* Adds an alias that will be associated with the index when it gets created
*/
Expand Down Expand Up @@ -524,6 +557,8 @@ public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler depre
}
} else if (ALIASES.match(name, deprecationHandler)) {
aliases((Map<String, Object>) entry.getValue());
} else if (CONTEXT.match(name, deprecationHandler)) {
context((Map<String, Object>) entry.getValue());
} else {
throw new OpenSearchParseException("unknown key [{}] for create index", name);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,27 @@

package org.opensearch.action.admin.indices.create;

import com.hubspot.jinjava.Jinjava;
import org.opensearch.action.support.ActionFilters;
import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.block.ClusterBlockException;
import org.opensearch.cluster.block.ClusterBlockLevel;
import org.opensearch.cluster.metadata.ContextTemplateMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.metadata.MetadataCreateIndexService;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.xcontent.MediaType;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;

import java.io.IOException;
import java.util.Map;

/**
* Create index action.
Expand Down Expand Up @@ -111,14 +117,27 @@ protected void clusterManagerOperation(
cause = "api";
}

Settings settingsToApply = request.settings();
if (request.context() != null) {
Jinjava jinjava = new Jinjava();
Map<String, Object> contextParams = request.context().params();
System.out.println("Params: " + contextParams);
String template = ((ContextTemplateMetadata) state.metadata().custom(ContextTemplateMetadata.TYPE)).getContextTemplates()
.get(request.context().name()).getSettings();
String appliedSettings = jinjava.render(template, contextParams);
System.out.println("Applied settings: " + appliedSettings);
Settings contextSettings = Settings.builder().loadFromSource(appliedSettings, XContentType.JSON).build();
settingsToApply = Settings.builder().put(contextSettings).put(request.settings()).build();
}

final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index());
final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(
cause,
indexName,
request.index()
).ackTimeout(request.timeout())
.masterNodeTimeout(request.clusterManagerNodeTimeout())
.settings(request.settings())
.settings(settingsToApply)
.mappings(request.mappings())
.aliases(request.aliases())
.waitForActiveShards(request.waitForActiveShards());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.action.admin.indices.template.contextaware;

import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ConstructingObjectParser;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

@PublicApi(since = "3.0.0")
public class Context implements Writeable, ToXContentFragment {
private static ParseField VERSION = new ParseField("version");
private static ParseField PARAMS = new ParseField("params");
private static ParseField NAME = new ParseField("name");

private final String name;
private final long version;
private final Map<String, Object> params;

public Context(String name, long version, Map<String, Object> params) {
this.name = name;
this.version = version;
this.params = params;
}

public Context(StreamInput in) throws IOException {
this.name = in.readString();
this.version = in.readLong();
this.params = in.readMap(StreamInput::readString, StreamInput::readString);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeLong(version);
out.writeMap(params, StreamOutput::writeString, (o, v) -> o.writeString(v.toString()));
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);

builder.field("version", version);

builder.startObject("params");
for (Map.Entry<String, Object> paramVal: this.params.entrySet()) {
builder.field(paramVal.getKey(), paramVal.getValue().toString());
}
builder.endObject();

builder.endObject();
return builder;
}

public static Context fromXContent(XContentParser parser) throws IOException {
String contextName = null;
long version = -1L;
Map<String, Object> params = null;

String currentFieldName = null;
XContentParser.Token token = parser.nextToken();
if (token == null) {
throw new IllegalArgumentException("No context is specified");
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (PARAMS.match(currentFieldName, parser.getDeprecationHandler())) {
params = parser.mapOrdered();
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
version = parser.longValue();
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if (NAME.match(currentFieldName, parser.getDeprecationHandler())) {
contextName = parser.text();
}
}
}

if (contextName == null || version < 0 || params == null) {
throw new IllegalArgumentException("Unable to parse context field");
}
return new Context(contextName, version, params);
}

public String name() {
return name;
}

public long version() {
return version;
}

public Map<String, Object> params() {
return params;
}

@Override
public String toString() {
return "Context{" +
"name='" + name + '\'' +
", version=" + version +
", params=" + params +
'}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@
@PublicApi(since = "3.0")
public class PutContextTemplateRequest extends ClusterManagerNodeRequest<PutContextTemplateRequest> implements IndicesRequest, ToXContentObject {

// Convert to a valid json
// TODO: Use as a valid json
private String settings;

private String name;

private String version = "-1";

public String settings() {
return settings;
}
Expand All @@ -36,18 +38,24 @@ public String name() {
return name;
}

public String version() {
return version;
}

public PutContextTemplateRequest() {
}

public PutContextTemplateRequest(StreamInput in) throws IOException {
this.name = in.readString();
this.settings = in.readString();
this.version = in.readString();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeString(settings);
out.writeString(version);
}


Expand All @@ -61,11 +69,17 @@ public PutContextTemplateRequest name(String name) {
return this;
}

public PutContextTemplateRequest version(String version) {
this.version = version;
return this;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field("name", name);
builder.field("settings", settings);
builder.field("version", version);
builder.endObject();
return null;
}
Expand All @@ -90,6 +104,7 @@ public String toString() {
return "PutContextTemplateRequest{" +
"settings='" + settings + '\'' +
", name='" + name + '\'' +
", version='" + version + '\'' +
'}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

package org.opensearch.rest.action.admin.indices;

import com.hubspot.jinjava.Jinjava;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.support.ActiveShardCount;
import org.opensearch.client.node.NodeClient;
Expand Down Expand Up @@ -106,4 +107,13 @@ static Map<String, Object> prepareMappings(Map<String, Object> source) {
newSource.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings));
return newSource;
}

public static void main(String[] args) {
String template = "\"index.codec\": \"best_compression\",\"index.refresh_interval\": \"120s\",\"index.merge.policy\": \"LOG_BYTE_SIZE\",\"index.replication.type\": \"{{ replication_type | default('SEGMENT') | tojson}}\"";
Jinjava jinjava = new Jinjava();
Map<String, Object> params = new HashMap<>();
params.put("replication_type", "DOCUMENT");

System.out.println(jinjava.render(template, params));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public String getName() {
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
String settings = (String) XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2()
.get("settings");
String version = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2()
.get("version").toString();
PutContextTemplateRequest putContextTemplateRequest = new PutContextTemplateRequest()
.name(request.param("name"))
.settings(settings);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,4 +187,6 @@ grant {
permission java.io.FilePermission "/sys/fs/cgroup/cpuacct/-", "read";
permission java.io.FilePermission "/sys/fs/cgroup/memory", "read";
permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read";
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";

};

0 comments on commit f417e4d

Please sign in to comment.