From 13933f615f1c021cd12c6cf55ee7de7906236bee Mon Sep 17 00:00:00 2001 From: zhangjunfeng Date: Fri, 19 Jul 2024 18:59:26 +0800 Subject: [PATCH] dist agent support --- .../modules/ignite-graph/pom.xml | 4 +- .../java/de/kp/works/ignite/IgniteUtil.java | 2 +- .../ignite/gremlin/IgniteConnection.java | 5 +- .../works/ignite/gremlin/IgniteElement.java | 6 +- .../kp/works/ignite/gremlin/IgniteGraph.java | 17 +- .../gremlin/IgniteGraphConfiguration.java | 26 +- .../ignite/gremlin/IgniteGraphUtils.java | 12 +- .../computer/TinkerGraphComputerView.java | 10 +- .../ignite/gremlin/readers/EdgeReader.java | 2 + .../ignite/gremlin/readers/VertexReader.java | 2 + .../ignite/gremlin/IgniteGraphManager.java | 2 +- .../plugin/GremlinServerPluginProvider.java | 81 +- .../modules/ignite-janus/pom.xml | 22 +- .../de/kp/works/ignite/IgniteContext.java | 20 +- .../de/kp/works/janus/IgniteStoreManager.java | 18 +- .../works/janus/IgniteStoreTransaction.java | 10 +- .../janus/gremlin/plugin/GremlinPlugin.java | 2 +- .../plugin/GremlinServerPluginProvider.java | 41 +- .../diskstorage/lucene/IgniteLuceneTest.java | 148 --- .../diskstorage/lucene/LuceneExample.java | 195 ---- .../diskstorage/lucene/LuceneIndexTest.java | 153 --- .../janusgraph/example/RemoteGraphUtil.java | 1 - .../connected-clusters-dialog/controller.js | 2 +- .../components/queries-notebook/controller.ts | 35 +- .../components/queries-notebook/style.scss | 4 +- .../queries-notebook/template.tpl.pug | 103 ++ .../permanent-notifications/controller.ts | 2 +- .../components/demo-mode-button/controller.ts | 19 +- .../modal-import-models/component.ts | 6 +- .../page-configure-basic/template.pug | 8 +- .../app/modules/agent/AgentManager.service.ts | 33 +- web-console/frontend/i18n/messages.en.json | 2 + web-console/frontend/i18n/messages.zh-CN.json | 2 + web-console/pom.xml | 1 + web-console/web-agent/pom.xml | 25 +- .../console/agent/AgentConfiguration.java | 30 +- .../console/agent/IgniteClusterLauncher.java | 60 +- .../commandline/CommandsProviderExtImpl.java | 16 +- .../console/agent/db/DataSourceManager.java | 145 ++- .../console/agent/db/DbMetadataReader.java | 11 +- .../console/agent/db/JdbcQueryExecutor.java | 3 - .../handlers/AbstractClusterHandler.java | 4 + .../agent/handlers/ClusterHandler.java | 2 + .../agent/handlers/ClustersWatcher.java | 4 +- .../agent/handlers/DatabaseHandler.java | 14 +- .../agent/handlers/DatabaseListener.java | 2 +- .../agent/handlers/DemoClusterHandler.java | 48 +- .../agent/handlers/RestClusterHandler.java | 40 +- .../agent/handlers/VertxClusterHandler.java | 162 ++- .../agent/handlers/WebSocketRouter.java | 161 ++- .../console/agent/rest/GremlinExecutor.java | 346 +++++++ .../console/agent/rest/GridTaskExecutor.java | 976 ++++++++++++++++++ .../console/agent/rest/JdbcExecutor.java | 22 +- .../console/agent/rest/RestExecutor.java | 3 +- .../ignite/console/agent/rest/RestResult.java | 10 + .../ignite/console/demo/AgentClusterDemo.java | 156 ++- .../console/websocket/WebSocketRequest.java | 20 + .../apache/ignite/console/dto/Notebook.java | 5 +- .../ignite/console/services/DemoService.java | 10 +- .../console/web/AbstractSocketHandler.java | 4 +- .../security/AuthenticationTokenFilter.java | 2 + .../console/web/socket/AgentsService.java | 33 +- .../console/web/socket/BrowsersService.java | 44 +- .../console/web/socket/TransitionService.java | 4 + .../console/web/socket/WebSocketConfig.java | 17 +- .../socket/WebSocketMessageBrokerConfig.java | 15 + .../src/main/resources/application.yml | 2 +- 67 files changed, 2407 insertions(+), 985 deletions(-) delete mode 100644 ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/IgniteLuceneTest.java delete mode 100644 ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneExample.java delete mode 100644 ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneIndexTest.java create mode 100644 web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/GremlinExecutor.java create mode 100644 web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/GridTaskExecutor.java create mode 100644 web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketMessageBrokerConfig.java diff --git a/ignite-extensions/modules/ignite-graph/pom.xml b/ignite-extensions/modules/ignite-graph/pom.xml index 998db70b3aabd..8433f5adac11d 100644 --- a/ignite-extensions/modules/ignite-graph/pom.xml +++ b/ignite-extensions/modules/ignite-graph/pom.xml @@ -30,8 +30,8 @@ 3.14.4 2.11.12 2.4.7 - 3.7.0 - 3.7.0 + 3.7.2 + 3.7.2 diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-common/src/main/java/de/kp/works/ignite/IgniteUtil.java b/ignite-extensions/modules/ignite-graph/works-ignite-common/src/main/java/de/kp/works/ignite/IgniteUtil.java index 17f18e9cfb252..20e0ad2165153 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-common/src/main/java/de/kp/works/ignite/IgniteUtil.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-common/src/main/java/de/kp/works/ignite/IgniteUtil.java @@ -67,7 +67,7 @@ public CacheConfiguration createCacheCfg(final String tabl final CacheConfiguration cfg = new CacheConfiguration<>(); cfg.setName(table); cfg.setStoreKeepBinary(false); - cfg.setIndexedTypes(new Class[] { String.class, BinaryObject.class }); + cfg.setCacheMode(cacheMode); cfg.setQueryEntities(qes); // add@byron diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteConnection.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteConnection.java index 26af72367dbe9..6c83fc6418f3c 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteConnection.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteConnection.java @@ -53,5 +53,8 @@ public IgniteConnection(String namespace,String cfg) { public IgniteAdmin getAdmin() { return admin; } - + + public String namespace() { + return admin.namespace(); + } } diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteElement.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteElement.java index 754e08937f866..ba616c32acfe9 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteElement.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteElement.java @@ -53,7 +53,7 @@ protected IgniteElement(IgniteGraph graph, Long updatedAt, Map properties, boolean propertiesFullyLoaded) { - this.namespace = graph.getIgniteGraphConfiguration().getGraphNamespace(); + this.namespace = graph.admin.namespace(); this.id = id; this.label = label; this.createdAt = createdAt; @@ -124,13 +124,13 @@ public boolean arePropertiesFullyLoaded() { return propertiesFullyLoaded; } - public void copyFrom(IgniteElement element) { + public void copyFrom(IgniteElement element) { if (element.label != null) this.label = element.label; if (element.createdAt != null) this.createdAt = element.createdAt; if (element.updatedAt != null) this.updatedAt = element.updatedAt; if (element.properties != null && (element.propertiesFullyLoaded || this.properties == null)) { - this.properties = new ConcurrentHashMap<>(element.properties); + this.properties = element.properties; this.propertiesFullyLoaded = element.propertiesFullyLoaded; } } diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraph.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraph.java index ad3b8f71947eb..081ec2071b975 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraph.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraph.java @@ -105,20 +105,20 @@ public class IgniteGraph implements Graph { * This method is invoked by Gremlin's GraphFactory * and defines the starting point for further tasks. */ - public static IgniteGraph open(final Configuration properties) throws IgniteGraphException { - return new IgniteGraph(properties); + public static IgniteGraph open(String name,Configuration properties) throws IgniteGraphException { + return new IgniteGraph(name,properties); } public static IgniteGraph getGraph(String name) throws IgniteGraphException { return graphInstances.get(name); } - public IgniteGraph(Configuration properties) { - this(new IgniteGraphConfiguration(properties)); + public IgniteGraph(String name,Configuration properties) { + this(new IgniteGraphConfiguration(name,properties)); } public IgniteGraph(IgniteGraphConfiguration config) throws IgniteGraphException { - this(config, IgniteGraphUtils.getConnection(config)); + this(config, IgniteGraphUtils.getConnection(config.getNamespace(),config)); } public IgniteGraph(IgniteGraphConfiguration config, IgniteConnection connection) throws IgniteGraphException { @@ -154,7 +154,7 @@ public IgniteGraph(IgniteGraphConfiguration config, IgniteConnection connection) .expireAfterAccess(config.getRelationshipCacheTtlSecs(), TimeUnit.SECONDS) .build(); - graphInstances.put(config.getGraphNamespace(), this); + graphInstances.put(connection.namespace(), this); } @@ -176,7 +176,7 @@ private void init() throws IgniteGraphException { IgniteGraphUtils.createTables(config, admin); } catch (Exception e) { - logger.error("Failed to create table for graph "+ config.getGraphNamespace(), e); + logger.error("Failed to create table for graph "+ admin.namespace(), e); throw new IgniteGraphException(e); } @@ -355,8 +355,7 @@ public Vertex vertex(Object id) { public Vertex findOrCreateVertex(Object id) { return findVertex(id, true); - } - + } /** diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphConfiguration.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphConfiguration.java index 9a066ca5e0da8..df3b1d9d04e79 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphConfiguration.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphConfiguration.java @@ -31,13 +31,13 @@ public class IgniteGraphConfiguration extends AbstractConfiguration implements S private static final long serialVersionUID = -7150699702127992270L; private final PropertiesConfiguration conf; + private final String namespace; public static final Class IGNITE_GRAPH_CLASS = IgniteGraph.class; public static final String IGNITE_GRAPH_CLASSNAME = IGNITE_GRAPH_CLASS.getCanonicalName(); - public static class Keys { - public static final String GRAPH_NAMESPACE = "gremlin.ignite.namespace"; + public static class Keys { public static final String GRAPH_CLASS = "gremlin.graph"; public static final String GRAPH_PROPERTY_TYPE = "gremlin.graph.propertyType"; public static final String GLOBAL_CACHE_MAX_SIZE = "gremlin.ignite.globalCacheMaxSize"; @@ -51,11 +51,13 @@ public static class Keys { * A minimal configuration for the IgniteGraph */ public IgniteGraphConfiguration() { + this.namespace = "default"; conf = new PropertiesConfiguration(); conf.setProperty(Keys.GRAPH_CLASS, IGNITE_GRAPH_CLASSNAME); } - public IgniteGraphConfiguration(Configuration config) { + public IgniteGraphConfiguration(String namespace,Configuration config) { + this.namespace = namespace; conf = new PropertiesConfiguration(); conf.setProperty(Keys.GRAPH_CLASS, IGNITE_GRAPH_CLASSNAME); if (config != null) { @@ -64,20 +66,6 @@ public IgniteGraphConfiguration(Configuration config) { } } - public String getGraphNamespace() { - return conf.getString(Keys.GRAPH_NAMESPACE, "default"); - } - - public IgniteGraphConfiguration setGraphNamespace(String name) { - if (!isValidGraphName(name)) { - throw new IllegalArgumentException("Invalid graph namespace." - + " Only alphanumerics and underscores are allowed"); - } - - conf.setProperty(Keys.GRAPH_NAMESPACE, name); - return this; - } - private static boolean isValidGraphName(String name) { return name.matches("^[A-Za-z0-9_]+$"); } @@ -133,4 +121,8 @@ protected void clearPropertyDirect(String key) { conf.clearProperty(key); } + public String getNamespace() { + return namespace; + } + } diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphUtils.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphUtils.java index 0f78cfdd8198e..fa0df2001f9ed 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphUtils.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphUtils.java @@ -38,26 +38,22 @@ public final class IgniteGraphUtils { private static final Map connections = new ConcurrentHashMap<>(); - public static IgniteConnection getConnection(IgniteGraphConfiguration config) { - - IgniteConnection conn; - - String namespace = config.getGraphNamespace(); + public static IgniteConnection getConnection(String namespace,IgniteGraphConfiguration config) { /* Check whether connection already exists */ - conn = connections.get(namespace); + IgniteConnection conn = connections.get(namespace); if (conn != null) return conn; String igniteCfg = config.getString("gremlin.graph.ignite.cfg"); conn = new IgniteConnection(namespace,igniteCfg); - connections.put(config.getGraphNamespace(), conn); + connections.put(namespace, conn); return conn; } public static String getTableName(IgniteGraphConfiguration config, String name) { - String ns = config.getGraphNamespace(); + String ns = config.getNamespace(); return ns + "_" + name; } diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputerView.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputerView.java index 3a5a9f412a48a..a31e318f24b39 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputerView.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputerView.java @@ -155,9 +155,8 @@ public Graph processResultGraphPersist(final GraphComputer.ResultGraph resultGra this.addPropertiesToOriginalGraph(); return this.graph; } else { - IgniteGraphConfiguration config = this.graph.configuration(); - config.setGraphNamespace(resultGraph.name()); - final IgniteGraph newGraph = IgniteGraph.open(config); + IgniteGraphConfiguration config = this.graph.configuration(); + final IgniteGraph newGraph = IgniteGraph.open(resultGraph.name(),config); this.graph.vertices().forEachRemaining(vertex -> { final Vertex newVertex = newGraph.addVertex(T.id, vertex.id(), T.label, vertex.label()); vertex.properties().forEachRemaining(vertexProperty -> { @@ -174,9 +173,8 @@ public Graph processResultGraphPersist(final GraphComputer.ResultGraph resultGra this.addPropertiesToOriginalGraph(); return this.graph; } else { - IgniteGraphConfiguration config = this.graph.configuration(); - config.setGraphNamespace(resultGraph.name()); - final IgniteGraph newGraph = IgniteGraph.open(config); + IgniteGraphConfiguration config = this.graph.configuration(); + final IgniteGraph newGraph = IgniteGraph.open(resultGraph.name(),config); this.graph.vertices().forEachRemaining(vertex -> { final Vertex newVertex = newGraph.addVertex(T.id, vertex.id(), T.label, vertex.label()); vertex.properties().forEachRemaining(vertexProperty -> { diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/EdgeReader.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/EdgeReader.java index af6cd3750b686..e9c48c1ab53a3 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/EdgeReader.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/EdgeReader.java @@ -63,6 +63,8 @@ public void load(Edge edge, IgniteResult result) { for (IgniteColumn column : result.getColumns()) { String colName = column.getColName(); switch (colName) { + case IgniteConstants.ID_COL_NAME: + break; case IgniteConstants.LABEL_COL_NAME: label = column.getColValue().toString(); break; diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/VertexReader.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/VertexReader.java index 7e7b8da2c9c0e..a48c3d991c1a7 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/VertexReader.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/readers/VertexReader.java @@ -59,6 +59,8 @@ public void load(Vertex vertex, IgniteResult result) { for (IgniteColumn column : result.getColumns()) { String colName = column.getColName(); switch (colName) { + case IgniteConstants.ID_COL_NAME: + break; case IgniteConstants.LABEL_COL_NAME: label = column.getColValue().toString(); break; diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphManager.java b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphManager.java index 49285db299b5e..a67e6bacf47b0 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphManager.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/IgniteGraphManager.java @@ -54,7 +54,7 @@ protected void addGraph(final String name, final String configurationFile) { try { Configurations configs = new Configurations(); - final Graph newGraph = IgniteGraph.open(configs.properties(configurationFile)); + final Graph newGraph = IgniteGraph.open(name,configs.properties(configurationFile)); putGraph(name, newGraph); logger.info("Graph [{}] was successfully configured via [{}].", name, configurationFile); diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/plugin/GremlinServerPluginProvider.java b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/plugin/GremlinServerPluginProvider.java index fe8fc5ab7b9ad..fe26a01455bc0 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/plugin/GremlinServerPluginProvider.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/src/main/java/de/kp/works/ignite/gremlin/plugin/GremlinServerPluginProvider.java @@ -44,6 +44,7 @@ import org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; + import org.jetbrains.annotations.Nullable; import de.kp.works.ignite.IgniteConnect; @@ -53,10 +54,12 @@ */ public class GremlinServerPluginProvider implements PluginProvider { - // One server per Instance - public GremlinServer gremlinServer; + // One server for all ignite Instance + public static GremlinServer gremlinServer; + + public static Settings settings; - public Settings settings; + private static int counter = 0; private String databaseName; @@ -76,7 +79,7 @@ public String name() { /** {@inheritDoc} */ @Override public String version() { - return "3.7.0"; + return "3.7.2"; } /** {@inheritDoc} */ @@ -114,8 +117,7 @@ public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { cfg = new GremlinPluginConfiguration(); } - boolean per = ignite.configuration().getDataStorageConfiguration().getDefaultDataRegionConfiguration() - .isPersistenceEnabled(); + boolean per = igniteCfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().isPersistenceEnabled(); if (cfg != null && per) { cfg.setPersistenceEnabled(true); } @@ -124,24 +126,27 @@ public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { try { databaseName = igniteCfg.getIgniteInstanceName(); gremlin.databaseName = databaseName; - - settings = Settings.read(cfg.getGremlinServerCfg()); + if(settings==null) { + settings = Settings.read(cfg.getGremlinServerCfg()); + } String configBase = ctx.igniteConfiguration().getIgniteHome() + "/config/gremlin-server"; - File graphFile = new File(configBase, "graph-" + databaseName + ".properties"); + File graphFile = new File(configBase, "ignite-" + databaseName + ".properties"); if (graphFile.exists()) { log.info(databaseName + "::load gremlim graph config file " + graphFile.getAbsolutePath()); gremlin.graphConfigFile = graphFile.getAbsolutePath(); } else { - graphFile = new File(configBase, "graph-default.properties"); + graphFile = new File(configBase, "ignite-default.properties"); log.info(databaseName + "::load default gremlim graph config file " + graphFile.getAbsolutePath()); gremlin.graphConfigFile = graphFile.getAbsolutePath(); - } + } + + settings.graphs.put(databaseName, gremlin.graphConfigFile); // 设置默认的ignite IgniteConnect.defaultIgnite = ignite; - + counter++; } catch (Exception e) { log.error(e.getMessage(), e); } @@ -164,19 +169,13 @@ public CachePluginProvider createCacheProvider(CachePluginContext ctx) { /** {@inheritDoc} */ @Override - public void start(PluginContext ctx) { - if (settings != null) { - - } - + public void start(PluginContext ctx) { } /** {@inheritDoc} */ @Override public void stop(boolean cancel) { - if (gremlinServer != null) { - - } + } /** {@inheritDoc} */ @@ -189,10 +188,9 @@ public void onIgniteStart() { if (workerPool == null) { workerPool = ((IgniteEx) IgniteConnect.defaultIgnite).context().pools().getServiceExecutorService(); } - gremlinServer = new GremlinServer(settings, workerPool); - gremlin.graphManager = gremlinServer.getServerGremlinExecutor().getGraphManager(); + gremlinServer = new GremlinServer(settings, workerPool); - CompletableFuture serverStarted = gremlinServer.start().thenAcceptAsync(GremlinServerPluginProvider::configure); + CompletableFuture serverStarted = gremlinServer.start().thenAcceptAsync(this::configure); serverStarted.join(); log.info("TinkerPop: {}\n", GremlinServer.getHeader()); @@ -206,25 +204,38 @@ public void onIgniteStart() { } - private static void configure(ServerGremlinExecutor serverGremlinExecutor) { - GraphManager graphManager = serverGremlinExecutor.getGraphManager(); - + private void configure(ServerGremlinExecutor serverGremlinExecutor) { + gremlin.graphManager = serverGremlinExecutor.getGraphManager(); } /** {@inheritDoc} */ @Override public void onIgniteStop(boolean cancel) { - if (gremlin.graphManager == null) { - return ; - } - try { - log.info("GremlinServer", "shutting down " + gremlinServer.toString()); - gremlinServer.stop(); - gremlinServer = null; - } catch (Exception e) { - log.error("GremlinServer close fail.", e); + + if(gremlin.graphManager!=null) { + try { + gremlin.graphManager.removeGraph(gremlin.databaseName); + } + catch (Exception e) { + e.printStackTrace(); + } + gremlin.graphManager = null; } + + if(cfg!=null) { + counter--; + if (counter<=0 && gremlinServer!= null) { + try { + log.info("GremlinServer", "shutting down " + gremlinServer.toString()); + gremlinServer.stop(); + gremlinServer = null; + } catch (Exception e) { + log.error("GremlinServer close fail.", e); + } + } + } + } /** {@inheritDoc} */ diff --git a/ignite-extensions/modules/ignite-janus/pom.xml b/ignite-extensions/modules/ignite-janus/pom.xml index 06d8951426d99..4f8384d7b21b7 100644 --- a/ignite-extensions/modules/ignite-janus/pom.xml +++ b/ignite-extensions/modules/ignite-janus/pom.xml @@ -12,9 +12,9 @@ UTF-8 2.16.999-SNAPSHOT - 1.0.0 + 1.1.0-20240712-100602.aaf21b1 8.11.2 - 3.7.0 + 3.7.2 @@ -58,15 +58,7 @@ org.apache.tinkerpop gremlin-console ${gremlin.version} - - - - - - com.github.jbellis - jamm - 0.3.0 - + org.apache.lucene @@ -78,13 +70,7 @@ lucene-backward-codecs ${lucene-solr.version} - - - org.janusgraph - janusgraph-test - ${janus.version} - test - + diff --git a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/ignite/IgniteContext.java b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/ignite/IgniteContext.java index 5f35c83fcca76..1d2a78b9c39de 100644 --- a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/ignite/IgniteContext.java +++ b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/ignite/IgniteContext.java @@ -76,24 +76,16 @@ private IgniteContext(String cfg,String name) { } catch(IgniteIllegalStateException e) { - if(cfg!=null) { - if(name==null || name.isBlank()) { - name = "ignite-bankend.cfg"; - } - IgniteConfiguration config = Ignition.loadSpringBean(cfg, name); - ignite = Ignition.start(config); - } - else { - throw e; + if(name==null || name.isBlank()) { + name = "ignite-bankend.cfg"; } + IgniteConfiguration config = Ignition.loadSpringBean(cfg, name); + ignite = Ignition.start(config); } } - // ignite not started - else if(cfg!=null) { - ignite = Ignition.start(cfg); - } else { - ignite = Ignition.start(); + // ignite not started + ignite = Ignition.start(cfg); } } diff --git a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreManager.java b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreManager.java index d8931e8b9d8a0..e6319ce3212bb 100644 --- a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreManager.java +++ b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreManager.java @@ -4,6 +4,8 @@ import de.kp.works.ignite.IgniteClient; import de.kp.works.ignite.IgniteContext; import org.apache.ignite.Ignite; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.transactions.Transaction; import org.janusgraph.diskstorage.BaseTransactionConfig; import org.janusgraph.diskstorage.StaticBuffer; import org.janusgraph.diskstorage.StoreMetaData.Container; @@ -40,7 +42,8 @@ public IgniteStoreManager(final Configuration configuration) { ignite = IgniteContext.getInstance(cfg,igniteNamespace).getIgnite(); namespace = ignite.name(); database = new IgniteClient(ignite); - + IgniteConfiguration igniteCfg = ignite.configuration(); + boolean persists = igniteCfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().isPersistenceEnabled(); /* * Initialize minimal store features; JanusGraph comes with * a wide range of additional features, so this is the place @@ -48,7 +51,7 @@ public IgniteStoreManager(final Configuration configuration) { */ features = new StandardStoreFeatures.Builder() .keyConsistent(configuration) - .persists(true) + .persists(persists) /* * If this flag is set to `false`, JanusGraph with do this * via [ExpectedValueCheckingStoreManager], which is less @@ -71,6 +74,7 @@ public IgniteStoreManager(final Configuration configuration) { */ .orderedScan(false) .unorderedScan(true) + .transactional(true) .build(); /* @@ -81,25 +85,21 @@ public IgniteStoreManager(final Configuration configuration) { } public StoreTransaction beginTransaction(BaseTransactionConfig config) { - return new IgniteStoreTransaction(config); + Transaction t = ignite.transactions().txStart(); + return new IgniteStoreTransaction(t, config); } public void close() { - for (IgniteStore store : stores.values()) { store.close(); } - - stores.clear(); - + stores.clear(); } public void clearStorage() { - for (IgniteStore store : stores.values()) { store.clear(); } - stores.clear(); } diff --git a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreTransaction.java b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreTransaction.java index bb76e2420c06e..e557882fcb35f 100644 --- a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreTransaction.java +++ b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/IgniteStoreTransaction.java @@ -18,6 +18,7 @@ * */ +import org.apache.ignite.transactions.Transaction; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BaseTransactionConfig; import org.janusgraph.diskstorage.common.AbstractStoreTransaction; @@ -25,19 +26,24 @@ * This class is introduced for future (additional) transaction support */ public class IgniteStoreTransaction extends AbstractStoreTransaction { + + final Transaction t; - public IgniteStoreTransaction(BaseTransactionConfig config) { + public IgniteStoreTransaction(Transaction t, BaseTransactionConfig config) { super(config); + this.t = t; } @Override public void commit() throws BackendException { + t.commit(); super.commit(); } @Override public void rollback() throws BackendException { - super.rollback(); + t.rollback(); + super.rollback(); } } diff --git a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinPlugin.java b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinPlugin.java index 6e4d789e4fcfe..7ac50edcf4748 100644 --- a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinPlugin.java +++ b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinPlugin.java @@ -8,7 +8,7 @@ public class GremlinPlugin implements IgnitePlugin{ String databaseName; - GraphManager graphManager; + GraphManager graphManager; public GraphManager getGraphManager() { diff --git a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinServerPluginProvider.java b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinServerPluginProvider.java index bd4f029bea2e8..03d427ecbf32e 100644 --- a/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinServerPluginProvider.java +++ b/ignite-extensions/modules/ignite-janus/src/main/java/de/kp/works/janus/gremlin/plugin/GremlinServerPluginProvider.java @@ -48,7 +48,6 @@ import org.janusgraph.core.JanusGraphFactory; import org.janusgraph.graphdb.server.JanusGraphServer; -import de.kp.works.ignite.IgniteContext; /** @@ -65,7 +64,7 @@ public class GremlinServerPluginProvider implements PluginProvider { + log.error("JanusGraph Server was unable to start and will now begin shutdown", t); janusGraphServer.stop().join(); return null; + }).join(); gremlin.graphManager = janusGraphServer.getGremlinServer().getServerGremlinExecutor().getGraphManager(); @@ -209,24 +210,26 @@ public void onIgniteStart() { /** {@inheritDoc} */ @Override - public void onIgniteStop(boolean cancel) { - if(cfg!=null) { - counter--; - } + public void onIgniteStop(boolean cancel) { if(gremlin.graphManager!=null) { - JanusGraph graph = (JanusGraph)gremlin.graphManager.getGraph(databaseName); - gremlin.graphManager = null; - if(graph!=null) { - graph.close(); + try { + gremlin.graphManager.removeGraph(gremlin.databaseName); + } + catch (Exception e) { + } + gremlin.graphManager = null; } - if (janusGraphServer!= null && counter<=0) { - log.info("JanusGremlinServer", "shutting down " + janusGraphServer.toString()); - janusGraphServer.stop(); - janusGraphServer = null; - } + if(cfg!=null) { + counter--; + if (janusGraphServer!= null && counter<=0) { + log.info("JanusGremlinServer", "shutting down " + janusGraphServer.toString()); + janusGraphServer.stop(); + janusGraphServer = null; + } + } } /** {@inheritDoc} */ diff --git a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/IgniteLuceneTest.java b/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/IgniteLuceneTest.java deleted file mode 100644 index 0d6f46a8adde6..0000000000000 --- a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/IgniteLuceneTest.java +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph.diskstorage.lucene; - -import org.apache.commons.codec.digest.DigestUtils; -import org.janusgraph.StorageSetup; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; -import org.janusgraph.diskstorage.configuration.WriteConfiguration; -import org.janusgraph.example.GraphOfTheGodsFactory; -import org.janusgraph.graphdb.JanusGraphIndexTest; -import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; -import org.junit.jupiter.api.Test; - - -import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_BACKEND; -import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_DIRECTORY; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author Matthias Broecheler (me@matthiasb.com) - */ -public class IgniteLuceneTest extends JanusGraphIndexTest { - - public IgniteLuceneTest() { - super(true, true, true); - } - - @Override - public WriteConfiguration getConfiguration() { - ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); - for (String indexBackend : getIndexBackends()) { - config.set(INDEX_BACKEND, "lucene", indexBackend); - config.set(INDEX_DIRECTORY, StorageSetup.getHomeDir("lucene"), indexBackend); - } - return config.getConfiguration(); - } - - @Override - public boolean supportsLuceneStyleQueries() { - return true; - } - - @Override - public boolean supportsGeoPointExistsQuery() { - return false; - } - - @Override - public boolean supportsWildcardQuery() { - return false; - } - - @Override - protected boolean supportsCollections() { - return true; - } - - @Override - protected boolean supportsGeoCollections() { - return false; - } - - @Test - public void testPrintSchemaElements() { - GraphOfTheGodsFactory.load(graph); - mgmt = graph.openManagement(); - - String expected = "026DCB18B35C198CC54FA66B3D86C3FB"; - String output = mgmt.printSchema(); - String outputHash = DigestUtils.md5Hex(output).toUpperCase(); - assertEquals(expected, outputHash); - - String expectedVertexLabels = - "------------------------------------------------------------------------------------------------\n" + - "Vertex Label Name | Partitioned | Static |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "titan | false | false |\n" + - "location | false | false |\n" + - "god | false | false |\n" + - "demigod | false | false |\n" + - "human | false | false |\n" + - "monster | false | false |\n" + - "---------------------------------------------------------------------------------------------------\n"; - assertEquals(expectedVertexLabels, mgmt.printVertexLabels()); - - String expectedEdgeLabels = - "------------------------------------------------------------------------------------------------\n" + - "Edge Label Name | Directed | Unidirected | Multiplicity |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "brother | true | false | MULTI |\n" + - "father | true | false | MANY2ONE |\n" + - "mother | true | false | MANY2ONE |\n" + - "battled | true | false | MULTI |\n" + - "lives | true | false | MULTI |\n" + - "pet | true | false | MULTI |\n" + - "---------------------------------------------------------------------------------------------------\n"; - assertEquals(expectedEdgeLabels, mgmt.printEdgeLabels()); - - String expectedPropertyKeys = - "------------------------------------------------------------------------------------------------\n" + - "Property Key Name | Cardinality | Data Type |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "name | SINGLE | class java.lang.String |\n" + - "age | SINGLE | class java.lang.Integer |\n" + - "time | SINGLE | class java.lang.Integer |\n" + - "reason | SINGLE | class java.lang.String |\n" + - "place | SINGLE | class org.janusgraph.core.attribute.Geoshape |\n" + - "---------------------------------------------------------------------------------------------------\n"; - assertEquals(expectedPropertyKeys, mgmt.printPropertyKeys()); - - String expectedIndexes = - "------------------------------------------------------------------------------------------------\n" + - "Graph Index (Vertex) | Type | Unique | Backing | Key: Status |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "name | Composite | true | internalindex | name: ENABLED |\n" + - "vertices | Mixed | false | search | age: ENABLED |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "Graph Index (Edge) | Type | Unique | Backing | Key: Status |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "edges | Mixed | false | search | reason: ENABLED |\n" + - " | | | | place: ENABLED |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "Relation Index (VCI) | Type | Direction | Sort Key | Order | Status |\n" + - "---------------------------------------------------------------------------------------------------\n" + - "battlesByTime | battled | BOTH | time | desc | ENABLED |\n" + - "---------------------------------------------------------------------------------------------------\n"; - assertEquals(expectedIndexes, mgmt.printIndexes()); - } - - @Override - public boolean supportsGeoShapePrefixTreeMapping() { - // TODO Auto-generated method stub - return false; - } - -} diff --git a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneExample.java b/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneExample.java deleted file mode 100644 index 2927b38a03f89..0000000000000 --- a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneExample.java +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph.diskstorage.lucene; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.SpatialStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.apache.lucene.spatial.query.SpatialArgs; -import org.apache.lucene.spatial.query.SpatialOperation; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.janusgraph.core.attribute.Geoshape; -import org.janusgraph.util.system.IOUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Shape; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author Matthias Broecheler (me@matthiasb.com) - */ - -public abstract class LuceneExample { - - public static final File path = new File("/tmp/lucene"); - private static final String STR_SUFFIX = "_str"; - private static final String TXT_SUFFIX = "_txt"; - - private static final int MAX_RESULT = 10000; - - private final Map spatial= new HashMap<>(); - private final SpatialContext ctx = SpatialContext.GEO; - - @BeforeEach - public void setup() { - if (path.exists()) IOUtils.deleteDirectory(path,false); - if (!path.exists() && path.isDirectory()) { - Preconditions.checkState(path.mkdirs()); - } - } - - private SpatialStrategy getSpatialStrategy(String key) { - SpatialStrategy strategy = spatial.get(key); - if (strategy==null) { - final int maxLevels = 11; - SpatialPrefixTree grid = new GeohashPrefixTree(ctx, maxLevels); - strategy = new RecursivePrefixTreeStrategy(grid, key); - spatial.put(key,strategy); - } - return strategy; - } - - @Test - public void example1() throws Exception { - Directory dir = FSDirectory.open(path.toPath()); - Analyzer analyzer = new StandardAnalyzer(); - IndexWriterConfig iwc = new IndexWriterConfig(analyzer); - - iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - IndexWriter writer = new IndexWriter(dir, iwc); - - indexDocs(writer, "doc1", ImmutableMap.of("name","The laborious work of John Doe as we know it", - "city","Blumenkamp", - "location",Geoshape.point(51.687882,6.612053), - "time",1000342034 - )); - - indexDocs(writer, "doc2", ImmutableMap.of("name","Life as we know it or not", - "city","Essen", - "location",Geoshape.point(51.787882,6.712053), - "time",1000342034-500 - )); - - indexDocs(writer, "doc3", ImmutableMap.of("name","Berlin - poor but sexy and a display of the extraordinary", - "city","Berlin", - "location",Geoshape.circle(52.509535, 13.425293, 50), - "time",1000342034+2000 - )); - writer.close(); - - //Search - IndexReader reader = DirectoryReader.open(FSDirectory.open(path.toPath())); - IndexSearcher searcher = new IndexSearcher(reader); - - //Auesee - BooleanQuery.Builder filter = new BooleanQuery.Builder(); - - SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects,Geoshape.circle(51.666167,6.58905,450).getShape()); - - filter.add(LongPoint.newRangeQuery("time", 1000342034, 1000342034), BooleanClause.Occur.MUST); - - - filter.add(new PrefixQuery(new Term("city_str","B")), BooleanClause.Occur.MUST); - - - BooleanQuery.Builder qb = new BooleanQuery.Builder(); - qb.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD); - qb.add(filter.build(), BooleanClause.Occur.FILTER); - TopDocs docs = searcher.search(qb.build(), MAX_RESULT); - if (docs.totalHits.value>=MAX_RESULT) throw new RuntimeException("Max results exceeded: " + MAX_RESULT); - - Set result = getResults(searcher,docs); - System.out.println(result); - - } - - private Set getResults(IndexSearcher searcher, TopDocs docs) throws IOException { - Set found = Sets.newHashSet(); - for (int i = 0; i < docs.totalHits.value; i++) { - found.add(searcher.doc(docs.scoreDocs[i].doc).getField("docid").stringValue()); - } - return found; - } - - void indexDocs(IndexWriter writer, String documentId, Map docMap) throws IOException { - Document doc = new Document(); - - Field documentIdField = new StringField("docid", documentId, Field.Store.YES); - doc.add(documentIdField); - - for (Map.Entry kv : docMap.entrySet()) { - String key = kv.getKey(); - Object value = kv.getValue(); - - if (value instanceof Number) { - final Field field; - if (value instanceof Integer || value instanceof Long) { - field = new LongPoint(key, ((Number)value).longValue()); - } else { //double or float - field = new DoublePoint(key, ((Number)value).doubleValue()); - } - doc.add(field); - } else if (value instanceof String) { - String str = (String)value; - Field field = new TextField(key+ TXT_SUFFIX, str, Field.Store.NO); - doc.add(field); - if (str.length()<256) - field = new StringField(key+STR_SUFFIX, str, Field.Store.NO); - doc.add(field); - } else if (value instanceof Geoshape) { - Shape shape = ((Geoshape)value).getShape(); - for (IndexableField f : getSpatialStrategy(key).createIndexableFields(shape)) { - doc.add(f); - } - } else throw new IllegalArgumentException("Unsupported type: " + value); - } - - writer.updateDocument(new Term("docid", documentId), doc); - - } - - -} diff --git a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneIndexTest.java b/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneIndexTest.java deleted file mode 100644 index ac5e1040a8c2a..0000000000000 --- a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/diskstorage/lucene/LuceneIndexTest.java +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2017 JanusGraph Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.janusgraph.diskstorage.lucene; - -import org.janusgraph.StorageSetup; -import org.janusgraph.core.Cardinality; -import org.janusgraph.core.attribute.Cmp; -import org.janusgraph.core.attribute.Geo; -import org.janusgraph.core.attribute.Geoshape; -import org.janusgraph.core.attribute.Text; -import org.janusgraph.core.schema.Mapping; -import org.janusgraph.core.schema.Parameter; -import org.janusgraph.diskstorage.BackendException; -import org.janusgraph.diskstorage.configuration.Configuration; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; -import org.janusgraph.diskstorage.indexing.IndexProvider; -import org.janusgraph.diskstorage.indexing.IndexProviderTest; -import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Date; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author Matthias Broecheler (me@matthiasb.com) - */ - -public class LuceneIndexTest extends IndexProviderTest { - - private static final Logger log = LoggerFactory.getLogger(LuceneIndexTest.class); - - private static char REPLACEMENT_CHAR = '\u2022'; - private static final String MAPPING = "mapping"; - - @Override - public IndexProvider openIndex() throws BackendException { - return new LuceneIndex(getLocalLuceneTestConfig()); - } - - @Override - public boolean supportsLuceneStyleQueries() { - return true; - } - - @Override - public String getEnglishAnalyzerName() { - return org.apache.lucene.analysis.en.EnglishAnalyzer.class.getName(); - } - - @Override - public String getKeywordAnalyzerName() { - return org.apache.lucene.analysis.core.KeywordAnalyzer.class.getName(); - } - - public static Configuration getLocalLuceneTestConfig() { - final String index = "lucene"; - ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); - config.set(GraphDatabaseConfiguration.INDEX_DIRECTORY, StorageSetup.getHomeDir("lucene"),index); - return config.restrictTo(index); - } - - @Test - public void testSupport() { - // DEFAULT(=TEXT) support - assertTrue( index.supports(of(String.class, Cardinality.SINGLE), Text.CONTAINS)); - assertTrue( index.supports(of(String.class, Cardinality.SINGLE), Text.CONTAINS_PREFIX)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE), Text.CONTAINS_REGEX)); // TODO Not supported yet - assertFalse(index.supports(of(String.class, Cardinality.SINGLE), Text.REGEX)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE), Text.PREFIX)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE), Cmp.EQUAL)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); - - // Same tests as above, except explicitly specifying TEXT instead of relying on DEFAULT - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.CONTAINS)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.CONTAINS_PREFIX)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.CONTAINS_FUZZY)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.CONTAINS_REGEX)); // TODO Not supported yet - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.REGEX)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Text.PREFIX)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Cmp.EQUAL)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.TEXT)), Cmp.NOT_EQUAL)); - - // STRING support - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Text.CONTAINS)); - assertFalse(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Text.CONTAINS_PREFIX)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Text.REGEX)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Text.PREFIX)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Text.FUZZY)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Cmp.EQUAL)); - assertTrue(index.supports(of(String.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.STRING)), Cmp.NOT_EQUAL)); - - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.EQUAL)); - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.LESS_THAN_EQUAL)); - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.LESS_THAN)); - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.GREATER_THAN)); - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.GREATER_THAN_EQUAL)); - assertTrue(index.supports(of(Date.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); - - assertTrue(index.supports(of(Boolean.class, Cardinality.SINGLE), Cmp.EQUAL)); - assertTrue(index.supports(of(Boolean.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); - - assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.EQUAL)); - assertTrue(index.supports(of(UUID.class, Cardinality.SINGLE), Cmp.NOT_EQUAL)); - - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE))); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.WITHIN)); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.INTERSECT)); - assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE), Geo.DISJOINT)); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.PREFIX_TREE)), Geo.WITHIN)); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.PREFIX_TREE)), Geo.CONTAINS)); - assertTrue(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.PREFIX_TREE)), Geo.INTERSECT)); - assertFalse(index.supports(of(Geoshape.class, Cardinality.SINGLE, new Parameter<>(MAPPING, Mapping.PREFIX_TREE)), Geo.DISJOINT)); - } - - - @Test - public void testMapKey2Field_IllegalCharacter() { - assertThrows(IllegalArgumentException.class, () ->{ - index.mapKey2Field("here is an illegal character: " + REPLACEMENT_CHAR, null); - }); - } - - @Test - public void testMapKey2Field_MappingSpaces() { - String expected = "field" + REPLACEMENT_CHAR + "name" + REPLACEMENT_CHAR + "with" + REPLACEMENT_CHAR + "spaces"; - assertEquals(expected, index.mapKey2Field("field name with spaces", null)); - } - - @Override - public Mapping preferredGeoShapeMapping() { - // TODO Auto-generated method stub - return null; - } -} diff --git a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/example/RemoteGraphUtil.java b/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/example/RemoteGraphUtil.java index 3f2f8c59a4d33..5e22e4d667b62 100644 --- a/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/example/RemoteGraphUtil.java +++ b/ignite-extensions/modules/ignite-janus/src/test/java/org/janusgraph/example/RemoteGraphUtil.java @@ -16,7 +16,6 @@ import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.tinkerpop.gremlin.GraphManager; import org.apache.tinkerpop.gremlin.driver.Client; import org.apache.tinkerpop.gremlin.driver.Cluster; import org.apache.tinkerpop.gremlin.driver.Result; diff --git a/web-console/frontend/app/components/connected-clusters-dialog/controller.js b/web-console/frontend/app/components/connected-clusters-dialog/controller.js index be7af39963788..5b29b6b8910c0 100644 --- a/web-console/frontend/app/components/connected-clusters-dialog/controller.js +++ b/web-console/frontend/app/components/connected-clusters-dialog/controller.js @@ -5,7 +5,7 @@ export default class { constructor(clusters) { this.clusters = _.map(clusters, (cluster) => - _.merge({}, cluster, { size: _.size(cluster.nodes) }) + _.merge({}, cluster, { size: _.size(cluster?cluster.nodes:[]) }) ); } } diff --git a/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts b/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts index 5ad5bc7ba4ddc..88f58e1bd5548 100644 --- a/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts +++ b/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.ts @@ -61,7 +61,7 @@ let paragraphId = 0; class Paragraph { name: string; - queryType: 'SCAN' | 'SQL_FIELDS'; + queryType: 'SCAN' | 'SQL_FIELDS' | 'GREMLIN'; constructor($animate, $timeout, JavaTypes, errorParser, paragraph, private $translate: ng.translate.ITranslateService) { const self = this; @@ -1121,6 +1121,28 @@ export class NotebookCtrl { $scope.addParagraph(paragraph, sz); }; + $scope.addGremlin = function() { + const sz = $scope.notebook.paragraphs.length; + + ActivitiesData.post({ group: 'sql', action: '/queries/add/gremlin' }); + + const paragraph = _newParagraph({ + name: $translate.instant('queries.notebook.newGremlinNamePrefix') + (sz === 0 ? '' : sz), + query: "g.V().property('label', 'value')", + pageSize: $scope.pageSizesOptions[1].value, + timeLineSpan: $scope.timeLineSpans[0], + result: 'NONE', + rate: { + value: 1, + unit: 60000, + installed: false + }, + queryType: 'GREMLIN' + }); + + $scope.addParagraph(paragraph, sz); + }; + function _saveChartSettings(paragraph) { if (!_.isEmpty(paragraph.charts)) { const chart = paragraph.charts[0].api.getScope().chart; @@ -1276,7 +1298,16 @@ export class NotebookCtrl { * @return {Observable} Observable with first query result page. */ const _executeQuery0 = (queryType, qryArg, onQueryStarted: (res) => any = () => {}) => { - return from(queryType === 'SCAN' ? agentMgr.queryScan(qryArg) : agentMgr.querySql(qryArg)).pipe( + let query = null; + if (queryType === 'SCAN'){ + query = agentMgr.queryScan(qryArg); + } + else if (queryType === 'GREMLIN'){ + query = agentMgr.queryGremlin(qryArg); + } + else + query = agentMgr.querySql(qryArg); + return from(query).pipe( tap((res) => { onQueryStarted(res); $scope.$applyAsync(); diff --git a/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss b/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss index 3b4387c71ce2d..39ccca7cd0ba2 100644 --- a/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss +++ b/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss @@ -104,8 +104,8 @@ queries-notebook { align-items: center; margin-left: auto; - button:first-of-type { - margin-right: 20px; + button { + margin-left: 20px; } a { diff --git a/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug index 7396ce91c478c..f1631e8940566 100644 --- a/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug +++ b/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug @@ -527,6 +527,98 @@ mixin paragraph-query i.fa.fa-chevron-circle-right a(translate='queries.notebook.queryResult.nextButtonLabel') +mixin paragraph-gremlin + panel-title {{ paragraph.name }} + panel-actions + query-actions-button(actions='$ctrl.queryActions' item='paragraph') + panel-content + .col-sm-12 + .col-xs-8.col-sm-9(style='border-right: 1px solid #eee') + .sql-editor(ignite-ace='{onLoad: aceInit(paragraph), theme: "chrome", mode: "groovy", require: ["ace/ext/language_tools"],' + + 'advanced: {enableSnippets: false, enableBasicAutocompletion: false, enableLiveAutocompletion: false}}' + ng-model='paragraph.query' on-selection-change='paragraph.partialQuery = $event') + .col-xs-4.col-sm-3 + div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches') + lable.labelField.labelFormField(translate='queries.notebook.queryResult.cachesMetadataInput.label') + i.fa.fa-database.tipField( + translate-attr='::{title: "queries.notebook.queryResult.cachesMetadataInput.metadataButtonTooltip"}' + bs-popover + data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' + data-placement='bottom-right' + data-trigger='click' + data-container=`#{{ paragraph.id }}` + ) + .input-tip + input.form-control( + type='text' + st-search='label' + translate-attr='::{placeholder: "queries.notebook.queryResult.cachesMetadataInput.filterPlaceholder"}' + ) + + .queries-notebook-displayed-caches + div(ng-repeat='cache in displayedCaches track by cache.value') + +form-field__radio({ + label: '{{ cache.label }}', + model: 'paragraph.cacheName', + name: '"cache_" + [paragraph.id, $index].join("_")', + value: 'cache.value' + }) + + .empty-caches(ng-show='displayedCaches.length == 0 && caches.length != 0') + no-data + label(translate='queries.notebook.queryResult.wrongCachesFilterError') + + .settings-row + .row(ng-if='displayedCaches.length > 0 && ddlAvailable(paragraph)') + +form-field__checkbox({ + label: '{{::"queries.notebook.queryResult.useAsDefaultSchemaInput.label"|translate}}', + model: 'paragraph.useAsDefaultSchema', + name: '"useAsDefaultSchema" + paragraph.id', + tip: '{{::"queries.notebook.queryResult.useAsDefaultSchemaInput.tooltip"|translate}}', + tipOpts: { placement: 'top' } + }) + .empty-caches(ng-show='caches.length == 0') + no-data(handle-cluster-inactive='true') + label(translate='queries.notebook.queryResult.noCaches') + .col-sm-12.sql-controls + div + +query-actions + + +query-settings + .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()') + .error(ng-switch-when='error') + label( + translate='queries.notebook.queryResult.genericError' + translate-values='{error: paragraph.error.message}' + ) + br + a( + ng-show='paragraph.resultType() === "error"' + ng-click='showStackTrace(paragraph)' + translate='queries.notebook.queryResult.showMoreButtonLabel' + ) + .empty( + ng-switch-when='empty' + translate='queries.notebook.queryResult.emptyResult' + translate-values='{duration: (paragraph.duration | duration)}' + ) + .table(ng-switch-when='table') + +table-result-heading-query + +table-result-body + .chart(ng-switch-when='chart') + +chart-result + .footer.clearfix(ng-show='paragraph.resultType() !== "error"') + a.pull-left( + ng-click='showResultQuery(paragraph)' + translate='queries.notebook.queryResult.showQueryButtonLabel' + ) + + -var nextVisibleCondition = 'paragraph.resultType() !== "error" && !paragraph.loading && paragraph.queryId && paragraph.nonRefresh() && (paragraph.table() || paragraph.chart() && !paragraph.scanExplain())' + + .pull-right(ng-show=`${nextVisibleCondition}` ng-class='{disabled: paragraph.loading}' ng-click='!paragraph.loading && nextPage(paragraph)') + i.fa.fa-chevron-circle-right + a(translate='queries.notebook.queryResult.nextButtonLabel') + div(ng-if='notebook') .notebooks-top h1(ng-hide='notebook.edit') @@ -562,6 +654,10 @@ div(ng-if='notebook') svg.icon-left(ignite-icon='plus') span(translate='queries.notebook.addScanButtonLabel') + button.btn-ignite.btn-ignite--primary(ng-click='addGremlin()' ignite-on-click-focus=focusId) + svg.icon-left(ignite-icon='plus') + span(translate='queries.notebook.addGremlinButtonLabel') + div breadcrumbs a.link-success( @@ -626,3 +722,10 @@ div on-open='$ctrl.onParagraphOpen($index)' ) +paragraph-query + panel-collapsible( + ng-if='paragraph.queryType === "GREMLIN"' + opened='$ctrl.isParagraphOpened($index)' + on-close='$ctrl.onParagraphClose($index)' + on-open='$ctrl.onParagraphOpen($index)' + ) + +paragraph-gremlin diff --git a/web-console/frontend/app/components/permanent-notifications/controller.ts b/web-console/frontend/app/components/permanent-notifications/controller.ts index 32e387dfa7d16..17d66315f88e1 100644 --- a/web-console/frontend/app/components/permanent-notifications/controller.ts +++ b/web-console/frontend/app/components/permanent-notifications/controller.ts @@ -21,7 +21,7 @@ export default class PermanentNotifications { closeDemo() { this.$window.close(); - this.AgentManager.stopCluster({id:'demo',name:'demo-server',demo:true}) + this.AgentManager.stopCluster({id:'demo-server',name:'demo-server',demo:true}) } revertIdentity() { diff --git a/web-console/frontend/app/components/web-console-header/components/demo-mode-button/controller.ts b/web-console/frontend/app/components/web-console-header/components/demo-mode-button/controller.ts index 8fbb63a57978e..a07b50370eaaf 100644 --- a/web-console/frontend/app/components/web-console-header/components/demo-mode-button/controller.ts +++ b/web-console/frontend/app/components/web-console-header/components/demo-mode-button/controller.ts @@ -29,18 +29,31 @@ export default class DemoModeButton { const user = await this.User.current$.pipe(take(1)).toPromise(); if (disconnected || demoEnabled || _.isNil(demoEnabled)) { - if (!user.demoCreated) + if (!user.demoCreated){ + this.clusterStart(); return this._openTab('demo.reset'); + } this.Confirm.confirm(this.$translate.instant('demoModeButton.continueConfirmationMessage'), true, false) .then((resume) => { - if (resume) + if (resume){ return this._openTab('demo.resume'); - + } this._openTab('demo.reset'); }); } else this.Messages.showError(this.$translate.instant('demoModeButton.demoModeDisabledErrorMessage')); } + + clusterStart() { + this.agentMgr.startCluster({id:'demo-server',name:'demo-server',demo:true}).then((msg) => { + if(msg.message){ + this.Messages.showError(msg.message); + } + }) + .catch((e) => { + this.Messages.showError(e.message); + }); + } } diff --git a/web-console/frontend/app/configuration/components/modal-import-models/component.ts b/web-console/frontend/app/configuration/components/modal-import-models/component.ts index f2db68e9a740e..76b3e1fc26924 100644 --- a/web-console/frontend/app/configuration/components/modal-import-models/component.ts +++ b/web-console/frontend/app/configuration/components/modal-import-models/component.ts @@ -846,7 +846,11 @@ export class ModalImportModels { return; const act = $scope.importDomain.action; - + // add@byron + if($scope.ui.selectedJdbcDriverJar>=0){ + $scope.selectedPreset.jdbcDriverJar = $scope.jdbcDriverJars[$scope.ui.selectedJdbcDriverJar].label + } + // end@ if (act === 'drivers' && $scope.importDomain.jdbcDriversNotFound) this.onHide(); else if (act === 'connect') diff --git a/web-console/frontend/app/console/components/page-configure-basic/template.pug b/web-console/frontend/app/console/components/page-configure-basic/template.pug index 5b11c9e071563..39ce49c48dc5a 100644 --- a/web-console/frontend/app/console/components/page-configure-basic/template.pug +++ b/web-console/frontend/app/console/components/page-configure-basic/template.pug @@ -25,7 +25,7 @@ form(novalidate name=form) name: '"clusterName"', placeholder: 'Input name', required: true, - disable: true, + disabled: true, tip: 'Instance name allows to indicate to what grid this particular grid instance belongs to' })( ignite-unique='$ctrl.shortClusters' @@ -42,7 +42,7 @@ form(novalidate name=form) name: '"discovery"', placeholder: 'Choose discovery', options: '$ctrl.Clusters.discoveries', - disable: true, + disabled: true, tip: 'Discovery allows to discover remote nodes in grid\
    \
  • Static IPs - IP Finder which works only with pre configured list of IP addresses specified
    • \ @@ -77,7 +77,7 @@ form(novalidate name=form) label: 'Name', model: '$item.name', name: '"name"', - disable: true, + disabled: true, required: true })( ignite-unique='$ctrl.shortCaches' @@ -98,6 +98,7 @@ form(novalidate name=form) model: '$item.atomicityMode', name: '"atomicityMode"', placeholder: 'ATOMIC', + disabled: true, options: '::$ctrl.Caches.atomicityModes' }) div(ng-show='$ctrl.Caches.shouldShowCacheBackupsCount($item)') @@ -106,6 +107,7 @@ form(novalidate name=form) model: '$item.backups', name: '"backups"', placeholder: '0', + disabled: true, min: 0 }) diff --git a/web-console/frontend/app/modules/agent/AgentManager.service.ts b/web-console/frontend/app/modules/agent/AgentManager.service.ts index 7313bffba512c..d48ef2355c3e0 100644 --- a/web-console/frontend/app/modules/agent/AgentManager.service.ts +++ b/web-console/frontend/app/modules/agent/AgentManager.service.ts @@ -544,7 +544,7 @@ export default class AgentManager { throw new Error('Access denied. You are not authorized to access this functionality.'); default: - throw new Error('Illegal status in node response'); + throw new Error('Illegal status in node response:' + res.error); } }); } @@ -868,6 +868,37 @@ export default class AgentManager { } } + /** + * @param {String} nid Node id. + * @param {String} cacheName Cache name. + * @param {String} [query] Query if null then scan query. + * @param {Boolean} nonCollocatedJoins Flag whether to execute non collocated joins. + * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled. + * @param {Boolean} replicatedOnly Flag whether query contains only replicated tables. + * @param {Boolean} local Flag whether to execute query locally. + * @param {Number} pageSize + * @param {Boolean} [lazy] query flag. + * @param {Boolean} [collocated] Collocated query. + * @returns {Promise.} Query execution result. + */ + queryGremlin({nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSize, lazy = false, collocated = false}) { + if (this.available(IGNITE_2_0)) { + let args = {cacheName, qry:query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSize, lazy, collocated}; + + return this.visorTask('queryGremlin', nid, args).then((data) => { + if (!('error' in data) || !(data.error)){ + if('result' in data){ + return data.result + } + return data; + } + + return Promise.reject(data.error); + }); + } + + } + /** * Change cluster active state. * diff --git a/web-console/frontend/i18n/messages.en.json b/web-console/frontend/i18n/messages.en.json index e6fbdbbc6d649..cb20c8c629b72 100644 --- a/web-console/frontend/i18n/messages.en.json +++ b/web-console/frontend/i18n/messages.en.json @@ -106,6 +106,7 @@ "loadingMessageWithDemoDisabled": "Loading query notebook screen...", "newQueryNamePrefix": "Query", "newScanNamePrefix": "Scan", + "newGremlinNamePrefix": "Gremlin", "clearQueryResultConfirmationMessage": "Are you sure you want to clear query result?", "queryTooltip": { "clusterIsInactive": "Cluster is inactive", @@ -274,6 +275,7 @@ "saveNotebookNameButtonTooltip": "Save notebook name", "addQueryButtonLabel": "Add query", "addScanButtonLabel": "Add scan", + "addGremlinButtonLabel": "Add Gremlin", "breadcrumbs": { "parent": "Notebooks", "here": "Notebook '{{name}}'" diff --git a/web-console/frontend/i18n/messages.zh-CN.json b/web-console/frontend/i18n/messages.zh-CN.json index 167964c535aa2..36574931ed0a1 100644 --- a/web-console/frontend/i18n/messages.zh-CN.json +++ b/web-console/frontend/i18n/messages.zh-CN.json @@ -106,6 +106,7 @@ "loadingMessageWithDemoDisabled": "Loading query notebook screen...", "newQueryNamePrefix": "Query", "newScanNamePrefix": "Scan", + "newGremlinNamePrefix": "Gremlin", "clearQueryResultConfirmationMessage": "Are you sure you want to clear query result?", "queryTooltip": { "clusterIsInactive": "Cluster is inactive", @@ -274,6 +275,7 @@ "saveNotebookNameButtonTooltip": "Save notebook name", "addQueryButtonLabel": "Add query", "addScanButtonLabel": "Add scan", + "addGremlinButtonLabel": "Add Gremlin", "breadcrumbs": { "parent": "Notebooks", "here": "Notebook '{{name}}'" diff --git a/web-console/pom.xml b/web-console/pom.xml index d5dd0fd271ce7..802f675396435 100644 --- a/web-console/pom.xml +++ b/web-console/pom.xml @@ -45,6 +45,7 @@ 1.20 2.4.7 4.4.5 + 3.7.0 diff --git a/web-console/web-agent/pom.xml b/web-console/web-agent/pom.xml index 285ee6594ae0a..579030bed7202 100644 --- a/web-console/web-agent/pom.xml +++ b/web-console/web-agent/pom.xml @@ -59,6 +59,18 @@ ${web-console.ignite.version} + + org.apache.tinkerpop + gremlin-groovy + ${tinkerpop.version} + + + + org.apache.tinkerpop + gremlin-driver + ${tinkerpop.version} + + org.apache.ignite ignite-control-utility @@ -157,13 +169,14 @@ mysql mysql-connector-java 5.1.44 - + - - com.zaxxer - HikariCP - ${java7.hikaricp.version} - + + + org.apache.commons + commons-dbcp2 + 2.8.0 + diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java index ff75a158fdf98..3e28adc15a2cc 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java @@ -161,8 +161,13 @@ public class AgentConfiguration implements PluginConfiguration { @Parameter(names = {"-pksp", "--passwords-key-store-password"}, description = "Password for passwords key store") private String passwordsStorePass; - + /** */ + @Parameter(names = {"-gmsp", "--gremlin-server-port"}, + description = "Port for Gremlin server, default 8182") + private int gremlinPort = 8182; + + /** */ @Parameter(names = {"-h", "--help"}, help = true, description = "Print this help message") private boolean help; @@ -521,6 +526,14 @@ public AgentConfiguration cipherSuites(List cipherSuites) { return this; } + public int gremlinPort() { + return gremlinPort; + } + + public void gremlinPort(int gremlinPort) { + this.gremlinPort = gremlinPort; + } + /** * @return {@code true} If agent options usage should be printed. */ @@ -572,6 +585,16 @@ public void load(URL cfgUrl) throws IOException { if (val != null) driversFolder(val); + + val = props.getProperty("disable-demo"); + + if (val != null) + disableDemo(val.equals("true")); + + val = props.getProperty("disable-vertx"); + + if (val != null) + disableVertx(val.equals("true")); val = props.getProperty("node-key-store"); @@ -622,6 +645,11 @@ public void load(URL cfgUrl) throws IOException { if (val != null) cipherSuites(Arrays.asList(val.split(","))); + + val = props.getProperty("gremlin-server-port"); + + if (val != null) + gremlinPort(Integer.parseInt(val)); } /** diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/IgniteClusterLauncher.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/IgniteClusterLauncher.java index f461bbad03c52..81218bdb49403 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/IgniteClusterLauncher.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/IgniteClusterLauncher.java @@ -46,6 +46,7 @@ import org.apache.ignite.binary.BinaryTypeConfiguration; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterStartNodeResult; +import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.ConnectorConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -204,7 +205,7 @@ public static void deployServices(IgniteServices services) { } /** */ - public static String registerNodeUrl(Ignite ignite) { + public static String registerNodeUrl(Ignite ignite,String clusterId) { ClusterNode node = ignite.cluster().localNode(); Collection jettyAddrs = node.attribute(ATTR_REST_JETTY_ADDRS); @@ -229,7 +230,7 @@ public static String registerNodeUrl(Ignite ignite) { String nodeUrl = String.format("http://%s:%d/%s", jettyHost, jettyPort, ignite.configuration().getIgniteInstanceName()); - RestClusterHandler.registerNodeUrl(ignite.cluster().localNode().id().toString(),nodeUrl,ignite.name()); + RestClusterHandler.registerNodeUrl(clusterId,nodeUrl,ignite.name()); return nodeUrl; } @@ -243,8 +244,7 @@ public static void stopIgnite(String clusterName,String nodeId) { String gridName = ignite.configuration().getIgniteInstanceName(); Ignition.stop(gridName,true); clusterName = null; - RestClusterHandler.clusterUrlMap.remove(nodeId); - RestClusterHandler.clusterNameMap.remove(nodeId); + } catch(IgniteIllegalStateException | IllegalArgumentException e) { //-log.error("Failed to stop cluster node: "+nodeId,e); @@ -260,27 +260,29 @@ public static void stopIgnite(String clusterName,String nodeId) { * Start ignite node with cacheEmployee and populate it with data. * @throws IgniteCheckedException */ - public static Ignite trySingleStart(String clusterId,String clusterName,String cfgFile) throws IgniteCheckedException { + public static Ignite trySingleStart(String clusterId,String clusterName,int nodeIndex,boolean isLastNode,String cfgFile) throws IgniteCheckedException { - return trySingleStart(clusterId,clusterName,cfgFile,null); + return trySingleStart(clusterId,clusterName,nodeIndex,isLastNode,cfgFile,null); } /** * Start ignite node with cacheEmployee and populate it with data. * @throws IgniteCheckedException */ - public static Ignite trySingleStart(String clusterId,String clusterName,String cfgFile,String preCfgFile) throws IgniteCheckedException { + public static Ignite trySingleStart(String clusterId,String clusterName,int nodeIndex,boolean isLastNode,String cfgFile,String preCfgFile) throws IgniteCheckedException { Ignite ignite = null; - // 单个节点： clusterID和nodeID相同 + // 最后一个节点： clusterID和nodeID相同 UUID nodeID = null; - try { - nodeID = UUID.fromString(clusterId); - ignite = Ignition.ignite(UUID.fromString(clusterId)); - return ignite; - } - catch(IgniteIllegalStateException | IllegalArgumentException e) { - + if(isLastNode) { + try { + nodeID = UUID.fromString(clusterId); + ignite = Ignition.ignite(UUID.fromString(clusterId)); + return ignite; + } + catch(IgniteIllegalStateException | IllegalArgumentException e) { + + } } // 基于Instance Name 查找ignite @@ -316,14 +318,23 @@ public static Ignite trySingleStart(String clusterId,String clusterName,String c cfgWorkMap = IgnitionEx.loadConfigurations(springCfgUrl); //only on node per jvm. IgniteConfiguration cfg = cfgWorkMap.get1().iterator().next(); - - cfg.setNodeId(nodeID); - if(cfg.getConsistentId()==null) - cfg.setConsistentId(clusterId); + + // 最后一个节点： clusterID和nodeID相同 + if(isLastNode) { + cfg.setNodeId(nodeID); + if(cfg.getConsistentId()==null) + cfg.setConsistentId(clusterId); + } + else { + cfg.setClusterStateOnStart(ClusterState.INACTIVE); + if(cfg.getConsistentId()==null) + cfg.setConsistentId(clusterId+"_"+nodeIndex); + } + if(cfg.getIgniteInstanceName()==null) cfg.setIgniteInstanceName(clusterName); - IgniteClusterLauncher.singleIgniteConfiguration(cfg,preCfg); + singleIgniteConfiguration(cfg,preCfg); ignite = IgnitionEx.start(cfg,cfgWorkMap.get2()); @@ -348,6 +359,15 @@ public static Ignite trySingleStart(String clusterId,String clusterName,String c } } } + + if(isLastNode && ignite!=null) { + try { + Thread.sleep(1000*nodeIndex); + } catch (InterruptedException e) { + e.printStackTrace(); + } + ignite.cluster().state(ClusterState.ACTIVE); + } return ignite; } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/commandline/CommandsProviderExtImpl.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/commandline/CommandsProviderExtImpl.java index db9f55e7238c2..a8993f1da9387 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/commandline/CommandsProviderExtImpl.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/commandline/CommandsProviderExtImpl.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.management.api.Argument; import org.apache.ignite.internal.management.api.Command; import org.apache.ignite.internal.management.api.LocalCommand; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; /** @@ -64,13 +65,13 @@ public static class NodeStartCommand implements LocalCommand printer) { printer.accept("Start Node: "+ arg.instanceName() + ", cfg: " + arg.cfgPath()); - + boolean isLastNode = !F.isEmpty(arg.clusterId()); // 启动一个独立的node，jvm内部的node之间相互隔离 Ignite ignite; try { - ignite = IgniteClusterLauncher.trySingleStart(arg.clusterId(), arg.instanceName(),arg.cfgPath()); - if(ignite!=null) { - IgniteClusterLauncher.registerNodeUrl(ignite); + ignite = IgniteClusterLauncher.trySingleStart(arg.clusterId(), arg.instanceName(), 0, isLastNode, arg.cfgPath()); + if(ignite!=null && isLastNode) { + IgniteClusterLauncher.registerNodeUrl(ignite,arg.clusterId()); IgniteClusterLauncher.deployServices(ignite.services(ignite.cluster().forServers())); return true; @@ -94,12 +95,13 @@ public static class NodeStartCommandArg extends IgniteDataTransferObject { @Argument private String instanceName; - @Argument + @Argument(optional=true) private String clusterId; // UUID @Argument private String cfgPath; + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, instanceName); @@ -160,7 +162,7 @@ public static class NodeStopCommand implements LocalCommand printer) { printer.accept("Stop Node: "+ arg.instanceName() + ", clusterId: " + arg.clusterId()); - + boolean isLastNode = !F.isEmpty(arg.clusterId()); IgniteClusterLauncher.stopIgnite(arg.instanceName(),arg.clusterId()); return true; @@ -176,7 +178,7 @@ public static class NodeStopCommandArg extends IgniteDataTransferObject { @Argument private String instanceName; - @Argument + @Argument(optional=true) private String clusterId; // UUID /** {@inheritDoc} */ diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DataSourceManager.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DataSourceManager.java index 46cacd6a634ba..31cd767b94c65 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DataSourceManager.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DataSourceManager.java @@ -3,6 +3,7 @@ import static org.apache.ignite.console.utils.Utils.fromJson; import java.io.IOException; +import java.sql.Driver; import java.sql.SQLException; import javax.sql.DataSource; @@ -20,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.Hashtable; import java.util.List; import java.util.Map; import java.util.Properties; @@ -34,26 +36,67 @@ import javax.naming.spi.NamingManager; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; - +import org.apache.commons.dbcp2.BasicDataSource; import io.vertx.core.json.JsonObject; public class DataSourceManager { private static Map POOL = new ConcurrentHashMap<>(); + /** */ + public static final Map drivers = new HashMap<>(); private static InitialContext initialContext = null; private static HttpClient serverClient; private static String datasourceGetUrl = ""; private static String datasourceCreateUrl = ""; private static Collection serverTokens; + static class DBinitialContext extends InitialContext { + + public DBinitialContext() throws NamingException { + super(true); + } + + public DBinitialContext(Hashtable environment) throws NamingException { + super(environment); + } + + public void bind(String key, Object value) { + POOL.put(key.toLowerCase(), value); + } + + public void rebind(String key, Object value) { + POOL.put(key.toLowerCase(), value); + } + + public void unbind(String name){ + POOL.remove(name); + } + + public Object lookup(String key) throws NamingException { + Object result = POOL.get(key.toLowerCase()); + if(result==null && key.startsWith("java:jdbc/")) { + result = getJNDIDataSource(key.substring("java:jdbc/".length())); + if(result!=null) { + POOL.put(key, result); + } + } + return result; + } + + public void close() throws NamingException { + + } + }; + public static void init(HttpClient client,String serverUri,Collection tokens) { serverClient = client; serverTokens = tokens; + + if(initialContext!=null) { + return; + } + if (serverUri.startsWith("ws:/")) { serverUri = serverUri.replaceAll("ws:/", "http:/"); } @@ -62,30 +105,12 @@ public static void init(HttpClient client,String serverUri,Collection to } datasourceGetUrl = serverUri+"/api/v1/datasource"; datasourceCreateUrl = serverUri+"/api/v1/datasource"; - if(initialContext!=null) { - return; - } + try { - initialContext = new InitialContext() { - - public void bind(String key, Object value) { - POOL.put(key.toLowerCase(), value); - } - - public Object lookup(String key) throws NamingException { - Object result = POOL.get(key.toLowerCase()); - if(result==null && key.startsWith("java:jdbc/")) { - result = getJNDIDataSource(key.substring("java:jdbc/".length())); - if(result!=null) { - POOL.put(key, result); - } - } - return result; - } - }; + initialContext = new DBinitialContext(); // Activate the initial context - NamingManager.setInitialContextFactoryBuilder(environment -> environment1 -> initialContext); + NamingManager.setInitialContextFactoryBuilder(environment -> environment1 -> new DBinitialContext()); } catch (NamingException e) { // TODO Auto-generated catch block @@ -114,32 +139,35 @@ public static DataSource getDataSource(String jndiName) throws SQLException { } - public static DataSource bindDataSource(String jndiName, DBInfo info) { - HikariConfig config = new HikariConfig(); - config.setMinimumIdle(1); - config.setMaximumPoolSize(8); - config.setJdbcUrl(info.jdbcUrl); - config.setUsername(info.getUserName()); - config.setPassword(info.getPassword()); - - if(info.getJdbcProp()!=null) { - config.setDataSourceProperties(info.getJdbcProp()); - } + public static DataSource bindDataSource(String jndiName, DBInfo info) { + return bindDataSource(jndiName,info,drivers.get(info.getDriverCls())); + } + + public static DataSource bindDataSource(String jndiName, DBInfo info, Driver driver) { - DataSource dataSource = new HikariDataSource(config); + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setDriver(driver); + dataSource.setUrl(info.jdbcUrl); + dataSource.setUsername(info.getUserName()); + dataSource.setPassword(info.getPassword()); + + dataSource.setDefaultSchema(info.getSchemaName()); + - if(info.jdbcUrl.startsWith("jdbc:h2:")) { - config.setIsolateInternalQueries(true); - config.setDriverClassName("org.h2.Driver"); - dataSource = new HikariDataSource(config); + if(info.jdbcUrl.startsWith("jdbc:h2:")) { + dataSource.setDriverClassName("org.h2.Driver"); } else { - config.setDriverClassName(info.getDriverCls()); - dataSource = new HikariDataSource(config); + dataSource.setDriverClassName(info.getDriverCls()); } try { - initialContext.bind("java:jdbc/"+jndiName, dataSource); + try { + initialContext.rebind("java:jdbc/"+jndiName, dataSource); + } catch (NamingException e) { + initialContext.bind("java:jdbc/"+jndiName, dataSource); + } + } catch (NamingException e) { e.printStackTrace(); } @@ -153,23 +181,26 @@ private static DataSource getJNDIDataSource(String jndi) { try { for(String token: serverTokens) { Request req = serverClient.newRequest(datasourceGetUrl); - req.header("TOKEN", token); + req.header("Authorization", "token "+token); req.method(HttpMethod.GET); ContentResponse response = req.send(); String body = response.getContentAsString(); - TypeReference> typeRef = new TypeReference>() {}; - List datasources = fromJson(body,typeRef); - for(DBInfo dbInfo: datasources) { - if(dbInfo.getJndiName().equalsIgnoreCase(jndi)) { - try { - dataSource = bindDataSource(jndi, dbInfo); - return dataSource; - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + if(body.startsWith("[")) { + TypeReference> typeRef = new TypeReference>() {}; + List datasources = fromJson(body,typeRef); + for(DBInfo dbInfo: datasources) { + if(dbInfo.getJndiName().equalsIgnoreCase(jndi)) { + try { + dataSource = bindDataSource(jndi, dbInfo); + return dataSource; + } catch (Exception e) { + e.printStackTrace(); + } } } - } + break; + } + } } catch (InterruptedException e1) { // TODO Auto-generated catch block diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DbMetadataReader.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DbMetadataReader.java index 27c24129cb1fb..025c94bd9c406 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DbMetadataReader.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/DbMetadataReader.java @@ -46,8 +46,6 @@ public class DbMetadataReader { /** Logger. */ private static final Logger log = LoggerFactory.getLogger(DbMetadataReader.class.getName()); - /** */ - private final Map drivers = new HashMap<>(); /** * Get specified dialect object for selected database. @@ -136,7 +134,7 @@ public Collection cachedMetadata(Connection conn, List schemas, */ public Connection connect(String jdbcDrvJarPath, DBInfo dbInfo) throws SQLException { - Driver drv = drivers.get(dbInfo.getDriverCls()); + Driver drv = DataSourceManager.drivers.get(dbInfo.getDriverCls()); if (drv == null && jdbcDrvJarPath!=null) { @@ -155,7 +153,8 @@ public Connection connect(String jdbcDrvJarPath, DBInfo dbInfo) drv = (Driver)Class.forName(dbInfo.getDriverCls(), true, ucl).newInstance(); - drivers.put(dbInfo.getDriverCls(), drv); + DataSourceManager.drivers.put(dbInfo.getDriverCls(), drv); + } catch (Exception e) { throw new IllegalStateException(e); @@ -166,7 +165,7 @@ else if (drv == null && jdbcDrvJarPath==null) { try { drv = (Driver)Class.forName(dbInfo.getDriverCls()).newInstance(); - drivers.put(dbInfo.getDriverCls(), drv); + DataSourceManager.drivers.put(dbInfo.getDriverCls(), drv); } catch (Exception e) { throw new IllegalStateException(e); @@ -176,7 +175,7 @@ else if (drv == null && jdbcDrvJarPath==null) { Connection conn = drv.connect(dbInfo.jdbcUrl, dbInfo.getJdbcProp()); if (conn == null) - throw new IllegalStateException("Connection was not established (JDBC driver returned null value)."); + throw new IllegalStateException("Connection was not established (JDBC driver returned null value)."); return conn; } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/JdbcQueryExecutor.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/JdbcQueryExecutor.java index 157cacde96d5d..8d812e2189e0b 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/JdbcQueryExecutor.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/JdbcQueryExecutor.java @@ -73,9 +73,6 @@ public JsonObject executeSqlVisor(int queryId,String nodeId) throws SQLException long end = System.currentTimeMillis(); queryResult.put("duration", end-start); - - - queryResult.put("protocolVersion", 1); return queryResult; } catch (SQLException ex) { diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractClusterHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractClusterHandler.java index 05c508a0b42cb..8cb4c5c1d2ce7 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractClusterHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractClusterHandler.java @@ -88,4 +88,8 @@ public static SslContextFactory createNodeSslFactory(AgentConfiguration cfg) { public abstract RestResult restCommand(String clusterId,JsonObject params) throws Throwable; public abstract List topologySnapshot(); + + public void close() { + + } } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java index fea012e7af36c..b3d653d89b7ec 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterHandler.java @@ -12,4 +12,6 @@ public interface ClusterHandler { public abstract RestResult restCommand(String clusterId,JsonObject params) throws Throwable; public abstract List topologySnapshot(); + + public void close(); } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClustersWatcher.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClustersWatcher.java index 045d5acc8f304..a87349aee0758 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClustersWatcher.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClustersWatcher.java @@ -123,7 +123,9 @@ void startWatchTask(Session ses) { for(ClusterHandler hnd: clusterHnds) { List ctops = hnd.topologySnapshot(); - tops.addAll(ctops); + if(ctops!=null) { + tops.addAll(ctops); + } } sendTopology(ses, tops); diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java index d58b73029a800..227769ca872a6 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseHandler.java @@ -68,6 +68,7 @@ import org.eclipse.jetty.client.HttpResponseException; import org.slf4j.LoggerFactory; +import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; import static java.nio.charset.StandardCharsets.UTF_8; @@ -358,8 +359,7 @@ public List topologySnapshot() { top.setName(info.getDb()+"-"+info.getJndiName()); top.setDemo(false); top.setActive(true); - if(schemas.isEmpty()) { - + if(schemas.isEmpty()) { top.setActive(false); databaseListener.deactivedCluster(info.getId().toString()); } @@ -368,8 +368,7 @@ public List topologySnapshot() { } else { Collection schemas = schemas(info); - if(schemas.isEmpty()) { - + if(schemas.isEmpty()) { info.top.setActive(false); databaseListener.deactivedCluster(info.getId().toString()); } @@ -385,4 +384,11 @@ public List topologySnapshot() { return tops; } + + public void close() { + for (Entry ent: databaseListener.clusters.entrySet()) { + DBInfo info = ent.getValue(); + databaseListener.deactivedCluster(info.getId().toString()); + } + } } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java index 6c3335f73dec1..99235bf663989 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java @@ -47,7 +47,7 @@ public class DatabaseListener { /** Index of alive node URI. jndiName->DBInfo*/ final public Map clusters = new ConcurrentHashMap<>(); - final public Map deactivedCluster = new ConcurrentHashMap<>(); + final public Map deactivedCluster = new ConcurrentHashMap<>(); public boolean deactivedCluster(String id) { Integer count = deactivedCluster.compute(id,(k,v)->{ return v==null? 1: ++v;}); diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoClusterHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoClusterHandler.java index 0222839616780..56cb2b77bf160 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoClusterHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoClusterHandler.java @@ -46,7 +46,7 @@ */ public class DemoClusterHandler extends AbstractClusterHandler{ /** Demo cluster ID. */ - public static final String DEMO_CLUSTER_ID = UUID.randomUUID().toString(); + public static String DEMO_CLUSTER_ID; /** Demo cluster name. */ public static final String DEMO_CLUSTER_NAME = "demo-cluster"; @@ -56,6 +56,7 @@ public class DemoClusterHandler extends AbstractClusterHandler{ */ DemoClusterHandler(AgentConfiguration cfg) { super(cfg, null); + DEMO_CLUSTER_ID = F.first(cfg.tokens()); } /** {@inheritDoc} */ @@ -63,11 +64,7 @@ public class DemoClusterHandler extends AbstractClusterHandler{ if (AgentClusterDemo.getDemoUrl() == null) { if (cfg.disableDemo()) return RestResult.fail(404, "Demo mode disabled by administrator."); - - IgniteConfiguration cfg = new IgniteConfiguration(); - AgentClusterDemo.tryStart(cfg).await(); - - if (AgentClusterDemo.getDemoUrl() == null) + else return RestResult.fail(404, "Failed to send request because of embedded node for demo mode is not started yet."); } @@ -84,25 +81,32 @@ public List topologySnapshot() { TopologySnapshot top; if (AgentClusterDemo.getDemoUrl() != null) { - IgniteEx ignite = (IgniteEx)F.first(Ignition.allGrids()); - - Collection nodes = collectNodes(ignite.context()); - - top = new TopologySnapshot(nodes); - - top.setActive(ignite.cluster().active()); + try { + IgniteEx ignite = (IgniteEx)Ignition.ignite(AgentClusterDemo.SRV_NODE_NAME); + + Collection nodes = collectNodes(ignite.context()); + + top = new TopologySnapshot(nodes); + top.setClusterVersion(VER_STR); + top.setActive(ignite.cluster().active()); + top.setId(DEMO_CLUSTER_ID.toString()); + top.setName(DEMO_CLUSTER_NAME); + top.setDemo(true); + + return List.of(top); + } + catch(Exception e) { + return null; + } } else { - top = new TopologySnapshot(); - + top = new TopologySnapshot(); top.setClusterVersion(VER_STR); - } - - top.setId(DEMO_CLUSTER_ID.toString()); - top.setName(DEMO_CLUSTER_NAME); - top.setDemo(true); - - return List.of(top); + top.setId(DEMO_CLUSTER_ID.toString()); + top.setName(DEMO_CLUSTER_NAME); + top.setDemo(true); + return List.of(top); + } } /** diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestClusterHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestClusterHandler.java index f0342e3f42775..91194b6f8af66 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestClusterHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestClusterHandler.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import javax.net.ssl.SSLException; import org.apache.ignite.IgniteLogger; @@ -66,6 +67,7 @@ public class RestClusterHandler extends AbstractClusterHandler { public static final Map clusterNameMap = U.newHashMap(2); + public static final Map deactivedCluster = new ConcurrentHashMap<>(); /** */ private static final String EXPIRED_SES_ERROR_MSG = "Failed to handle request - unknown session token (maybe expired session)"; @@ -96,6 +98,15 @@ else if(!urls.contains(url)){ urls.add(url); } } + + public static boolean deactivedCluster(String id) { + Integer count = deactivedCluster.compute(id,(k,v)->{ return v==null? 1: ++v;}); + if(count>1) { + + return true; + } + return false; + } /** {@inheritDoc} */ @@ -256,7 +267,7 @@ private RestResult topology(String nid) throws Throwable { public List topologySnapshot() { List tops = new LinkedList<>(); - for(String cluster: RestClusterHandler.clusterUrlMap.keySet()) { + for(String cluster: clusterUrlMap.keySet()) { try { RestResult res = topology(cluster); @@ -271,13 +282,19 @@ public List topologySnapshot() { TopologySnapshot newTop = new TopologySnapshot(nodes); - if (!newTop.sameNodes(latestTop)) - log.info("Connection successfully established to cluster with nodes: " + nid8(newTop.nids())); - else if (!Objects.equals(latestTop.nids(), newTop.nids())) + if (!newTop.sameNodes(latestTop)) { + if(log.isDebugEnabled()) + log.info("Connection successfully established to cluster with nodes: " + nid8(newTop.nids())); + } + else if (!Objects.equals(latestTop.nids(), newTop.nids())) { log.info("Cluster topology changed, new topology: " + nid8(newTop.nids())); + } if(cluster.isEmpty()) { cluster = F.first(nodes).getConsistentId().toString(); + if(clusterNameMap.containsValue(cluster)) { + continue; + } } boolean active = active(fromString(newTop.getClusterVersion()), cluster, F.first(newTop.nids())); @@ -294,10 +311,23 @@ else if (!Objects.equals(latestTop.nids(), newTop.nids())) } catch (Throwable e) { onFailedClusterRequest(e); - + + TopologySnapshot dieTop = new TopologySnapshot(); + dieTop.setId(cluster); + dieTop.setActive(false); + dieTop.setName(RestClusterHandler.clusterNameMap.getOrDefault(cluster, cluster)); + tops.add(dieTop); + + RestClusterHandler.deactivedCluster(cluster); latestTop = null; } } + + for(String nodeId: deactivedCluster.keySet()) { + RestClusterHandler.clusterUrlMap.remove(nodeId); + RestClusterHandler.clusterNameMap.remove(nodeId); + } + deactivedCluster.clear(); return tops; } } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/VertxClusterHandler.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/VertxClusterHandler.java index 331198bcbf347..fede1ff968efb 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/VertxClusterHandler.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/VertxClusterHandler.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; +import java.util.Optional; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; @@ -30,6 +31,9 @@ import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.console.agent.AgentConfiguration; +import org.apache.ignite.console.agent.rest.GremlinExecutor; +import org.apache.ignite.console.agent.rest.GridTaskExecutor; +import org.apache.ignite.console.agent.rest.RestExecutor; import org.apache.ignite.console.agent.rest.RestResult; @@ -43,6 +47,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.logger.slf4j.Slf4jLogger; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.slf4j.LoggerFactory; import io.vertx.core.Vertx; @@ -57,55 +62,86 @@ /** * API to transfer topology from vertx cluster to Web Console. */ -public class VertxClusterHandler extends AbstractClusterHandler{ +public class VertxClusterHandler implements ClusterHandler{ /** */ private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(VertxClusterHandler.class)); - /** Vertx cluster ID. */ - public static final String VERTX_CLUSTER_ID = UUID.randomUUID().toString(); - /** Vertx cluster name. */ - public static final String VERTX_CLUSTER_NAME = "vertx-cluster"; - - public static final Map clusterVertxMap = U.newHashMap(2); + public static final String VERTX_CLUSTER_NAME_PREFIX = "Vertx-"; + /** ignite name -> vertx */ + public static final Map clusterVertxMap = U.newHashMap(4); + /** ignite name -> exception */ + public static final Map lastErrors = U.newLinkedHashMap(4); + /** ignite cluster id -> ignite name */ + public static final Map clusterNameMap = U.newHashMap(4); + private GremlinExecutor gremlinExecutor; + + private GridTaskExecutor gridTaskExecutor; + + /** Agent configuration. */ + protected final AgentConfiguration cfg; + + private String lastClusterId = ""; - public static final Map clusterNameMap = RestClusterHandler.clusterNameMap; - /** * @param cfg Config. */ - VertxClusterHandler(AgentConfiguration cfg) { - super(cfg, null); + public VertxClusterHandler(AgentConfiguration cfg) { + this.cfg = cfg; + gremlinExecutor = new GremlinExecutor(cfg.gremlinPort()); + gridTaskExecutor = new GridTaskExecutor(); } /** {@inheritDoc} */ @Override public RestResult restCommand(String clusterId,JsonObject params) throws Throwable { - if (clusterNameMap.containsKey(clusterId)) { - if (cfg.disableVertx()) - return RestResult.fail(404, "Vertx disabled by administrator."); - - String clusterName = clusterNameMap.get(clusterId); - - if (!clusterVertxMap.containsKey(clusterName)) { - startVertxCluster(clusterName,cfg); - Thread.sleep(400); - } - + String cmd = params.getString("cmd"); + String code = params.getString("qry"); + Optional clusterNameOpt = getClusterName(clusterId); + if (clusterNameOpt.isPresent()) { + String clusterName = clusterNameOpt.get(); + + if("qrygremlinexe".equals(cmd) || "qrygroovyexe".equals(cmd)) { + + if(code.indexOf("vertx.")<0) { + // Gremlin Query + Ignite ignite = Ignition.ignite(clusterName); + return gremlinExecutor.sendRequest(ignite, clusterId, params); + } + + if (cfg.disableVertx()) + return RestResult.fail(404, "Vertx disabled by administrator."); + + + if (!clusterVertxMap.containsKey(clusterName)) { + startVertxCluster(clusterName,cfg); + if(!lastErrors.containsKey(clusterName) && !clusterVertxMap.containsKey(clusterName)) { + Thread.sleep(200); + } + } - if (clusterVertxMap.containsKey(clusterName)) { - List nodeURIs = this.cfg.nodeURIs(); - for(String url: nodeURIs) { - try { - return restExecutor.sendRequest(url, params); + if (clusterVertxMap.containsKey(clusterName)) { + try { + // Gremlin Query + Ignite ignite = Ignition.ignite(clusterName); + Vertx vertx = clusterVertxMap.get(clusterName); + return gremlinExecutor.execRequest(ignite, vertx, clusterId, params); } catch(Exception e) { - log.error("Vertx cluster rest call fail!",e); + log.error("Vertx cluster rest call fail!",e); } - } - } - - } + } + + if (lastErrors.containsKey(clusterName)) { + return RestResult.fail(500, lastErrors.get(clusterName).getMessage()); + } + } + else { + IgniteEx igniteEx = (IgniteEx)Ignition.ignite(clusterName); + return gridTaskExecutor.execRequest(igniteEx,clusterId, params); + } + + } return RestResult.fail(404, "Failed to send request because of embedded node for vertx is not started yet."); } @@ -121,15 +157,37 @@ void startVertxCluster(String clusterName, AgentConfiguration cfg) { Vertx.clusteredVertx(options,res->{ if(res.succeeded()) { Vertx vertx = res.result(); - clusterVertxMap.put(clusterName, vertx); + clusterVertxMap.put(clusterName, vertx); + clusterNameMap.put(ignite.cluster().id().toString(), clusterName); }else{ //失败的时候做什么！ + lastErrors.put(clusterName, res.cause()); log.error("Failt to start Vertx cluster.",res.cause()); } }); } + + public boolean isVertxCluster(String clusterId) { + if(clusterId!=null && clusterNameMap.containsKey(clusterId)) { + String clusterName = clusterNameMap.get(clusterId); + return clusterVertxMap.containsKey(clusterName); + } + return false; + } + + public Optional getClusterName(String clusterId) { + if(clusterNameMap.containsKey(clusterId)) { + String clusterName = clusterNameMap.get(clusterId); + return Optional.of(clusterName); + } + if(RestClusterHandler.clusterNameMap.containsKey(clusterId)) { + String clusterName = RestClusterHandler.clusterNameMap.get(clusterId); + return Optional.of(clusterName); + } + return Optional.empty(); + } /** * @return Topology snapshot for demo cluster. @@ -138,20 +196,32 @@ public List topologySnapshot() { if (cfg.disableVertx()) return null; - List tops = new LinkedList<>(); + List tops = new LinkedList<>(); for (Entry ent: clusterVertxMap.entrySet()) { TopologySnapshot top; String clusterName = ent.getKey(); - IgniteEx ignite = (IgniteEx)Ignition.ignite(clusterName); - - Collection nodes = collectNodes(ignite.context()); - top = new TopologySnapshot(nodes); - - top.setId(VERTX_CLUSTER_ID.toString()); - top.setName("Vertx-"+VERTX_CLUSTER_NAME); - top.setDemo(false); - top.setClusterVersion(VER_STR); + try { + IgniteEx ignite = (IgniteEx)Ignition.ignite(clusterName); + + Collection nodes = collectNodes(ignite.context()); + top = new TopologySnapshot(nodes); + lastClusterId = ignite.cluster().id().toString(); + top.setId(lastClusterId); + top.setName(VERTX_CLUSTER_NAME_PREFIX+clusterName); + top.setDemo(false); + top.setClusterVersion(VER_STR); + top.setActive(ignite.active()); + } + catch(Exception e) { + top = new TopologySnapshot(); + + top.setId(lastClusterId); + top.setName(VERTX_CLUSTER_NAME_PREFIX+clusterName); + top.setDemo(false); + top.setClusterVersion(VER_STR); + top.setActive(false); + } tops.add(top); } return tops; @@ -177,4 +247,10 @@ private Collection collectNodes(GridKernalContext ctx) { return Collections.emptyList(); } } + + public void close() { + for (Entry ent: clusterVertxMap.entrySet()) { + ent.getValue().close(); + } + } } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java index a813da58953ff..2b36bd8aec7d1 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/WebSocketRouter.java @@ -42,6 +42,7 @@ import java.io.File; import java.net.URI; +import java.net.URL; import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.SQLException; @@ -52,6 +53,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -79,10 +81,13 @@ import org.apache.ignite.console.websocket.AgentHandshakeResponse; import org.apache.ignite.console.websocket.WebSocketRequest; import org.apache.ignite.console.websocket.WebSocketResponse; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.logger.slf4j.Slf4jLogger; import org.apache.ignite.services.Service; import org.eclipse.jetty.client.HttpClient; @@ -98,6 +103,7 @@ import org.eclipse.jetty.websocket.client.WebSocketClient; import org.slf4j.LoggerFactory; +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; /** @@ -170,9 +176,9 @@ public WebSocketRouter(AgentConfiguration cfg) { httpClient = new HttpClient(createServerSslFactory(cfg)); httpClient.setMaxConnectionsPerDestination(2); httpClient.setConnectBlocking(false); - httpClient.setConnectTimeout(500); + httpClient.setConnectTimeout(1000); httpClient.setFollowRedirects(false); - httpClient.setExecutor(RestExecutor.executor); + httpClient.setExecutor(RestExecutor.executor); // TODO GG-18379 Investigate how to establish native websocket connection with proxy. @@ -242,6 +248,11 @@ private void stopClient() { /** {@inheritDoc} */ @Override public void close() { log.info("Stopping Web Console Agent..."); + + vertxClusterHnd.close(); + dbHnd.close(); + demoClusterHnd.close(); + clusterHnd.close(); stopClient(); @@ -270,10 +281,15 @@ private void connect0() { client = new WebSocketClient(httpClient); - client.start(); - client.connect(this, URI.create(cfg.serverUri()).resolve(AGENTS_PATH)).get(5L, TimeUnit.SECONDS); + Session session = client.connect(this, URI.create(cfg.serverUri()).resolve(AGENTS_PATH)).get(5L, TimeUnit.SECONDS); + session.getPolicy().setMaxTextMessageSize(1024*1024); + session.getPolicy().setMaxBinaryMessageSize(1024*1024); + session.getPolicy().setMaxTextMessageBufferSize(1024*1024); + session.getPolicy().setMaxBinaryMessageBufferSize(1024*1024); + session.setIdleTimeout(60*60000); + reconnectCnt.set(0); } @@ -385,13 +401,14 @@ private void processRevokeToken(String tok) { /** * @param tok Token to revoke. */ - private JsonObject processClusterStart(String msg) { - log.info("Cluster start msg has been revoked: " + msg); + private JsonObject processClusterStart(WebSocketRequest evt) { + log.info("Cluster start msg has been revoked: " + evt.getNodeSeq()); JsonObject stat = new JsonObject(); - JsonObject json = fromJson(msg); - + JsonObject json = fromJson(evt.getPayload()); + + boolean isLastNode = evt.isLastNode(); String clusterId = json.getString("id"); - if(DEMO_CLUSTER_ID.equals(clusterId)) { + if(DEMO_CLUSTER_ID.equals(clusterId) || AgentClusterDemo.SRV_NODE_NAME.equals(clusterId)) { json.put("demo", true); } String clusterName = Utils.escapeFileName(json.getString("name")); @@ -399,8 +416,8 @@ private JsonObject processClusterStart(String msg) { String unzipDest = IgniteClusterLauncher.saveBlobToFile(json); - Boolean restart = json.getBoolean("restart"); - if(restart!=null && restart) { + Boolean restart = json.getBoolean("restart",false); + if(restart) { IgniteClusterLauncher.stopIgnite(clusterName,clusterId); } @@ -414,16 +431,18 @@ private JsonObject processClusterStart(String msg) { File configFile = new File(U.getIgniteHome()+ "/config/clusters/"+clusterName+"-config.xml"); if(startIniFile.exists()) { - try { - ignite = Ignition.allGrids().get(0); - ignite.cluster().startNodes(startIniFile, restart, 60*1000, 1); - stat.put("status", "started"); - } - catch(IgniteIllegalStateException e) { - stat.put("message", e.getMessage()); - stat.put("status", "stoped"); - return stat; - } + if(isLastNode) { + try { + ignite = Ignition.allGrids().get(0); + ignite.cluster().startNodes(startIniFile, restart, 60*1000, 1); + stat.put("status", "started"); + } + catch(IgniteIllegalStateException e) { + stat.put("message", e.getMessage()); + stat.put("status", "stoped"); + return stat; + } + } } else { @@ -433,46 +452,70 @@ private JsonObject processClusterStart(String msg) { if(configWorkFile.exists() && configFile.exists()) { // 合并并启动一个服务端已经配置好的node - ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,cfgFile,configFile.toString()); + ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,evt.getNodeSeq(),isLastNode,cfgFile,configFile.toString()); } else if(configWorkFile.exists()) { // 启动一个独立的node，jvm内部的node之间相互隔离 - ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,cfgFile); + ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,evt.getNodeSeq(),isLastNode,cfgFile); } else if(configFile.exists()) { // 启动一个服务端已经配置好的node - ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,null,configFile.toString()); + ignite = IgniteClusterLauncher.trySingleStart(clusterId,clusterName,evt.getNodeSeq(),isLastNode,null,configFile.toString()); + } + else { + stat.put("message", "not found cfg file, please deploy cfg first!"); } if(ignite!=null) { - IgniteClusterLauncher.registerNodeUrl(ignite); - - IgniteClusterLauncher.deployServices(ignite.services(ignite.cluster().forServers())); + IgniteClusterLauncher.registerNodeUrl(ignite,clusterId); + if(isLastNode) { + IgniteClusterLauncher.deployServices(ignite.services(ignite.cluster().forServers())); + } stat.put("status", "started"); + stat.put("message","Ignite started successfully."); } else { stat.put("status", "stoped"); + return stat; } } } else { - // 启动一个共享型的node，jvm内部的grid node会组成集群 + // 启动一个内存型的node，有多少个Agent就有多少个Demo Node + String work = U.workDirectory(null, null); + String cfgFile = String.format("%s/config/%s/src/main/resources/META-INF/%s-server.xml", work, clusterName,clusterName); + File configWorkFile = new File(cfgFile); + IgniteConfiguration cfg = new IgniteConfiguration(); - cfg.setIgniteInstanceName(clusterName); - cfg.setConsistentId(clusterId); - cfg.setNodeId(UUID.fromString(clusterId)); - // 启动多个节点，并且在最后一个节点部署服务 - boolean rv = AgentClusterDemo.tryStart(cfg).await(60,TimeUnit.SECONDS); - if(rv) { + if(configWorkFile.exists()) { + IgniteBiTuple, ? extends GridSpringResourceContext> cfgMap=null; + if(ignite==null) { + URL springPreCfgUrl = U.resolveSpringUrl(configWorkFile.toString()); + cfgMap = IgnitionEx.loadConfigurations(springPreCfgUrl); + + Collection cfgList = cfgMap.get1(); + for(IgniteConfiguration cfg0: cfgList) { + if(clusterName.equals(cfg0.getIgniteInstanceName())){ + cfg = cfg0; + } + } + } + } + + // 启动Demo节点，并且在最后一个节点部署服务 + ignite = AgentClusterDemo.tryStart(cfg,evt.getNodeSeq(),isLastNode); + if(ignite!=null) { stat.put("status", "started"); } - } + else { + stat.put("message","Demo Ignite already started."); + } + return stat; + } - stat.put("message","Ignite started successfully."); - } catch (Exception e) { - // TODO Auto-generated catch block + } catch (Exception e) { e.printStackTrace(); stat.put("message", e.getMessage()); stat.put("status", "stoped"); @@ -484,10 +527,11 @@ else if(configFile.exists()) { /** * @param tok Token to revoke. */ - private JsonObject processClusterStop(String msg) { - log.info("Cluster stop msg has been revoked: " + msg); + private JsonObject processClusterStop(WebSocketRequest evt) { + log.info("Cluster stop msg has been revoked: " + evt.getPayload()); JsonObject stat = new JsonObject(); - JsonObject json = fromJson(msg); + JsonObject json = fromJson(evt.getPayload()); + boolean isLastNode = evt.isLastNode(); String clusterName = Utils.escapeFileName(json.getString("name")); if(json.getBoolean("demo",false)) { AgentClusterDemo.stop(); @@ -497,7 +541,7 @@ private JsonObject processClusterStop(String msg) { File startIniFile = new File(U.getIgniteHome()+ "/config/clusters/"+clusterName+"-start-nodes.ini"); - if(startIniFile.exists()) { + if(isLastNode && startIniFile.exists()) { try { Ignite ignite = Ignition.ignite(clusterName); ignite.cluster().stopNodes(); @@ -651,12 +695,12 @@ public void onMessage(Session ses, String msg) { return; case AGENT_START_CLUSTER: - msgRet = processClusterStart(evt.getPayload()); + msgRet = processClusterStart(evt); send(ses, evt.response(msgRet)); break; case AGENT_STOP_CLUSTER: - msgRet = processClusterStop(evt.getPayload()); + msgRet = processClusterStop(evt); send(ses, evt.response(msgRet)); break; @@ -693,19 +737,40 @@ public void onMessage(Session ses, String msg) { RestRequest reqRest = fromJson(evt.getPayload(), RestRequest.class); JsonObject params = new JsonObject(reqRest.getParams()); - + String cmd = params.getString("cmd"); RestResult res; try { - if(DEMO_CLUSTER_ID.equals(reqRest.getClusterId())) { + if("qrygremlinexe".equals(cmd) || "qrygroovyexe".equals(cmd)) { + // Gremlin Query or Execute groovy code + res = vertxClusterHnd.restCommand(reqRest.getClusterId(),params); + } + else if(DEMO_CLUSTER_ID.equals(reqRest.getClusterId())) { res = demoClusterHnd.restCommand(reqRest.getClusterId(),params); } else if(dbHnd.isDBCluster(reqRest.getClusterId())) { res = dbHnd.restCommand(reqRest.getClusterId(), params); } - else { + else if(vertxClusterHnd.isVertxCluster(reqRest.getClusterId())) { + // inner task call + res = vertxClusterHnd.restCommand(reqRest.getClusterId(),params); + } + else { res = clusterHnd.restCommand(reqRest.getClusterId(),params); } + + if(cmd.equals("top") && params.getBoolean("attr",false)) { + JsonArray nodes = new JsonArray(res.getData()); + for(int i=0;i keys) { + for(String key: keys) { + addFieldMetadata(metaDataArray, key, "String"); + } + } + + private void addFieldMetadata(JsonArray metaDataArray, Map dict) { + for(Map.Entry ent: dict.entrySet()) { + Class vCls = ent.getValue().getClass(); + if(vCls.isArray()) { + vCls = vCls.getComponentType(); + } + addFieldMetadata(metaDataArray, ent.getKey(), vCls.getSimpleName()); + } + } + + private JsonObject parseResponse(Object object, String nodeId, JsonObject params) throws IOException { + JsonObject queryResult = new JsonObject(); + String err = null; + try { + + int rowCount = 0; //To count the number of rows + + queryResult.put("hasMore", false); + queryResult.put("queryId", 0); + queryResult.put("responseNodeId", nodeId); + + JsonArray metaDataArray = new JsonArray(); + + JsonArray dataArray = new JsonArray(); + + if (object instanceof Collection) { + Collection list = (Collection) object; + for(Object item: list) { + JsonArray row = parseObject(item,metaDataArray,rowCount); + dataArray.add(row); + + ++rowCount; + } + } + else if (object instanceof Object[]) { + Object[] list = (Object[]) object; + for(Object item: list) { + JsonArray row = parseObject(item,metaDataArray,rowCount); + dataArray.add(row); + + ++rowCount; + } + } + else if (object!=null) { + JsonArray row = parseObject(object,metaDataArray,rowCount); + dataArray.add(row); + ++rowCount; + } + queryResult.put("rows", dataArray); + queryResult.put("columns", metaDataArray); + queryResult.put("protocolVersion", 1); + return queryResult; + + } catch (Exception ex) { + err = ex.getMessage(); + queryResult.put("error",err); + } + + return queryResult; + } + + + + private JsonObject parseResponse(ResultSet resultSet,String nodeId,JsonObject params) throws IOException { + + JsonObject queryResult = new JsonObject(); + String err = null; + try { + + int rowCount = 0; //To count the number of rows + + queryResult.put("hasMore", false); + queryResult.put("queryId", 0); + queryResult.put("responseNodeId", nodeId); + + + JsonArray metaDataArray = new JsonArray(); + + JsonArray dataArray = new JsonArray(); + + for (Result res: resultSet) { + Object object = res.getObject(); + JsonArray row = parseObject(object,metaDataArray,rowCount); + dataArray.add(row); + ++rowCount; + } + + queryResult.put("rows", dataArray); + queryResult.put("columns", metaDataArray); + queryResult.put("protocolVersion", 1); + return queryResult; + + } catch (Exception ex) { + err = ex.getMessage(); + queryResult.put("error",err); + } + + return queryResult; + } + + private JsonArray parseObject(Object object, JsonArray metaDataArray,int rowCount) { + JsonArray row = new JsonArray(); + if(object instanceof Vertex) { + Vertex v = (Vertex) object; + if(rowCount==0) { + addFieldMetadata(metaDataArray,"id","String"); + addFieldMetadata(metaDataArray,"label","String"); + addFieldMetadata(metaDataArray,v.keys()); + } + row.add(v.id()); + row.add(v.label()); + for(String key: v.keys()) { + row.add(v.value(key)); + } + } + else if(object instanceof Edge) { + Edge e = (Edge)object; + if(rowCount==0) { + addFieldMetadata(metaDataArray,"id","String"); + addFieldMetadata(metaDataArray,"label","String"); + addFieldMetadata(metaDataArray,"in","String"); + addFieldMetadata(metaDataArray,"out","String"); + addFieldMetadata(metaDataArray,e.keys()); + } + row.add(e.id()); + row.add(e.label()); + row.add(e.inVertex().id()); + row.add(e.outVertex().id()); + for(String key: e.keys()) { + row.add(e.value(key)); + } + } + else if(object instanceof VertexProperty) { + VertexProperty vp = (VertexProperty)object; + if(rowCount==0) { + addFieldMetadata(metaDataArray,"id","String"); + addFieldMetadata(metaDataArray,"label","String"); + addFieldMetadata(metaDataArray,"key","String"); + addFieldMetadata(metaDataArray,"value","String"); + } + row.add(vp.id()); + row.add(vp.label()); + row.add(vp.key()); + row.add(vp.value()); + } + else if(object instanceof Property) { + Property vp = (Property)object; + if(rowCount==0) { + addFieldMetadata(metaDataArray,"key","String"); + addFieldMetadata(metaDataArray,"value","String"); + } + row.add(vp.key()); + row.add(vp.value()); + } + else if(object instanceof Map) { + Map dict = (Map)object; + if(rowCount==0) { + addFieldMetadata(metaDataArray,dict); + } + for(Map.Entry ent: dict.entrySet()) { + if(ent.getValue() instanceof Object[]) { + Object[] list = (Object[])ent.getValue(); + if(list.length==1) { + row.add(list[0]); + } + else { + row.add(ent.getValue()); + } + } + else { + row.add(ent.getValue()); + } + } + } + else { + if(rowCount==0) { + addFieldMetadata(metaDataArray,"value","Object"); + } + row.add(object); + } + return row; + } + + /** + * @param script Groovy script. + * @param params Request parameters. + * @return Request result. + * @throws IOException If failed to parse REST result. + * @throws Throwable If failed to send request. + */ + public RestResult sendRequest(Ignite ignite, String clusterId,JsonObject params) throws Throwable { + String code = params.getString("qry"); + try { + long start = System.currentTimeMillis(); + String addr = "localhost"; + + JsonObject res = new JsonObject(); + res.put("error",(String)null); + + Cluster cluster = Cluster.build(addr).port(gremlinPort).create(); + Client client = cluster.connect(); + Map context = new HashMap<>(); + context.put("ignite_name", ignite.name()); + ResultSet result = client.submit(code,context); + + String nodeId = ignite.cluster().localNode().id().toString(); + JsonObject data = parseResponse(result, nodeId, params); + long end = System.currentTimeMillis(); + data.put("duration", end-start); + res.put("result", data); + res.put("id", "~"+clusterId); + res.put("finished",true); + return RestResult.success(res.toString(), params.getString("sessionToken")); + } + catch (Exception e) { + return RestResult.fail(STATUS_FAILED, "Failed to execute Groovy command [code=" + + code + ", msg=" + e.getCause().getMessage() + "]"); + } + } + + /** + * @param script Groovy script. + * @param params Request parameters. + * @return Request result. + * @throws IOException If failed to parse REST result. + * @throws Throwable If failed to send request. + */ + public RestResult execRequest(Ignite ignite, Vertx vertx, String clusterId, JsonObject params) throws Throwable { + String code = params.getString("qry"); + try { + JsonObject res = new JsonObject(); + res.put("error",(String)null); + long start = System.currentTimeMillis(); + Bindings bindings = engine.createBindings(); + bindings.put("vertx",vertx); + bindings.put("ignite",ignite); + + Object result = engine.eval(code, bindings); + String nodeId = ignite.cluster().localNode().id().toString(); + JsonObject data = parseResponse(result, nodeId, params); + long end = System.currentTimeMillis(); + data.put("duration", end-start); + res.put("result", data); + res.put("id", "~"+clusterId); + res.put("finished",true); + return RestResult.success(res.toString(), params.getString("sessionToken")); + } + catch (ScriptException e) { + return RestResult.fail(STATUS_FAILED, "Failed to execute Groovy command [code=" + + code + ", msg=" + e.getCause().getMessage() + "]"); + } + } +} diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/GridTaskExecutor.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/GridTaskExecutor.java new file mode 100644 index 0000000000000..ad40acf27ca18 --- /dev/null +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/GridTaskExecutor.java @@ -0,0 +1,976 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.console.agent.rest; + +import static java.lang.String.format; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_GETALL_AS_ARRAY; +import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES_MASK; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_CONTAINS_KEYS; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_GET_ALL; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_PUT_ALL; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_REMOVE_ALL; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVATE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_CURRENT_STATE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_STATE; +import static org.apache.ignite.internal.processors.rest.GridRestCommand.EXECUTE_SQL_QUERY; +import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED; +import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SUCCESS; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.CacheConfigurationOverride; +import org.apache.ignite.internal.processors.rest.GridRestCommand; +import org.apache.ignite.internal.processors.rest.GridRestProcessor; +import org.apache.ignite.internal.processors.rest.GridRestProtocolHandler; +import org.apache.ignite.internal.processors.rest.GridRestResponse; +import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; +import org.apache.ignite.internal.processors.rest.protocols.http.jetty.IgniteBinaryObjectJsonDeserializer; +import org.apache.ignite.internal.processors.rest.request.DataStructuresRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestBaselineRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestCacheRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestChangeStateRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestClusterNameRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestClusterStateRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestLogRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestNodeStateBeforeStartRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestTaskRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestTopologyRequest; +import org.apache.ignite.internal.processors.rest.request.GridRestWarmUpRequest; +import org.apache.ignite.internal.processors.rest.request.RestQueryRequest; +import org.apache.ignite.internal.processors.rest.request.RestUserActionRequest; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.logger.slf4j.Slf4jLogger; +import org.apache.ignite.plugin.security.SecurityCredentials; +import org.jetbrains.annotations.Nullable; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.vertx.core.json.JsonObject; + +/** + * Jetty REST handler. The following URL format is supported: {@code /ignite?cmd=cmdName¶m1=abc¶m2=123} + */ +public class GridTaskExecutor { + /** Used to sent request charset. */ + private static final String CHARSET = StandardCharsets.UTF_8.name(); + + /** */ + private static final String FAILED_TO_PARSE_FORMAT = "Failed to parse parameter of %s type [%s=%s]"; + + /** */ + private static final String USER_PARAM = "user"; + + /** */ + private static final String PWD_PARAM = "password"; + + /** */ + private static final String CACHE_NAME_PARAM = "cacheName"; + + /** */ + private static final String BACKUPS_PARAM = "backups"; + + /** */ + private static final String CACHE_GROUP_PARAM = "cacheGroup"; + + /** */ + private static final String DATA_REGION_PARAM = "dataRegion"; + + /** */ + private static final String WRITE_SYNCHRONIZATION_MODE_PARAM = "writeSynchronizationMode"; + + /** @deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */ + private static final String IGNITE_LOGIN = "ignite.login"; + + /** @deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */ + private static final String IGNITE_PASSWORD = "ignite.password"; + + /** */ + private static final String TEMPLATE_NAME_PARAM = "templateName"; + + + private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(GridTaskExecutor.class)); + + /** Mapper from Java object to JSON. */ + private static ObjectMapper jsonMapper; + + + public int index = 0; + + /** */ + private final boolean getAllAsArray = IgniteSystemProperties.getBoolean(IGNITE_REST_GETALL_AS_ARRAY); + + /** + * Creates new Task requests handler. + * + * @param hnd Handler. + * @param authChecker Authentication checking closure. + * @param ctx Kernal context. + */ + public GridTaskExecutor() { + + } + + /** + * Retrieves long value from parameters map. + * + * @param key Key. + * @param params Parameters map. + * @param dfltVal Default value. + * @return Long value from parameters map or {@code dfltVal} if null or not exists. + * @throws IgniteCheckedException If parsing failed. + */ + @Nullable private static Long longValue(String key, Map params, + Long dfltVal) throws IgniteCheckedException { + assert key != null; + + String val = params.get(key); + + try { + return val == null ? dfltVal : Long.valueOf(val); + } + catch (NumberFormatException ignore) { + throw new IgniteCheckedException(format(FAILED_TO_PARSE_FORMAT, "Long", key, val)); + } + } + + /** + * Retrieves boolean value from parameters map. + * + * @param key Key. + * @param params Parameters map. + * @param dfltVal Default value. + * @return Boolean value from parameters map or {@code dfltVal} if null or not exists. + */ + private static boolean booleanValue(String key, Map params, boolean dfltVal) { + assert key != null; + + String val = params.get(key); + + return val == null ? dfltVal : Boolean.parseBoolean(val); + } + + /** + * Retrieves int value from parameters map. + * + * @param key Key. + * @param params Parameters map. + * @param dfltVal Default value. + * @return Integer value from parameters map or {@code dfltVal} if null or not exists. + * @throws IgniteCheckedException If parsing failed. + */ + private static int intValue(String key, Map params, int dfltVal) throws IgniteCheckedException { + assert key != null; + + String val = params.get(key); + + try { + return val == null ? dfltVal : Integer.parseInt(val); + } + catch (NumberFormatException ignore) { + throw new IgniteCheckedException(format(FAILED_TO_PARSE_FORMAT, "Integer", key, val)); + } + } + + private static > @Nullable T enumValue( + String key, + Map params, + Class enumClass + ) throws IgniteCheckedException { + assert key != null; + assert enumClass != null; + + String val = params.get(key); + + if (val == null) + return null; + + try { + return Enum.valueOf(enumClass, val); + } + catch (IllegalArgumentException e) { + throw new IgniteCheckedException(format(FAILED_TO_PARSE_FORMAT, enumClass.getSimpleName(), key, val), e); + } + } + + /** + * Retrieves UUID value from parameters map. + * + * @param key Key. + * @param params Parameters map. + * @return UUID value from parameters map or {@code null} if null or not exists. + * @throws IgniteCheckedException If parsing failed. + */ + @Nullable private static UUID uuidValue(String key, Map params) throws IgniteCheckedException { + assert key != null; + + String val = params.get(key); + + try { + return val == null ? null : UUID.fromString(val); + } + catch (NumberFormatException ignore) { + throw new IgniteCheckedException(format(FAILED_TO_PARSE_FORMAT, "UUID", key, val)); + } + } + + + /** {@inheritDoc} */ + public RestResult execRequest(Ignite ignite,String clusterId, JsonObject params) + throws IOException { + if (log.isDebugEnabled()) + log.debug("Handling request [target=" + clusterId + ", params=" + params + ']'); + + IgniteEx igniteEx = (IgniteEx)ignite; + GridRestProcessor restProcessor = (GridRestProcessor)igniteEx.context().rest(); + if(jsonMapper==null) { + jsonMapper = new GridJettyObjectMapper(igniteEx.context()); + } + + /** Request handlers. */ + GridRestProtocolHandler hnd = U.field(restProcessor, "protoHnd"); + GridRestResponse cmdRes = processRequest(hnd,clusterId, params); + + if(cmdRes.getSuccessStatus()==STATUS_SUCCESS) { + String json = jsonMapper.writer().writeValueAsString(cmdRes.getResponse()); + return RestResult.success(json, cmdRes.getSessionToken()); + } + else { + return RestResult.fail(cmdRes.getSuccessStatus(),cmdRes.getError()); + } + } + + /** + * Process HTTP request. + * + * @param act Action. + * @param req Http request. + * @param res Http response. + */ + private GridRestResponse processRequest(GridRestProtocolHandler hnd, String act, JsonObject params) { + + GridRestCommand cmd = command(params); + GridRestResponse cmdRes; + if (cmd == null) { + cmdRes = new GridRestResponse(STATUS_FAILED, "BAD_REQUEST"); + return cmdRes; + } + + Map strParams = new HashMap<>(); + params.forEach((ent)->{ + if(ent.getValue()!=null) + strParams.put(ent.getKey(), ent.getValue().toString()); + }); + + try { + GridRestRequest cmdReq = createRequest(cmd, strParams); + + if (log.isDebugEnabled()) + log.debug("Initialized command request: " + cmdReq); + + cmdRes = hnd.handle(cmdReq); + + if (cmdRes == null) + throw new IllegalStateException("Received null result from handler: " + hnd); + + if (getAllAsArray && cmd == GridRestCommand.CACHE_GET_ALL) { + List resKeyValue = new ArrayList<>(); + + for (Map.Entry me : ((Map)cmdRes.getResponse()).entrySet()) + resKeyValue.add(new IgniteBiTuple<>(me.getKey(), me.getValue())); + + cmdRes.setResponse(resKeyValue); + } + + byte[] sesTok = cmdRes.sessionTokenBytes(); + + if (sesTok != null) + cmdRes.setSessionToken(U.byteArray2HexString(sesTok)); + + } + catch (Throwable e) { + + U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + act + ']', e); + + if (e instanceof Error) + throw (Error)e; + + cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage()); + } + return cmdRes; + + } + + /** + * Creates REST request. + * + * @param cmd Command. + * @param params Parameters. + * @param req Servlet request. + * @return REST request. + * @throws IgniteCheckedException If creation failed. + */ + @Nullable private GridRestRequest createRequest(GridRestCommand cmd,Map params) throws IgniteCheckedException { + GridRestRequest restReq; + + switch (cmd) { + case GET_OR_CREATE_CACHE: { + GridRestCacheRequest restReq0 = new GridRestCacheRequest(); + + restReq0.cacheName(params.get(CACHE_NAME_PARAM)); + + String templateName = params.get(TEMPLATE_NAME_PARAM); + + if (!F.isEmpty(templateName)) + restReq0.templateName(templateName); + + String backups = params.get(BACKUPS_PARAM); + + CacheConfigurationOverride cfg = new CacheConfigurationOverride(); + + // Set cache backups. + if (!F.isEmpty(backups)) { + try { + cfg.backups(Integer.parseInt(backups)); + } + catch (NumberFormatException e) { + throw new IgniteCheckedException("Failed to parse number of cache backups: " + backups, e); + } + } + + // Set cache group name. + String cacheGrp = params.get(CACHE_GROUP_PARAM); + + if (!F.isEmpty(cacheGrp)) + cfg.cacheGroup(cacheGrp); + + // Set cache data region name. + String dataRegion = params.get(DATA_REGION_PARAM); + + if (!F.isEmpty(dataRegion)) + cfg.dataRegion(dataRegion); + + // Set cache write mode. + String wrtSyncMode = params.get(WRITE_SYNCHRONIZATION_MODE_PARAM); + + if (!F.isEmpty(wrtSyncMode)) { + try { + cfg.writeSynchronizationMode(CacheWriteSynchronizationMode.valueOf(wrtSyncMode)); + } + catch (IllegalArgumentException e) { + throw new IgniteCheckedException("Failed to parse cache write synchronization mode: " + wrtSyncMode, e); + } + } + + if (!cfg.isEmpty()) + restReq0.configuration(cfg); + + restReq = restReq0; + + break; + } + + case DESTROY_CACHE: { + GridRestCacheRequest restReq0 = new GridRestCacheRequest(); + + restReq0.cacheName(params.get(CACHE_NAME_PARAM)); + + restReq = restReq0; + + break; + } + + case ATOMIC_DECREMENT: + case ATOMIC_INCREMENT: { + DataStructuresRequest restReq0 = new DataStructuresRequest(); + + restReq0.key(params.get("key")); + restReq0.initial(longValue("init", params, null)); + restReq0.delta(longValue("delta", params, null)); + + restReq = restReq0; + + break; + } + + case CACHE_CONTAINS_KEY: + case CACHE_CONTAINS_KEYS: + case CACHE_GET: + case CACHE_GET_ALL: + case CACHE_GET_AND_PUT: + case CACHE_GET_AND_REPLACE: + case CACHE_PUT_IF_ABSENT: + case CACHE_GET_AND_PUT_IF_ABSENT: + case CACHE_PUT: + case CACHE_PUT_ALL: + case CACHE_REMOVE: + case CACHE_REMOVE_VALUE: + case CACHE_REPLACE_VALUE: + case CACHE_GET_AND_REMOVE: + case CACHE_REMOVE_ALL: + case CACHE_CLEAR: + case CACHE_ADD: + case CACHE_CAS: + case CACHE_METRICS: + case CACHE_SIZE: + case CACHE_UPDATE_TLL: + case CACHE_METADATA: + case CACHE_REPLACE: + case CACHE_APPEND: + case CACHE_PREPEND: { + GridRestCacheRequest restReq0 = new GridRestCacheRequest(); + + String cacheName = params.get(CACHE_NAME_PARAM); + restReq0.cacheName(F.isEmpty(cacheName) ? null : cacheName); + + String keyType = params.get("keyType"); + String valType = params.get("valueType"); + + Converter converter = new Converter(cacheName); + + restReq0.key(converter.convert(keyType, params.get("key"))); + restReq0.value(converter.convert(valType, params.get("val"))); + restReq0.value2(converter.convert(valType, params.get("val2"))); + + Object val1 = converter.convert(valType, params.get("val1")); + + if (val1 != null) + restReq0.value(val1); + + // Cache operations via REST will use binary objects. + restReq0.cacheFlags(intValue("cacheFlags", params, KEEP_BINARIES_MASK)); + restReq0.ttl(longValue("exp", params, null)); + + if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL || + cmd == CACHE_CONTAINS_KEYS) { + List keys = converter.values(keyType, "k", params); + List vals = converter.values(valType, "v", params); + + if (keys.size() < vals.size()) + throw new IgniteCheckedException("Number of keys must be greater or equals to number of values."); + + Map map = U.newHashMap(keys.size()); + + Iterator keyIt = keys.iterator(); + Iterator valIt = vals.iterator(); + + while (keyIt.hasNext()) + map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null); + + restReq0.values(map); + } + + restReq = restReq0; + + break; + } + + case TOPOLOGY: + case NODE: { + GridRestTopologyRequest restReq0 = new GridRestTopologyRequest(); + + restReq0.includeMetrics(Boolean.parseBoolean(params.get("mtr"))); + restReq0.includeAttributes(Boolean.parseBoolean(params.get("attr"))); + + String caches = params.get("caches"); + restReq0.includeCaches(caches == null || Boolean.parseBoolean(caches)); + + restReq0.nodeIp(params.get("ip")); + + restReq0.nodeId(uuidValue("id", params)); + + restReq = restReq0; + + break; + } + + case EXE: + case RESULT: + case NOOP: { + GridRestTaskRequest restReq0 = new GridRestTaskRequest(); + + restReq0.taskId(params.get("id")); + restReq0.taskName(params.get("name")); + + restReq0.params(new Converter().values(null, "p", params)); + + restReq0.async(Boolean.parseBoolean(params.get("async"))); + + restReq0.timeout(longValue("timeout", params, 0L)); + + restReq = restReq0; + + break; + } + + case LOG: { + GridRestLogRequest restReq0 = new GridRestLogRequest(); + + restReq0.path(params.get("path")); + + restReq0.from(intValue("from", params, -1)); + restReq0.to(intValue("to", params, -1)); + + restReq = restReq0; + + break; + } + + case DATA_REGION_METRICS: + case NAME: + case VERSION: + case PROBE: { + restReq = new GridRestRequest(); + + break; + } + + case CLUSTER_ACTIVE: + case CLUSTER_INACTIVE: + case CLUSTER_ACTIVATE: + case CLUSTER_DEACTIVATE: + case CLUSTER_CURRENT_STATE: { + GridRestChangeStateRequest restReq0 = new GridRestChangeStateRequest(); + + if (cmd == CLUSTER_CURRENT_STATE) + restReq0.reqCurrentState(); + else if (cmd == CLUSTER_ACTIVE || cmd == CLUSTER_ACTIVATE) + restReq0.active(true); + else + restReq0.active(false); + + restReq0.forceDeactivation(booleanValue(GridRestClusterStateRequest.ARG_FORCE, params, false)); + + restReq = restReq0; + + break; + } + + case CLUSTER_STATE: + case CLUSTER_SET_STATE: { + GridRestClusterStateRequest restReq0 = new GridRestClusterStateRequest(); + + if (cmd == CLUSTER_STATE) + restReq0.reqCurrentMode(); + else { + ClusterState newState = enumValue("state", params, ClusterState.class); + + restReq0.state(newState); + + restReq0.forceDeactivation(booleanValue(GridRestClusterStateRequest.ARG_FORCE, params, false)); + } + + restReq = restReq0; + + break; + } + + case CLUSTER_NAME: { + restReq = new GridRestClusterNameRequest(); + + break; + } + + case BASELINE_CURRENT_STATE: + case BASELINE_SET: + case BASELINE_ADD: + case BASELINE_REMOVE: { + GridRestBaselineRequest restReq0 = new GridRestBaselineRequest(); + + restReq0.topologyVersion(longValue("topVer", params, null)); + restReq0.consistentIds(new Converter().values(null, "consistentId", params)); + + restReq = restReq0; + + break; + } + + case AUTHENTICATE: { + restReq = new GridRestRequest(); + + break; + } + + case ADD_USER: + case REMOVE_USER: + case UPDATE_USER: { + RestUserActionRequest restReq0 = new RestUserActionRequest(); + + restReq0.user(params.get("user")); + restReq0.password(params.get("password")); + + restReq = restReq0; + + break; + } + + case EXECUTE_SQL_QUERY: + case EXECUTE_SQL_FIELDS_QUERY: { + RestQueryRequest restReq0 = new RestQueryRequest(); + + String cacheName = params.get(CACHE_NAME_PARAM); + + restReq0.sqlQuery(params.get("qry")); + + restReq0.arguments(new Converter(cacheName).values(null, "arg", params).toArray()); + + restReq0.typeName(params.get("type")); + + Object pageSize = params.get("pageSize"); + + if (pageSize != null) + restReq0.pageSize(Integer.parseInt(pageSize.toString())); + + Object keepBinary = params.get("keepBinary"); + + if (keepBinary != null) + restReq0.keepBinary(Boolean.parseBoolean(keepBinary.toString())); + + Object distributedJoins = params.get("distributedJoins"); + + if (distributedJoins != null) + restReq0.distributedJoins(Boolean.parseBoolean(distributedJoins.toString())); + + restReq0.cacheName(cacheName); + + if (cmd == EXECUTE_SQL_QUERY) + restReq0.queryType(RestQueryRequest.QueryType.SQL); + else + restReq0.queryType(RestQueryRequest.QueryType.SQL_FIELDS); + + restReq = restReq0; + + break; + } + + case EXECUTE_SCAN_QUERY: { + RestQueryRequest restReq0 = new RestQueryRequest(); + + restReq0.sqlQuery(params.get("qry")); + + String pageSize = params.get("pageSize"); + + if (pageSize != null) + restReq0.pageSize(Integer.parseInt(pageSize)); + + restReq0.cacheName(params.get(CACHE_NAME_PARAM)); + + restReq0.className(params.get("className")); + + String keepBinary = params.get("keepBinary"); + + if (keepBinary != null) + restReq0.keepBinary(Boolean.parseBoolean(keepBinary)); + + restReq0.queryType(RestQueryRequest.QueryType.SCAN); + + restReq = restReq0; + + break; + } + + case FETCH_SQL_QUERY: { + RestQueryRequest restReq0 = new RestQueryRequest(); + + String qryId = params.get("qryId"); + + if (qryId != null) + restReq0.queryId(Long.parseLong(qryId)); + + String pageSize = params.get("pageSize"); + + if (pageSize != null) + restReq0.pageSize(Integer.parseInt(pageSize)); + + restReq0.cacheName(params.get(CACHE_NAME_PARAM)); + + restReq = restReq0; + + break; + } + + case CLOSE_SQL_QUERY: { + RestQueryRequest restReq0 = new RestQueryRequest(); + + String qryId = params.get("qryId"); + + if (qryId != null) + restReq0.queryId(Long.parseLong(qryId)); + + restReq0.cacheName(params.get(CACHE_NAME_PARAM)); + + restReq = restReq0; + + break; + } + + case NODE_STATE_BEFORE_START: { + restReq = new GridRestNodeStateBeforeStartRequest(); + + break; + } + + case WARM_UP: { + GridRestWarmUpRequest restReq0 = new GridRestWarmUpRequest(); + + restReq0.stopWarmUp(Boolean.parseBoolean(String.valueOf(params.get("stopWarmUp")))); + + restReq = restReq0; + + break; + } + + default: + throw new IgniteCheckedException("Invalid command: " + cmd); + } + + restReq.address(new InetSocketAddress("127.0.0.1", 1)); + + restReq.command(cmd); + + + // TODO: In IGNITE 3.0 we should check credentials only for AUTHENTICATE command. + if (!credentials(params, IGNITE_LOGIN, IGNITE_PASSWORD, restReq)) + credentials(params, USER_PARAM, PWD_PARAM, restReq); + + String clientId = params.get("clientId"); + + try { + if (clientId != null) + restReq.clientId(UUID.fromString(clientId)); + } + catch (Exception ignored) { + // Ignore invalid client id. Rest handler will process this logic. + } + + String destId = params.get("destId"); + + try { + if (destId != null) + restReq.destinationId(UUID.fromString(destId)); + } + catch (IllegalArgumentException ignored) { + // Don't fail - try to execute locally. + } + + String sesTokStr = params.get("sessionToken"); + + try { + if (sesTokStr != null) { + // Token is a UUID encoded as 16 bytes as HEX. + byte[] bytes = U.hexString2ByteArray(sesTokStr); + + if (bytes.length == 16) + restReq.sessionToken(bytes); + } + } + catch (IllegalArgumentException ignored) { + // Ignore invalid session token. + } + + return restReq; + } + + /** + * @param params Parameters. + * @param userParam Parameter name to take user name. + * @param pwdParam Parameter name to take password. + * @param restReq Request to add credentials if any. + * @return {@code true} If params contains credentials. + */ + private boolean credentials(Map params, String userParam, String pwdParam, + GridRestRequest restReq) { + boolean hasCreds = params.containsKey(userParam) || params.containsKey(pwdParam); + + if (hasCreds) { + SecurityCredentials cred = new SecurityCredentials(params.get(userParam), params.get(pwdParam)); + + restReq.credentials(cred); + } + + return hasCreds; + } + + /** + * @param req Request. + * @return Command. + */ + @Nullable private GridRestCommand command(JsonObject req) { + String cmd = req.getString("cmd"); + + return cmd == null ? null : GridRestCommand.fromKey(cmd.toLowerCase()); + } + + /** + * Converter from string into specified type. + */ + private class Converter { + /** Cache name. */ + private final String cacheName; + + /** + * @param cacheName Cache name. + */ + private Converter(String cacheName) { + this.cacheName = cacheName; + } + + /** */ + private Converter() { + this(null); + } + + /** + * Gets and converts values referenced by sequential keys, e.g. {@code key1...keyN}. + * + * @param type Optional value type. + * @param keyPrefix Key prefix, e.g. {@code key} for {@code key1...keyN}. + * @param params Parameters map. + * @return Values. + * @throws IgniteCheckedException If failed to convert. + */ + private List values(String type, String keyPrefix, + Map params) throws IgniteCheckedException { + assert keyPrefix != null; + + List vals = new LinkedList<>(); + + for (int i = 1; ; i++) { + String key = keyPrefix + i; + + if (params.containsKey(key)) + vals.add(convert(type, params.get(key))); + else + break; + } + + return vals; + } + + /** + * @param type Optional value type. + * @param str String to convert. + * @return Converted value. + * @throws IgniteCheckedException If failed to convert. + */ + private Object convert(@Nullable String type, @Nullable String str) throws IgniteCheckedException { + if (F.isEmpty(type) || str == null) + return str; + + try { + switch (type.toLowerCase()) { + case "boolean": + case "java.lang.boolean": + return Boolean.valueOf(str); + + case "byte": + case "java.lang.byte": + return Byte.valueOf(str); + + case "short": + case "java.lang.short": + return Short.valueOf(str); + + case "int": + case "integer": + case "java.lang.integer": + return Integer.valueOf(str); + + case "long": + case "java.lang.long": + return Long.valueOf(str); + + case "float": + case "java.lang.float": + return Float.valueOf(str); + + case "double": + case "java.lang.double": + return Double.valueOf(str); + + case "date": + case "java.sql.date": + return Date.valueOf(str); + + case "time": + case "java.sql.time": + return Time.valueOf(str); + + case "timestamp": + case "java.sql.timestamp": + return Timestamp.valueOf(str); + + case "uuid": + case "java.util.uuid": + return UUID.fromString(str); + + case "igniteuuid": + case "org.apache.ignite.lang.igniteuuid": + return IgniteUuid.fromString(str); + + case "string": + case "java.lang.string": + return str; + } + + // Creating an object of the specified type, if its class is available. + Class cls = U.classForName(type, null); + + if (cls != null) + return jsonMapper.readValue(str, cls); + + // Creating a binary object if the type is not a class name or it cannot be loaded. + InjectableValues.Std prop = new InjectableValues.Std() + .addValue(IgniteBinaryObjectJsonDeserializer.BINARY_TYPE_PROPERTY, type) + .addValue(IgniteBinaryObjectJsonDeserializer.CACHE_NAME_PROPERTY, cacheName); + + return jsonMapper.reader(prop).forType(BinaryObject.class).readValue(str); + } + catch (Throwable e) { + throw new IgniteCheckedException("Failed to convert value to specified type [type=" + type + + ", val=" + str + ", reason=" + e.getClass().getName() + ": " + e.getMessage() + "]", e); + } + } + } +} diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/JdbcExecutor.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/JdbcExecutor.java index c390c8efdbff9..c1b7db96b62ed 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/JdbcExecutor.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/JdbcExecutor.java @@ -77,11 +77,9 @@ public class JdbcExecutor implements AutoCloseable { private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(JdbcExecutor.class)); public static final JsonNodeFactory jsonNodeFactory = new JsonNodeFactory(true); - private DatabaseListener dbListener; - - + private DatabaseListener dbListener; - int jdbcQueryCancellationTime = 10000; + private int jdbcQueryCancellationTime = 10000; private DbMetadataReader metadataReader = new DbMetadataReader(); @@ -179,8 +177,12 @@ else if("top".equals(cmd)) { node.put("replicaCount", dbInfo.top.getNodes().size()); node.put("attributes", dbInfo.getJdbcProp()); node.put("isActive", dbInfo.top.isActive()); - node.getJsonObject("attributes").put("database.version", dbInfo.driverCls); - node.getJsonObject("attributes").put("os.name", "Linux"); + node.getJsonObject("attributes").put("database.driverCls", dbInfo.driverCls); + if(params.getBoolean("attr",false)) { + System.getenv().forEach((k,v)->{ + node.getJsonObject("attributes").put(k, v); + }); + } result.add(node); res.put("result", result); @@ -188,7 +190,7 @@ else if("top".equals(cmd)) { } else if("metadata".equals(cmd)) { - List schemas = new ArrayList<>(2); + List schemas = new ArrayList<>(); String schema = params.getString("cacheName"); if(!StringUtil.isEmpty(schema)) { schemas.add(schema); @@ -202,7 +204,7 @@ else if("metadata".equals(cmd)) { ArrayNode arr = new ArrayNode(jsonNodeFactory); ObjectNode cachesMap = new ObjectNode(jsonNodeFactory); for (DbTable table: meta) { - ObjectNode caches = cachesMap.with(table.getSchema()); + ObjectNode caches = cachesMap.withObject(table.getSchema()); if(caches.isEmpty()) { caches.put("cacheName",table.getSchema()); arr.add(caches); @@ -213,7 +215,7 @@ else if("metadata".equals(cmd)) { ArrayNode types = caches.withArray("types"); types.add(typeName); - ObjectNode fields = caches.with("fields"); + ObjectNode fields = caches.withObject("fields"); ObjectNode column = new ObjectNode(jsonNodeFactory); for(DbColumn col: table.getColumns()) { @@ -228,7 +230,7 @@ else if("metadata".equals(cmd)) { fields.set(typeName, column); - ObjectNode indexes = caches.with("indexes"); + ObjectNode indexes = caches.withObject("indexes"); ArrayNode index = new ArrayNode(jsonNodeFactory); for(VisorQueryIndex idx: table.getIndexes()) { diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java index f9b0dafe8d537..92affff0f369b 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java @@ -52,7 +52,7 @@ public class RestExecutor implements AutoCloseable { /** */ private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(RestExecutor.class)); /** */ - public static QueuedThreadPool executor = new QueuedThreadPool(32); + public static QueuedThreadPool executor = new QueuedThreadPool(64); static { executor.setName("Agent http client"); } @@ -65,6 +65,7 @@ public class RestExecutor implements AutoCloseable { */ public RestExecutor(SslContextFactory sslCtxFactory) { httpClient = new HttpClient(sslCtxFactory); + httpClient.setMaxConnectionsPerDestination(1); httpClient.setExecutor(executor); } diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java index 13dbfefa770a4..4ab326809b57a 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonRawValue; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.apache.ignite.console.json.RawContentDeserializer; +import org.apache.ignite.internal.processors.rest.GridRestResponse; /** * Request result. @@ -88,6 +89,15 @@ private RestResult( public static RestResult fail(int status, String error) { return new RestResult(status, error, null, null); } + + /** + * @param status REST http code. + * @param error The field contains description of error if server could not handle the request. + * @return Request result. + */ + public static RestResult authFail(String error) { + return new RestResult(GridRestResponse.STATUS_AUTH_FAILED, error, null, null); + } /** * @param data The field contains result of command. diff --git a/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java index 2c64dc103afa8..5b8106cd7274d 100644 --- a/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java +++ b/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java @@ -33,6 +33,7 @@ import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.Ignition; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -78,13 +79,13 @@ public class AgentClusterDemo { private static final AtomicBoolean initGuard = new AtomicBoolean(); /** */ - public static final String SRV_NODE_NAME = "demo-server-"; + public static final String SRV_NODE_NAME = "demo-server"; /** */ - private static final String CLN_NODE_NAME = "demo-client-"; + public static final String CLN_NODE_NAME = "demo-client"; - /** Node count 2 means 3 node*/ - private static final int NODE_CNT = 2; + /** Node count */ + private static final int NODE_CNT = 10; /** */ private static final int WAL_SEGMENTS = 5; @@ -92,8 +93,6 @@ public class AgentClusterDemo { /** WAL file segment size, 16MBytes. */ private static final int WAL_SEGMENT_SZ = 16 * 1024 * 1024; - /** */ - private static CountDownLatch initLatch = new CountDownLatch(1); /** */ private static volatile String demoUrl; @@ -111,10 +110,10 @@ private static IgniteConfiguration igniteConfiguration(IgniteConfiguration cfg, cfg.setGridLogger(new Slf4jLogger()); - cfg.setIgniteInstanceName((client ? CLN_NODE_NAME : SRV_NODE_NAME) + gridIdx); + cfg.setIgniteInstanceName((client ? CLN_NODE_NAME : SRV_NODE_NAME)); cfg.setLocalHost("127.0.0.1"); cfg.setEventStorageSpi(new MemoryEventStorageSpi()); - cfg.setConsistentId(cfg.getIgniteInstanceName()); + cfg.setConsistentId(cfg.getIgniteInstanceName()+"_"+ gridIdx); File workDir = new File(U.workDirectory(null, null), "demo-work"); @@ -127,7 +126,7 @@ private static IgniteConfiguration igniteConfiguration(IgniteConfiguration cfg, cfg.setIncludeEventTypes(evts); - cfg.getConnectorConfiguration().setPort(basePort); + cfg.getConnectorConfiguration().setPort(basePort + gridIdx); System.setProperty(IGNITE_JETTY_PORT, String.valueOf(basePort + 10 + gridIdx)); @@ -162,7 +161,7 @@ private static IgniteConfiguration igniteConfiguration(IgniteConfiguration cfg, dataRegCfg.setName("demo"); dataRegCfg.setMetricsEnabled(true); dataRegCfg.setMaxSize(DFLT_DATA_REGION_INITIAL_SIZE); - dataRegCfg.setPersistenceEnabled(!true); + dataRegCfg.setPersistenceEnabled(false); dataRegCfg.setLazyMemoryAllocation(true); DataStorageConfiguration dataStorageCfg = new DataStorageConfiguration(); @@ -209,7 +208,7 @@ public static String getDemoUrl() { /** * Start ignite node with cacheEmployee and populate it with data. */ - public static CountDownLatch tryStart(IgniteConfiguration cfg) { + public static Ignite tryStart(IgniteConfiguration cfg, int idx, boolean lastNode) { if (initGuard.compareAndSet(false, true)) { log.info("DEMO: Starting embedded nodes for demo..."); @@ -222,93 +221,92 @@ public static CountDownLatch tryStart(IgniteConfiguration cfg) { System.setProperty(IGNITE_SQL_DISABLE_SYSTEM_VIEWS, "false"); final AtomicInteger basePort = new AtomicInteger(60700); - final AtomicInteger cnt = new AtomicInteger(-1); - - final ScheduledExecutorService execSrv = newScheduledThreadPool(1, "demo-nodes-start"); - - execSrv.scheduleWithFixedDelay(() -> { - int idx = cnt.incrementAndGet(); - int port = basePort.get(); - - boolean first = idx == 0; + + int port = basePort.get(); + Ignite ignite = null; + + try { + igniteConfiguration(cfg, port, idx, false); + + if (lastNode) { + U.delete(Paths.get(cfg.getWorkDirectory())); + + U.resolveWorkDirectory( + cfg.getWorkDirectory(), + cfg.getDataStorageConfiguration().getStoragePath(), + true + ); + cfg.setNodeId(UUID.fromString(DemoClusterHandler.DEMO_CLUSTER_ID)); + } + else { + cfg.setNodeId(null); + cfg.setClusterStateOnStart(ClusterState.INACTIVE); + } - try { - igniteConfiguration(cfg, port, idx, false); + ignite = Ignition.start(cfg); - if (first) { - U.delete(Paths.get(cfg.getWorkDirectory())); + if (ignite!=null) { + ClusterNode node = ignite.cluster().localNode(); - U.resolveWorkDirectory( - cfg.getWorkDirectory(), - cfg.getDataStorageConfiguration().getStoragePath(), - true - ); - cfg.setNodeId(UUID.fromString(DemoClusterHandler.DEMO_CLUSTER_ID)); + Collection jettyAddrs = node.attribute(ATTR_REST_JETTY_ADDRS); + Integer jettyPort = node.attribute(ATTR_REST_JETTY_PORT); + if (jettyAddrs==null || jettyPort == null) { + ignite.close(); + throw new IgniteException("DEMO: Failed to start Jetty REST server on embedded node"); } - else { - cfg.setNodeId(null); + + String jettyHost = "127.0.0.1"; + for(String host: jettyAddrs) { + if(!host.startsWith("0")) { + jettyHost = host; + } } - Ignite ignite = Ignition.start(cfg); + log.info("Cluster: Started embedded node for data analysis purpose [TCP binary port={}, Jetty REST port={}]", ignite.configuration().getConnectorConfiguration().getPort(), jettyPort); - if (first) { - ClusterNode node = ignite.cluster().localNode(); + String nodeUrl = String.format("http://%s:%d/%s", jettyHost, jettyPort, ignite.configuration().getIgniteInstanceName()); - Collection jettyAddrs = node.attribute(ATTR_REST_JETTY_ADDRS); + demoUrl = nodeUrl; + + } + } + catch (Throwable e) { + if (lastNode) { + basePort.getAndAdd(50); - if (jettyAddrs == null) { - Ignition.stopAll(true); - - throw new IgniteException("DEMO: Failed to start Jetty REST server on embedded node"); - } - demoUrl = IgniteClusterLauncher.registerNodeUrl(ignite); - - initLatch.countDown(); - } + log.warn("DEMO: Failed to start embedded node.", e); } - catch (Throwable e) { - if (first) { - basePort.getAndAdd(50); - - log.warn("DEMO: Failed to start embedded node.", e); - } - else - log.error("DEMO: Failed to start embedded node.", e); - } - finally { - if (idx == NODE_CNT) { - try { - Ignite ignite = Ignition.ignite(SRV_NODE_NAME + 0); - - if (ignite != null) { - ignite.cluster().active(true); - - deployServices(ignite.services(ignite.cluster().forServers())); - } - - log.info("DEMO: All embedded nodes for demo successfully started"); - } - catch (Throwable ignored) { - log.info("DEMO: Failed to launch demo load"); - } - - execSrv.shutdown(); + else + log.error("DEMO: Failed to start embedded node.", e); + } + finally { + if (lastNode && ignite != null) { + try { + Thread.sleep(1000*idx); + ignite.cluster().state(ClusterState.ACTIVE); + + deployServices(ignite.services(ignite.cluster().forServers())); + + log.info("DEMO: All embedded nodes for demo successfully started"); } + catch (Throwable ignored) { + log.info("DEMO: Failed to launch demo load"); + } } - }, 1, 5, TimeUnit.SECONDS); + } + return ignite; } - - return initLatch; + return null; } /** */ public static void stop() { demoUrl = null; - Ignition.stopAll(true); - - initLatch = new CountDownLatch(1); - + Ignition.stop(SRV_NODE_NAME,true); + + Ignition.stop(CLN_NODE_NAME,true); + initGuard.compareAndSet(true, false); } } diff --git a/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketRequest.java b/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketRequest.java index a782fa4e24585..77adb0eb3c253 100644 --- a/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketRequest.java +++ b/web-console/web-console-common/src/main/java/org/apache/ignite/console/websocket/WebSocketRequest.java @@ -40,6 +40,10 @@ public class WebSocketRequest implements WebSocketEvent { /** */ @GridToStringInclude private String payload; + + private int nodeSeq = 0; + + private boolean isLastNode = false; /** {@inheritDoc} */ @Override public String getRequestId() { @@ -100,4 +104,20 @@ public WebSocketResponse withError(String prefix, Throwable e) { @Override public String toString() { return S.toString(WebSocketRequest.class, this); } + + public int getNodeSeq() { + return nodeSeq; + } + + public void setNodeSeq(int nodeSeq) { + this.nodeSeq = nodeSeq; + } + + public boolean isLastNode() { + return isLastNode; + } + + public void setLastNode(boolean isLastNode) { + this.isLastNode = isLastNode; + } } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/dto/Notebook.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/dto/Notebook.java index 88de3b12d65fb..45f77f5c96878 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/dto/Notebook.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/dto/Notebook.java @@ -361,7 +361,10 @@ public enum QueryType { SQL_FIELDS, /** */ - SCAN + SCAN, + + /** graph */ + GREMLIN } /** diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/services/DemoService.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/services/DemoService.java index 13045074ccbb9..2ef960fed00ea 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/services/DemoService.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/services/DemoService.java @@ -21,7 +21,8 @@ import java.util.Collections; import java.util.List; import java.util.UUID; -import com.fasterxml.jackson.core.type.TypeReference; + +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.apache.ignite.console.repositories.ConfigurationsRepository; import org.apache.ignite.console.web.model.ConfigurationKey; @@ -46,7 +47,7 @@ public class DemoService { private static final Logger log = LoggerFactory.getLogger(DemoService.class); /** Demo clusters. */ - private List clusters; + private JsonArray clusters; /** Repository to work with configurations. */ private final ConfigurationsRepository cfgsRepo; @@ -107,7 +108,7 @@ public void reset(UUID accId) { String content = FileCopyUtils.copyToString(new InputStreamReader(res.getInputStream(), UTF_8)); - clusters = fromJson(content, new TypeReference>() { }); + clusters = new JsonArray(content); } catch (Exception e) { log.error("Failed to get demo clusters", e); @@ -115,7 +116,8 @@ public void reset(UUID accId) { } if (!F.isEmpty(clusters)) { - for (JsonObject json : clusters) { + for (int i=0;i findLocalAgent(AgentKey key) { .findFirst() .map(Map.Entry::getKey); } + + /** + * @param key Agent key. + */ + public List findLocalAgents(AgentKey key) { + return locAgents.entrySet().stream() + .filter((e) -> { + Set accIds = e.getValue().getAccIds(); + Set clusterIds = e.getValue().getClusterIds(); + + UUID accId = key.getAccId(); + String clusterId = key.getClusterId(); + + if (F.isEmpty(clusterId)) + return accIds.contains(accId); + + if (accId == null) + return clusterIds.contains(clusterId); + + return accIds.contains(accId) && clusterIds.contains(clusterId); + }) + .map(Map.Entry::getKey).collect(java.util.stream.Collectors.toList()); + } /** * @param accIds Account IDs. @@ -429,5 +455,8 @@ private void updateAgentsCache(WebSocketSession ses, Set accIds) { locAgents.put(ses, new AgentSession(accIds)); } - + public void sendMessageWithResponse(WebSocketSession ses, WebSocketEvent evt, UUID fromNodeId) throws IOException { + super.sendMessage(ses, evt); + srcOfRequests.put(evt.getRequestId(), fromNodeId); + } } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/BrowsersService.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/BrowsersService.java index b97a74c253a01..afa7d56dda1b4 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/BrowsersService.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/BrowsersService.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -182,6 +183,44 @@ public void sendToAgent(AgentKey key, WebSocketRequest evt) { case AGENT_START_CLUSTER: case AGENT_STOP_CLUSTER: + try { + payload = fromJson(evt.getPayload()); + + clusterId = payload.getString("id"); + + if (F.isEmpty(clusterId)) + throw new IllegalStateException(messages.getMessage("err.missing-cluster-id-param")); + + // last node will active cluster + int nodeIndex = 0; + AgentKey key = new AgentKey(accId); + List nids = agentsSrvc.findLocalAgents(key); + if (!nids.isEmpty()) { + for(WebSocketSession sess: nids) { + evt.setNodeSeq(nodeIndex); + if(nodeIndex==nids.size()-1) { + evt.setLastNode(true); + agentsSrvc.sendMessageWithResponse(sess, evt, this.transitionSrvc.localNodeId()); + } + else { + agentsSrvc.sendMessage(sess, evt); + } + nodeIndex++; + } + } + else { + log.warn("Not found any cluster agent for : " + evt); + sendMessageQuiet(ses, evt.response("Failed to send event to agent: Not found any cluster agent")); + } + } + catch (ClusterGroupEmptyException ignored) { + // No-op. + } + catch (Exception e) { + log.warn("Failed to send response to browser: " + evt, e); + } + break; + case AGENT_CALL_CLUSTER_SERVICE: case AGENT_CALL_CLUSTER_COMMAND: try { @@ -314,7 +353,9 @@ protected void registerVisorTasks() { registerVisorTask("querySqlX2", igniteCmd("qryfldexe"), Map.class.getName()); // scanf - registerVisorTask("queryScanX2", igniteCmd("qryscanexe"), Map.class.getName()); + registerVisorTask("queryScanX2", igniteCmd("qryscanexe"), Map.class.getName()); + // gremlin + registerVisorTask("queryGremlin", igniteCmd("qrygremlinexe"), Map.class.getName()); registerVisorTask("queryFetch", igniteCmd("qryfetch"), Map.class.getName()); @@ -392,7 +433,6 @@ protected JsonObject fillVisorGatewayTaskParams(JsonObject payload) { return payload; } - } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/TransitionService.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/TransitionService.java index 741c5d6e50ab8..700895d4b8d38 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/TransitionService.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/TransitionService.java @@ -218,4 +218,8 @@ public void sendToBrowser(UserKey userKey, WebSocketResponse evt) { public void sendResponse(UUID nid, WebSocketRequest evt) { ignite.message(ignite.cluster().forNodeId(nid)).send(SEND_RESPONSE, evt); } + + public UUID localNodeId() { + return ignite.cluster().localNode().id(); + } } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketConfig.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketConfig.java index 4d235038f7610..5da5ec049a99b 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketConfig.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketConfig.java @@ -11,6 +11,7 @@ import org.springframework.web.socket.config.annotation.WebSocketConfigurationSupport; import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean; import static org.apache.ignite.console.websocket.WebSocketEvents.AGENTS_PATH; import static org.apache.ignite.console.websocket.WebSocketEvents.BROWSERS_PATH; @@ -41,9 +42,21 @@ public WebSocketConfig(AgentsService agentsSrvc, BrowsersService browsersSrvc) { * @param registry Registry. */ @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(agentsSrvc, AGENTS_PATH);//.setAllowedOrigins("*"); - registry.addHandler(browsersSrvc, BROWSERS_PATH);//.setAllowedOrigins("*"); + registry.addHandler(agentsSrvc, AGENTS_PATH).setAllowedOrigins("*"); + registry.addHandler(browsersSrvc, BROWSERS_PATH).setAllowedOrigins("*"); + } + @Bean + public ServletServerContainerFactoryBean createWebSocketContainer() { + ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean(); + // 在此处设置bufferSize + container.setMaxTextMessageBufferSize(WebSocketMessageBrokerConfig.MSG_SIZE); + container.setMaxBinaryMessageBufferSize(WebSocketMessageBrokerConfig.MSG_SIZE); + container.setMaxSessionIdleTimeout(60 * 60000L); + return container; + } + + } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketMessageBrokerConfig.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketMessageBrokerConfig.java new file mode 100644 index 0000000000000..f7cbc479bcbe0 --- /dev/null +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/socket/WebSocketMessageBrokerConfig.java @@ -0,0 +1,15 @@ +package org.apache.ignite.console.web.socket; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketTransportRegistration; + +@Configuration +public class WebSocketMessageBrokerConfig implements WebSocketMessageBrokerConfigurer { + public static int MSG_SIZE = 1024*1024; + + public void configureWebSocketTransport(WebSocketTransportRegistration registry) { + registry.setMessageSizeLimit(MSG_SIZE); + registry.setSendBufferSizeLimit(MSG_SIZE); + } +} diff --git a/web-console/web-console-server/src/main/resources/application.yml b/web-console/web-console-server/src/main/resources/application.yml index d29667e84b73f..90356df940e91 100644 --- a/web-console/web-console-server/src/main/resources/application.yml +++ b/web-console/web-console-server/src/main/resources/application.yml @@ -19,7 +19,7 @@ server: acceptors: 1 # acceptor线程数 selectors: 2 # selector线程数 max-http-post-size: 1073741824 # put或post方法最大字节数 - max-threads: 8 + max-threads: 256 spring: mvc: