From 25e695b5dac0ed58c5f90f9811423bd20557677d Mon Sep 17 00:00:00 2001 From: zhangjunfeng Date: Wed, 25 Oct 2023 10:14:32 +0800 Subject: [PATCH] remove-async --- bin/control.bat | 8 +- bin/control.sh | 12 +- bin/gremlin.bat | 10 +- bin/gremlin.sh | 20 +- bin/ignite.bat | 10 +- bin/ignite.sh | 12 +- bin/igniterouter.sh | 106 ++--- bin/index-reader.bat | 6 +- bin/index-reader.sh | 8 +- .../shaofan/s3/FileManagerInitializer.java | 2 +- .../s3/controller/ConsoleController.java | 1 + .../s3/controller/FileManagerController.java | 1 + .../s3/controller/JSONObjectController.java | 120 +++-- .../shaofan/s3/controller/S3Controller.java | 8 +- .../java/org/shaofan/s3/util/DateUtil.java | 13 +- .../resources/META-INF/springmvc-servlet.xml | 50 +++ .../modules/ignite-graph/pom.xml | 6 +- .../ignite/gremlin/IgniteVertexProperty.java | 38 +- .../process/computer/TinkerGraphComputer.java | 11 +- .../process/computer/TinkerWorkerPool.java | 1 - .../works-ignite-gremlin-server/pom.xml | 6 + .../store/GridCacheStoreManagerAdapter.java | 17 +- .../TransactionEventProxyImpl.java | 17 +- .../transactions/TransactionProxyImpl.java | 17 - .../ignite/transactions/Transaction.java | 2 +- .../GridCacheAbstractFullApiSelfTest.java | 47 +- ...gniteCacheConfigVariationsFullApiTest.java | 46 -- .../cache/GridAbstractCacheStoreSelfTest.java | 17 +- .../ml/clustering/kmeans/KMeansTrainer.java | 2 +- .../org/apache/ignite/ml/knn/KNNModel.java | 11 +- .../ml/knn/utils/PointWithDistanceUtil.java | 8 +- .../utils/indices/BallTreeSpatialIndex.java | 389 ++++------------ .../knn/utils/indices/KDTreeSpatialIndex.java | 417 ++++++++++++++---- .../primitives/matrix/AbstractMatrix.java | 3 +- .../vector/impl/DelegatingVector.java | 10 +- modules/mongodb-relay/.factorypath | 7 +- .../mongo/backend/ignite/IgniteBackend.java | 24 + .../ignite/IgniteBinaryCollection.java | 36 +- .../mongo/backend/ignite/IgniteDatabase.java | 6 +- .../backend/ignite/IgniteVectorIndex.java | 47 +- .../backend/ignite/util/DocumentUtil.java | 27 +- .../backend/ignite/util/TransformerUtil.java | 71 +++ .../de/bwaldvogel/examples/SimpleTest.java | 2 +- .../bwaldvogel/examples/TransactionTest.java | 159 +++++++ web-console/web-console-server/pom.xml | 6 + .../apache/ignite/console/db/CacheHolder.java | 2 +- .../ignite/console/db/NestedTransaction.java | 19 +- .../org/apache/ignite/console/db/Table.java | 35 +- .../apache/ignite/console/db/UniqueIndex.java | 37 ++ .../web/security/IgniteSessionRepository.java | 1 + .../ignite/console/MockConfiguration.java | 21 +- 51 files changed, 1123 insertions(+), 829 deletions(-) create mode 100644 ignite-extensions/modules/filemanager/src/main/resources/META-INF/springmvc-servlet.xml create mode 100644 modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/util/TransformerUtil.java create mode 100644 modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/TransactionTest.java create mode 100644 web-console/web-console-server/src/main/java/org/apache/ignite/console/db/UniqueIndex.java diff --git a/bin/control.bat b/bin/control.bat index 4174125c00751..970f43521125c 100644 --- a/bin/control.bat +++ b/bin/control.bat @@ -116,7 +116,7 @@ if "%OS%" == "Windows_NT" set PROG_NAME=%~nx0% :: Set IGNITE_LIBS :: call "%SCRIPTS_HOME%\include\setenv.bat" -call "%SCRIPTS_HOME%\include\build-classpath.bat" + set CP=%IGNITE_LIBS%;%IGNITE_HOME%\libs\optional\ignite-zookeeper\* :: @@ -165,7 +165,7 @@ if %ERRORLEVEL% equ 0 ( :: Assertions are disabled by default since version 3.5. :: If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. :: -set ENABLE_ASSERTIONS=1 +set ENABLE_ASSERTIONS=0 :: :: Set '-ea' options if assertions are enabled. @@ -198,11 +198,11 @@ if defined JVM_OPTS ( if "%INTERACTIVE%" == "1" ( "%JAVA_HOME%\bin\java.exe" %CONTROL_JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% ^ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ + -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ -cp "%CP%" %MAIN_CLASS% %* ) else ( "%JAVA_HOME%\bin\java.exe" %CONTROL_JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% ^ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ + -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ -cp "%CP%" %MAIN_CLASS% %* ) diff --git a/bin/control.sh b/bin/control.sh index 0e41be7faf37d..287fa4df78c4d 100644 --- a/bin/control.sh +++ b/bin/control.sh @@ -63,7 +63,7 @@ fi # Set IGNITE_LIBS. # . "${SCRIPTS_HOME}"/include/setenv.sh -. "${SCRIPTS_HOME}"/include/build-classpath.sh # Will be removed in the binary release. + CP="${IGNITE_LIBS}:${IGNITE_HOME}/libs/optional/ignite-zookeeper/*" RANDOM_NUMBER=$("$JAVA" -cp "${CP}" org.apache.ignite.startup.cmdline.CommandLineRandomNumberGenerator) @@ -113,7 +113,7 @@ fi # Assertions are disabled by default since version 3.5. # If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. # -ENABLE_ASSERTIONS="1" +ENABLE_ASSERTIONS="0" # # Set '-ea' options if assertions are enabled. @@ -148,12 +148,12 @@ fi case $osname in Darwin*) "$JAVA" ${CONTROL_JVM_OPTS} ${QUIET:-} "${DOCK_OPTS}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ - -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} "$@" + -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} $@ ;; *) "$JAVA" ${CONTROL_JVM_OPTS} ${QUIET:-} \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ - -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} "$@" + -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} $@ ;; esac diff --git a/bin/gremlin.bat b/bin/gremlin.bat index 3c8cb7ebc90da..444714333dc0a 100644 --- a/bin/gremlin.bat +++ b/bin/gremlin.bat @@ -17,7 +17,7 @@ :: Windows launcher script for Gremlin Console -@ECHO ON +@ECHO OFF SETLOCAL EnableDelayedExpansion SET work=%CD% @@ -65,8 +65,12 @@ call "%IGNITE_HOME%\bin\include\jvmdefaults.bat" %MAJOR_JAVA_VER% "%JVM_OPTS%" J CD %IGNITE_HOME% +:: +:: Set IGNITE_LIBS +:: +call "%IGNITE_HOME%\bin\include\setenv.bat" -SET CP=%CLASSPATH%;libs\ignite-gremlin-server\*;libs\*;libs\ignite-log4j\*;exts\* +SET CP=%CLASSPATH%;%IGNITE_LIBS%;exts\* :: jline.terminal workaround for https://issues.apache.org/jira/browse/GROOVY-6453 :: to debug plugin :install include -Divy.message.logger.level=4 -Dgroovy.grape.report.downloads=true @@ -95,6 +99,8 @@ IF "%1" == "-jpda" GOTO debug java %JAVA_OPTIONS% %JAVA_ARGS% -cp %CP% org.apache.tinkerpop.gremlin.console.Console %* +GOTO finally + :debug SET debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=19003 diff --git a/bin/gremlin.sh b/bin/gremlin.sh index 7d72a625aa2f6..3f29a443cd4dd 100644 --- a/bin/gremlin.sh +++ b/bin/gremlin.sh @@ -47,26 +47,19 @@ EXT=$(pwd) cd .. -SYSTEM_EXT_DIR="`pwd`/ext" +SYSTEM_EXT_DIR="`pwd`/exts" +IGNITE_HOME=`pwd` +echo IGNITE_HOME=$IGNITE_HOME JAVA_OPTIONS=${JAVA_OPTIONS:-} -if [ ! -z "${JAVA_OPTIONS}" ]; then - USER_EXT_DIR=$(grep -o '\-Dtinkerpop.ext=\(\([^"][^ ]*\)\|\("[^"]*"\)\)' <<< "${JAVA_OPTIONS}" | cut -f2 -d '=' | xargs -0 echo) - if [ ! -z "${USER_EXT_DIR}" -a ! -d "${USER_EXT_DIR}" ]; then - mkdir -p "${USER_EXT_DIR}" - cp -R ${SYSTEM_EXT_DIR}/* ${USER_EXT_DIR}/ - fi -fi - # # Set IGNITE_LIBS. # -. "${BIN}"/include/setenv.sh +. "${IGNITE_HOME}"/bin/include/setenv.sh - -CP=${SYSTEM_EXT_DIR}/*:${LIB}/*:${LIB}/ignite-gremlin-server/*:${LIB}/ignite-log4j/*:libs/netty/* +CP=${SYSTEM_EXT_DIR}/*:${LIB}/*:${IGNITE_LIBS} export CLASSPATH="${CLASSPATH:-}:$CP" @@ -107,7 +100,7 @@ if [ -z "${HADOOP_GREMLIN_LIBS:-}" ]; then export HADOOP_GREMLIN_LIBS="$LIB" fi -JAVA_OPTIONS="${JAVA_OPTIONS} -Duser.working_dir=${USER_DIR} -Dtinkerpop.ext=${USER_EXT_DIR:-${SYSTEM_EXT_DIR}} -Dlog4j.configurationFile=file:conf/log4j2-console.xml +JAVA_OPTIONS="${JAVA_OPTIONS} -Duser.working_dir=${USER_DIR} -Dtinkerpop.ext=${USER_EXT_DIR:-${SYSTEM_EXT_DIR}} -Dlog4j.configurationFile=file:config/log4j2-console.xml -cp $CP " JAVA_OPTIONS=$(awk -v RS=' ' '!/^$/ {if (!x[$0]++) print}' <<< "${JAVA_OPTIONS}" | grep -v '^$' | paste -sd ' ' -) if [ -n "$SCRIPT_DEBUG" ]; then @@ -119,3 +112,4 @@ fi # Start the JVM, execute the application, and return its exit code exec $JAVA $JAVA_OPTIONS $MAIN_CLASS $CFG "$@" + diff --git a/bin/ignite.bat b/bin/ignite.bat index 5bf74486de66c..091fa8fafa5fb 100644 --- a/bin/ignite.bat +++ b/bin/ignite.bat @@ -21,7 +21,7 @@ @echo off Setlocal EnableDelayedExpansion -set JVM_OPTS=-agentlib:jdwp=transport=dt_socket,address=0.0.0.0:19001,server=y,suspend=n + if "%OS%" == "Windows_NT" setlocal :: Check JAVA_HOME. @@ -115,7 +115,7 @@ if "%OS%" == "Windows_NT" set PROG_NAME=%~nx0% :: Set IGNITE_LIBS :: call "%SCRIPTS_HOME%\include\setenv.bat" -call "%SCRIPTS_HOME%\include\build-classpath.bat" &:: Will be removed in the binary release. + set CP=%IGNITE_LIBS% :: @@ -174,7 +174,7 @@ if %ERRORLEVEL% equ 0 ( :: Assertions are disabled by default since version 3.5. :: If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. :: -set ENABLE_ASSERTIONS=1 +set ENABLE_ASSERTIONS=0 :: :: Set '-ea' options if assertions are enabled. @@ -202,11 +202,11 @@ call "%SCRIPTS_HOME%\include\jvmdefaults.bat" %MAJOR_JAVA_VER% "%JVM_OPTS%" JVM_ if "%INTERACTIVE%" == "1" ( "%JAVA_HOME%\bin\java.exe" %JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% ^ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ + -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ -cp "%CP%" %MAIN_CLASS% ) else ( "%JAVA_HOME%\bin\java.exe" %JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% ^ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ + -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ -cp "%CP%" %MAIN_CLASS% "%CONFIG%" ) diff --git a/bin/ignite.sh b/bin/ignite.sh index 0bd6c5808d586..2d4095d042fed 100644 --- a/bin/ignite.sh +++ b/bin/ignite.sh @@ -69,7 +69,7 @@ fi # Set IGNITE_LIBS. # . "${SCRIPTS_HOME}"/include/setenv.sh -. "${SCRIPTS_HOME}"/include/build-classpath.sh # Will be removed in the binary release. + CP="${IGNITE_LIBS}" RANDOM_NUMBER=$("$JAVA" -cp "${CP}" org.apache.ignite.startup.cmdline.CommandLineRandomNumberGenerator) @@ -113,7 +113,7 @@ fi # Assertions are disabled by default since version 3.5. # If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. # -ENABLE_ASSERTIONS="1" +ENABLE_ASSERTIONS="0" # # Set '-ea' options if assertions are enabled. @@ -148,12 +148,12 @@ do case $osname in Darwin*) "$JAVA" ${JVM_OPTS} ${QUIET} "${DOCK_OPTS}" "${RESTART_SUCCESS_OPT}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS} -cp "${CP}" ${MAIN_CLASS} && ERRORCODE="$?" || ERRORCODE="$?" ;; *) "$JAVA" ${JVM_OPTS} ${QUIET} "${RESTART_SUCCESS_OPT}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS} -cp "${CP}" ${MAIN_CLASS} && ERRORCODE="$?" || ERRORCODE="$?" ;; esac @@ -161,12 +161,12 @@ do case $osname in Darwin*) "$JAVA" ${JVM_OPTS} ${QUIET} "${DOCK_OPTS}" "${RESTART_SUCCESS_OPT}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS} -cp "${CP}" ${MAIN_CLASS} "${CONFIG}" && ERRORCODE="$?" || ERRORCODE="$?" ;; *) "$JAVA" ${JVM_OPTS} ${QUIET} "${RESTART_SUCCESS_OPT}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS} -cp "${CP}" ${MAIN_CLASS} "${CONFIG}" && ERRORCODE="$?" || ERRORCODE="$?" ;; esac diff --git a/bin/igniterouter.sh b/bin/igniterouter.sh index d92b570f87a2b..af27f8bfea439 100755 --- a/bin/igniterouter.sh +++ b/bin/igniterouter.sh @@ -1,53 +1,53 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Router command line loader. -# - -# -# Import common functions. -# -if [ "${IGNITE_HOME}" = "" ]; then - IGNITE_HOME_TMP="$(dirname "$(cd "$(dirname "$0")"; "pwd")")" -else - IGNITE_HOME_TMP=${IGNITE_HOME} -fi - -# -# Set SCRIPTS_HOME - base path to scripts. -# -SCRIPTS_HOME="${IGNITE_HOME_TMP}/bin" - -source "${SCRIPTS_HOME}"/include/functions.sh - -# -# Discover IGNITE_HOME environment variable. -# -setIgniteHome - -# -# Set router service environment. -# -export DEFAULT_CONFIG=config/router/default-router.xml -export MAIN_CLASS=org.apache.ignite.internal.client.router.impl.GridRouterCommandLineStartup - -# -# Start router service. -# -. "${SCRIPTS_HOME}"/ignite.sh $@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Router command line loader. +# + +# +# Import common functions. +# +if [ "${IGNITE_HOME}" = "" ]; then + IGNITE_HOME_TMP="$(dirname "$(cd "$(dirname "$0")"; "pwd")")" +else + IGNITE_HOME_TMP=${IGNITE_HOME} +fi + +# +# Set SCRIPTS_HOME - base path to scripts. +# +SCRIPTS_HOME="${IGNITE_HOME_TMP}/bin" + +source "${SCRIPTS_HOME}"/include/functions.sh + +# +# Discover IGNITE_HOME environment variable. +# +setIgniteHome + +# +# Set router service environment. +# +export DEFAULT_CONFIG=config/router/default-router.xml +export MAIN_CLASS=org.apache.ignite.internal.client.router.impl.GridRouterCommandLineStartup + +# +# Start router service. +# +. "${SCRIPTS_HOME}"/ignite.sh $@ diff --git a/bin/index-reader.bat b/bin/index-reader.bat index c1b1dda2fcff1..d96263e1f40bc 100644 --- a/bin/index-reader.bat +++ b/bin/index-reader.bat @@ -115,7 +115,7 @@ if "%OS%" == "Windows_NT" set PROG_NAME=%~nx0% :: Set IGNITE_LIBS :: call "%SCRIPTS_HOME%\include\setenv.bat" -call "%SCRIPTS_HOME%\include\build-classpath.bat" + set CP=%IGNITE_LIBS% :: @@ -149,7 +149,7 @@ if %ERRORLEVEL% equ 0 ( :: Assertions are disabled by default since version 3.5. :: If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. :: -set ENABLE_ASSERTIONS=1 +set ENABLE_ASSERTIONS=0 :: :: Set '-ea' options if assertions are enabled. @@ -175,7 +175,7 @@ if defined JVM_OPTS ( ) "%JAVA_HOME%\bin\java.exe" %CONTROL_JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% ^ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ + -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^ -cp "%CP%" %MAIN_CLASS% %* set JAVA_ERRORLEVEL=%ERRORLEVEL% diff --git a/bin/index-reader.sh b/bin/index-reader.sh index 04908bd7d5d44..7514ae62977ea 100755 --- a/bin/index-reader.sh +++ b/bin/index-reader.sh @@ -59,7 +59,7 @@ setIgniteHome # Set IGNITE_LIBS. # . "${SCRIPTS_HOME}"/include/setenv.sh -. "${SCRIPTS_HOME}"/include/build-classpath.sh # Will be removed in the binary release. + CP="${IGNITE_LIBS}" # Mac OS specific support to display correct name in the dock. @@ -92,7 +92,7 @@ fi # Assertions are disabled by default since version 3.5. # If you want to enable them - set 'ENABLE_ASSERTIONS' flag to '1'. # -ENABLE_ASSERTIONS="1" +ENABLE_ASSERTIONS="0" # # Set '-ea' options if assertions are enabled. @@ -121,12 +121,12 @@ fi case $osname in Darwin*) "$JAVA" ${CONTROL_JVM_OPTS} ${QUIET:-} "${DOCK_OPTS}" \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} "$@" ;; *) "$JAVA" ${CONTROL_JVM_OPTS} ${QUIET:-} \ - -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="${IGNITE_HOME}" \ + -DIGNITE_HOME="${IGNITE_HOME}" \ -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS:-} -cp "${CP}" ${MAIN_CLASS} "$@" ;; esac diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/FileManagerInitializer.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/FileManagerInitializer.java index cbf7bf1f6e6a5..c10ecb3e390b3 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/FileManagerInitializer.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/FileManagerInitializer.java @@ -47,7 +47,7 @@ public void onStartup(ServletContext context) throws ServletException { } XmlWebApplicationContext appContext = new XmlWebApplicationContext(); - appContext.setConfigLocation("/WEB-INF/springmvc-servlet.xml"); + appContext.setConfigLocation("classpath:META-INF/springmvc-servlet.xml"); DispatcherServlet appServlet = new DispatcherServlet(appContext); diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/ConsoleController.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/ConsoleController.java index e894bbfc20ed1..91914e63790a9 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/ConsoleController.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/ConsoleController.java @@ -22,6 +22,7 @@ @RestController @RequestMapping("/console") +@CrossOrigin public class ConsoleController { @Autowired private S3Util s3Util; diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/FileManagerController.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/FileManagerController.java index 5532a48a655cc..1960d9b9cfe65 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/FileManagerController.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/FileManagerController.java @@ -64,6 +64,7 @@ */ @RestController @RequestMapping(value = "s3-rest") +@CrossOrigin public class FileManagerController { @Autowired diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/JSONObjectController.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/JSONObjectController.java index b50afb7d43128..21fcb36deacd1 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/JSONObjectController.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/JSONObjectController.java @@ -31,7 +31,8 @@ * for amis */ @RestController -@RequestMapping(value = "json") +@RequestMapping(value = "docs") +@CrossOrigin public class JSONObjectController { private static final String ACCEPT_JSON = "Accept=application/json"; @@ -56,13 +57,13 @@ public JSONObject list(@RequestParam(value="name",required=false) String name) { String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; SimpleDateFormat dt = new SimpleDateFormat(DATE_FORMAT); for (S3Object pathObj : list) { - String fname = pathObj.key(); - if(!fname.endsWith(".json")) { - continue; - } + String fname = pathObj.key(); if(name!=null && !name.isEmpty()) { - if(fname.indexOf(name)<0) { + if(name.charAt(0)=='/' && !fname.startsWith(name)) { + continue; + } + if(name.charAt(0)!='/' && fname.indexOf(name)<0) { continue; } } @@ -84,16 +85,62 @@ public JSONObject list(@RequestParam(value="name",required=false) String name) { return error(e.getMessage(),500); } } + /** - * 文件创建 + * 展示JSON对象列表 + */ + @RequestMapping(value="/{collection}",method=RequestMethod.GET, headers=ACCEPT_JSON) + public JSONObject all_collection_docs(@PathVariable("collection") String collection,@RequestParam(value="name",required=false) String name) { + JSONObject jsonObject = new JSONObject(); + try { + // 需要显示的目录路径 + // 返回的结果集 + List fileItems = new ArrayList<>(); + + List list = s3Util.getObjectList(bucketName, collection); + + String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + SimpleDateFormat dt = new SimpleDateFormat(DATE_FORMAT); + for (S3Object pathObj : list) { + String fname = pathObj.key(); + + if(name!=null && !name.isEmpty()) { + if(name.charAt(0)=='/' && !fname.startsWith(name)) { + continue; + } + if(name.charAt(0)!='/' && fname.indexOf(name)<0) { + continue; + } + } + + // 封装返回JSON数据 + JSONObject fileItem = new JSONObject(); + fileItem.put("name", fname); + fileItem.put("date", dt.format(new Date(pathObj.lastModified().toEpochMilli()))); + fileItem.put("size", pathObj.size()); + fileItem.put("etag", pathObj.eTag()); + fileItem.put("type", fname.endsWith("/")?"dir":"file"); + fileItems.add(fileItem); + } + + jsonObject.put("data", fileItems); + jsonObject.put("status",0); + return jsonObject; + } catch (Exception e) { + return error(e.getMessage(),500); + } + } + + /** + * 文档创建 */ - @RequestMapping(value="/{path}",method=RequestMethod.POST, headers=ACCEPT_JSON) - public JSONObject upload(@PathVariable("path") String destination,@RequestBody JSONObject json) { + @RequestMapping(value="/{collection}/{path}",method=RequestMethod.POST, headers=ACCEPT_JSON) + public JSONObject upload(@PathVariable("collection") String collection,@PathVariable("path") String destination,@RequestBody JSONObject json) { try { StringInputStream in = new StringInputStream(json.toJSONString()); - s3Util.upload(bucketName, destination, in); + s3Util.upload(bucketName, collection+"/"+destination, in); return success(destination); } catch (Exception e) { return error(e.getMessage(),500); @@ -102,17 +149,17 @@ public JSONObject upload(@PathVariable("path") String destination,@RequestBody J /** - * 文件下载/预览 + * 文档下载/预览 * @throws IOException */ - @RequestMapping(value="/{path}",method=RequestMethod.GET, headers=ACCEPT_JSON) - public void preview(HttpServletResponse response, @PathVariable("path") String path) throws IOException { + @RequestMapping(value="/{collection}/{path}",method=RequestMethod.GET, headers=ACCEPT_JSON) + public void preview(HttpServletResponse response,@PathVariable("collection") String collection, @PathVariable("path") String path) throws IOException { response.setContentType("application/json"); response.setHeader("Content-Disposition", "inline; filename=\"" + MimeUtility.encodeWord(FilenameUtils.getName(path)) + "\""); try ( - InputStream in = s3Util.getFileInputStream(bucketName, path); + InputStream in = s3Util.getFileInputStream(bucketName, collection+"/"+path); InputStream inputStream = new BufferedInputStream(in)) { FileCopyUtils.copy(inputStream, response.getOutputStream()); } @@ -125,16 +172,16 @@ public void preview(HttpServletResponse response, @PathVariable("path") String p /** * 文件下载/预览 */ - @RequestMapping(value="/{path}",method=RequestMethod.PUT, headers=ACCEPT_JSON) + @RequestMapping(value="/{collection}/{path}",method=RequestMethod.PUT, headers=ACCEPT_JSON) @ResponseBody - public JSONObject put(@PathVariable("path") String path,@RequestParam("key") String key, @RequestBody JSONObject updates) { + public JSONObject put(@PathVariable("collection") String collection,@PathVariable("path") String path,@RequestParam("key") String key, @RequestBody JSONObject updates) { if (key==null) { return error("Key Not Set",HttpServletResponse.SC_BAD_REQUEST); } try { - String jsonString = new String(s3Util.getFileByte(bucketName, path),"UTF-8"); + String jsonString = new String(s3Util.getFileByte(bucketName, collection+"/"+path),"UTF-8"); JSONObject json = JSONObject.parseObject(jsonString); json.put(key, updates); @@ -151,12 +198,12 @@ public JSONObject put(@PathVariable("path") String path,@RequestParam("key") Str /** * 删除文件或目录 */ - @RequestMapping(value="/{path}",method=RequestMethod.DELETE, headers=ACCEPT_JSON) + @RequestMapping(value="/{collection}/{path}",method=RequestMethod.DELETE, headers=ACCEPT_JSON) @ResponseBody - public JSONObject remove(@PathVariable("path") String path,@RequestBody JSONObject deletes) { + public JSONObject remove(@PathVariable("collection") String collection,@PathVariable("path") String path,@RequestBody JSONObject deletes) { try { - String jsonString = new String(s3Util.getFileByte(bucketName, path),"UTF-8"); + String jsonString = new String(s3Util.getFileByte(bucketName, collection+"/"+path),"UTF-8"); JSONObject json = JSONObject.parseObject(jsonString); for(String key: deletes.keySet()) { @@ -178,12 +225,31 @@ public JSONObject remove(@PathVariable("path") String path,@RequestBody JSONObje /** * 查看文件内容,针对html、txt等可编辑文件 */ - @RequestMapping("/getContent/{path}") - public JSONObject getContent(@PathVariable("path") String path) { - try { - String jsonString = new String(s3Util.getFileByte(bucketName, path),"UTF-8"); + @RequestMapping("/{collection}/{path}/meta") + public JSONObject getContent(@PathVariable("collection") String collection,@PathVariable("path") String path) { + try { + JSONObject jsonObject = new JSONObject(); + List list = s3Util.getObjectList(bucketName, collection+"/"+path); + String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + SimpleDateFormat dt = new SimpleDateFormat(DATE_FORMAT); + for (S3Object pathObj : list) { + String fname = pathObj.key(); + + // 封装返回JSON数据 + JSONObject fileItem = new JSONObject(); + fileItem.put("name", fname); + fileItem.put("date", dt.format(new Date(pathObj.lastModified().toEpochMilli()))); + fileItem.put("size", pathObj.size()); + fileItem.put("etag", pathObj.eTag()); + fileItem.put("type", fname.endsWith("/")?"dir":"file"); + jsonObject.put("data", fileItem); + jsonObject.put("status",0); + } + if(list.size()!=1) { + jsonObject.put("status",400); + } - return success(jsonString); + return jsonObject; } catch (Exception e) { return error(e.getMessage(),500); @@ -192,8 +258,8 @@ public JSONObject getContent(@PathVariable("path") String path) { - @RequestMapping(value="/{path}",method=RequestMethod.PATCH, headers=ACCEPT_JSON) - public JSONObject patch(@PathVariable("path") String path,@RequestBody JSONObject updates) { + @RequestMapping(value="/{collection}/{path}",method=RequestMethod.PATCH, headers=ACCEPT_JSON) + public JSONObject patch(@PathVariable("collection") String collection,@PathVariable("path") String path,@RequestBody JSONObject updates) { try { String jsonString = new String(s3Util.getFileByte(bucketName, path),"UTF-8"); diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/S3Controller.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/S3Controller.java index f69f4ebd650b9..af7b0b214dba3 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/S3Controller.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/controller/S3Controller.java @@ -11,6 +11,7 @@ import org.shaofan.s3.model.*; import org.shaofan.s3.service.S3Service; import org.shaofan.s3.util.*; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.MediaType; @@ -31,9 +32,14 @@ import java.util.List; import java.util.Map; +/** + * api参考地址 https://docs.aws.amazon.com/AmazonS3/latest/API/ + * @author admin + * + */ @RestController @RequestMapping("/s3") -//api参考地址https://docs.aws.amazon.com/AmazonS3/latest/API/ +@CrossOrigin public class S3Controller { @Autowired private S3Service s3Service; diff --git a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/util/DateUtil.java b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/util/DateUtil.java index 1ad04ee7ccab9..e0369411a52ec 100644 --- a/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/util/DateUtil.java +++ b/ignite-extensions/modules/filemanager/src/main/java/org/shaofan/s3/util/DateUtil.java @@ -3,14 +3,17 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Clock; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Date; +import software.amazon.awssdk.utils.DateUtils; public class DateUtil { + public static String getDateFormatToSecond(Date date) { DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -19,13 +22,11 @@ public static String getDateFormatToSecond(Date date) { } public static String getDateGMTFormat(Date date) { - //DateFormat df = new SimpleDateFormat("MM/dd/yyyy HH:mm+00:00"); - //String tag = df.format(date); + DateFormat df = new SimpleDateFormat("ddd, DD MMM YYYY HH:mm:ss ZZ"); + String tag = df.format(date); - //LocalDateTime t = LocalDateTime.ofEpochSecond(date.getTime()/1000, (int)(date.getTime()%1000*1000), ZoneOffset.ofHours(8)); - //ZonedDateTime zt = ZonedDateTime.of(t, ZoneId.systemDefault()); - //String tag = zt.format(DateTimeFormatter.ISO_INSTANT); - return date.toGMTString(); + String str = DateUtils.formatIso8601Date(Instant.ofEpochMilli(date.getTime())); + return str; } public static String getDateTagToSecond() { diff --git a/ignite-extensions/modules/filemanager/src/main/resources/META-INF/springmvc-servlet.xml b/ignite-extensions/modules/filemanager/src/main/resources/META-INF/springmvc-servlet.xml new file mode 100644 index 0000000000000..3d8ed6953799a --- /dev/null +++ b/ignite-extensions/modules/filemanager/src/main/resources/META-INF/springmvc-servlet.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ignite-extensions/modules/ignite-graph/pom.xml b/ignite-extensions/modules/ignite-graph/pom.xml index 48c64e4aba79b..998db70b3aabd 100644 --- a/ignite-extensions/modules/ignite-graph/pom.xml +++ b/ignite-extensions/modules/ignite-graph/pom.xml @@ -25,13 +25,13 @@ 2.11 2.16.999-SNAPSHOT - 2.12.7 + 2.14.0 5.1.1 3.14.4 2.11.12 2.4.7 - 3.6.4 - 3.6.4 + 3.7.0 + 3.7.0 diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteVertexProperty.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteVertexProperty.java index 5470426e30064..150986663bb4a 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteVertexProperty.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/IgniteVertexProperty.java @@ -24,10 +24,17 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; +import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; public class IgniteVertexProperty implements VertexProperty,java.io.Serializable { protected final IgniteVertex vertex; @@ -64,15 +71,31 @@ public V value() throws NoSuchElementException { public boolean isPresent() { return null != this.value; } + + @Override + public Object clone() { + final IgniteVertexProperty vp = new IgniteVertexProperty<>(vertex, key, value); + return vp; + } @Override - public Iterator> properties(final String... propertyKeys) { - throw new UnsupportedOperationException(); + public Iterator> properties(final String... propertyKeys) { + // add@byron + List> list = new ArrayList<>(propertyKeys.length); + for(String pkey: propertyKeys) { + String key = this.key+"."+pkey; + Object v = vertex.getProperty(key); + if(v!=null){ + list.add(new IgniteVertexProperty(vertex, key, v)); + } + } + return (Iterator)list.iterator(); + } @Override public Property property(final String key, final U value) { - throw new UnsupportedOperationException(); + return vertex.property(this.key+"."+key, value); } @Override @@ -82,7 +105,14 @@ public void remove() { @Override public Set keys() { - throw new UnsupportedOperationException(); + Iterable keys = vertex.getPropertyKeys(); + Set pKeys = new TreeSet<>(); + for(String key: keys) { + if(key.startsWith(this.key+".")){ + pKeys.add(key.substring(this.key.length()+1)); + } + } + return pKeys; } @Override diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputer.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputer.java index 115a9002f5dc7..b09f7f3fe0b40 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputer.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerGraphComputer.java @@ -34,6 +34,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedException; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; @@ -70,7 +71,7 @@ public final class TinkerGraphComputer implements GraphComputer { private VertexProgram vertexProgram; private final IgniteGraph graph; private TinkerMemory memory; - private final TinkerMessageBoard messageBoard = new TinkerMessageBoard<>(); + private final TinkerMessageBoard messageBoard = new TinkerMessageBoard(); private boolean executed = false; private final Set mapReducers = new HashSet<>(); @@ -129,6 +130,12 @@ public GraphComputer edges(final Traversal edgeFilter) { return this; } + @Override + public GraphComputer vertexProperties(Traversal> vertexPropertyFilter) { + this.graphFilter.setVertexPropertyFilter(vertexPropertyFilter); + return this; + } + @Override public Future submit() { // a graph computer can only be executed once @@ -311,5 +318,5 @@ public boolean supportsEdgePropertyRemoval() { return false; } }; - } + } } \ No newline at end of file diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerWorkerPool.java b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerWorkerPool.java index 4e905359bc677..8984939057ff3 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerWorkerPool.java +++ b/ignite-extensions/modules/ignite-graph/works-ignite-graph/src/main/java/de/kp/works/ignite/gremlin/process/computer/TinkerWorkerPool.java @@ -47,7 +47,6 @@ * @author Stephen Mallette (http://stephen.genoprime.com) */ public final class TinkerWorkerPool implements AutoCloseable { - private final int numberOfWorkers; private final ExecutorService workerPool; diff --git a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/pom.xml b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/pom.xml index 4ca17aa1596bc..97b7d8044037e 100644 --- a/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/pom.xml +++ b/ignite-extensions/modules/ignite-graph/works-ignite-gremlin-server/pom.xml @@ -25,6 +25,12 @@ gremlin-server ${tinkerpop.version} + + + org.apache.tinkerpop + gremlin-console + ${tinkerpop.version} + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java index 24ac8dc8675ca..2bcba17c81fec 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java @@ -1482,22 +1482,7 @@ private static class TxProxy implements Transaction { /** {@inheritDoc} */ @Override public void resume() throws IgniteException { throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override public IgniteAsyncSupport withAsync() { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override public boolean isAsync() { - return false; - } - - /** {@inheritDoc} */ - @Override public IgniteFuture future() { - throw new UnsupportedOperationException(); - } + } } /** Enumeration that represents possible operations on the underlying store. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionEventProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionEventProxyImpl.java index f6bc6688f3d48..70a1c5fa866c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionEventProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionEventProxyImpl.java @@ -175,22 +175,7 @@ public TransactionEventProxyImpl(GridNearTxLocal tx) { /** {@inheritDoc} */ @Nullable @Override public String label() { return tx().label(); - } - - /** {@inheritDoc} */ - @Override public IgniteAsyncSupport withAsync() { - throw new UnsupportedOperationException("Operation deprecated."); - } - - /** {@inheritDoc} */ - @Override public boolean isAsync() { - throw new UnsupportedOperationException("Operation deprecated."); - } - - /** {@inheritDoc} */ - @Override public IgniteFuture future() { - throw new UnsupportedOperationException("Operation deprecated."); - } + } /** * @return local transaction diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java index 3cc390e147737..d7c0e70902ef4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java @@ -38,7 +38,6 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteAsyncSupport; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.transactions.Transaction; @@ -261,22 +260,6 @@ private void leave() { return tx.timeout(timeout); } - /** {@inheritDoc} */ - @Override public IgniteAsyncSupport withAsync() { - return new TransactionProxyImpl<>(tx, cctx, true); - } - - /** {@inheritDoc} */ - @Override public boolean isAsync() { - return async; - } - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override public IgniteFuture future() { - return asyncRes; - } - /** {@inheritDoc} */ @Override public boolean setRollbackOnly() { enter(); diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java index b2dd692f07d5d..9b63dfdf84c43 100644 --- a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java +++ b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java @@ -117,7 +117,7 @@ * } * */ -public interface Transaction extends AutoCloseable, IgniteAsyncSupport { +public interface Transaction extends AutoCloseable { /** * Gets unique identifier for this transaction. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 499c48b8a1e25..b79646fe36b6b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -1917,52 +1917,7 @@ private void checkPut(boolean inTx) throws Exception { assert cache.get("key2") == 2; assert cache.get("wrong") == null; } - - /** - * @throws Exception If failed. - */ - @Test - public void testPutAsyncOld() throws Exception { - Transaction tx = txShouldBeUsed() ? transactions().txStart() : null; - - IgniteCache cacheAsync = jcache().withAsync(); - - try { - jcache().put("key2", 1); - - cacheAsync.put("key1", 10); - - IgniteFuture fut1 = cacheAsync.future(); - - cacheAsync.put("key2", 11); - - IgniteFuture fut2 = cacheAsync.future(); - - IgniteFuture f = null; - - if (tx != null) { - tx = (Transaction)tx.withAsync(); - - tx.commit(); - - f = tx.future(); - } - - assertNull(fut1.get()); - assertNull(fut2.get()); - - assert f == null || f.get().state() == COMMITTED; - } - finally { - if (tx != null) - tx.close(); - } - - checkSize(F.asSet("key1", "key2")); - - assert jcache().get("key1") == 10; - assert jcache().get("key2") == 11; - } + /** * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java index dc9efb9f7dd45..edb0be1ee65f7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java @@ -1853,52 +1853,6 @@ private void checkPut(boolean inTx) throws Exception { assert cache.get("wrong") == null; } - /** - * @throws Exception If failed. - */ - @Test - public void testPutAsyncOld() throws Exception { - Transaction tx = txShouldBeUsed() ? transactions().txStart() : null; - - IgniteCache cacheAsync = jcache().withAsync(); - - try { - jcache().put("key2", 1); - - cacheAsync.put("key1", 10); - - IgniteFuture fut1 = cacheAsync.future(); - - cacheAsync.put("key2", 11); - - IgniteFuture fut2 = cacheAsync.future(); - - IgniteFuture f = null; - - if (tx != null) { - tx = (Transaction)tx.withAsync(); - - tx.commit(); - - f = tx.future(); - } - - assertNull(fut1.get()); - assertNull(fut2.get()); - - assert f == null || f.get().state() == COMMITTED; - } - finally { - if (tx != null) - tx.close(); - } - - checkSize(F.asSet("key1", "key2")); - - assert (Integer)jcache().get("key1") == 10; - assert (Integer)jcache().get("key2") == 11; - } - /** * @throws Exception If failed. */ diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java index c9dafe95dd640..7652fa300d504 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java @@ -35,7 +35,7 @@ import org.apache.ignite.internal.util.typedef.CI2; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteAsyncSupport; + import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.testframework.GridTestUtils; @@ -564,21 +564,6 @@ public static class DummyTx extends GridMetadataAwareAdapter implements Transact // No-op. } - /** {@inheritDoc} */ - @Override public IgniteAsyncSupport withAsync() { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override public boolean isAsync() { - return false; - } - - /** {@inheritDoc} */ - @Override public IgniteFuture future() { - return null; - } - /** {@inheritDoc} */ @Override public void rollback() { // No-op. diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java index c36dd341c81f4..dd53667f6decf 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java @@ -216,7 +216,7 @@ private IgniteBiTuple findClosestCentroid(Vector[] centers, Lab */ private Vector[] initClusterCentersRandomly(Dataset> dataset, int k) { - Vector[] initCenters = new DenseVector[k]; + Vector[] initCenters = new Vector[k]; // Gets k or less vectors from each partition. List rndPnts = dataset.compute(data -> { diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNModel.java index 099ff826b8ccb..dfabfd8f62973 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNModel.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNModel.java @@ -79,9 +79,14 @@ protected KNNModel(Dataset> dataset, DistanceMeasu (Collection> a, Collection> b) -> { if(a==null || a.isEmpty()) return b; if(b==null || b.isEmpty()) return a; - Queue> heap = new PriorityQueue<>(a); - //-tryToAddIntoHeap(heap, k, pnt, a, distanceMeasure); - tryToAddIntoHeap(heap, k, pnt, b, distanceMeasure); + Queue> heap = null; + if(a instanceof PriorityQueue) { + heap = (PriorityQueue>)a; + } + else { + heap = new PriorityQueue>(a); + } + tryToAddIntoHeap(heap, k, pnt, b); return heap; }); return res; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/PointWithDistanceUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/PointWithDistanceUtil.java index cd418f8524e1f..8286457b80167 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/PointWithDistanceUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/PointWithDistanceUtil.java @@ -88,7 +88,7 @@ public static List> transformToListOrdered(Collection void tryToAddIntoHeap(Queue> heap, int k, LabeledVector dataPnt, double distance) { if (dataPnt != null) { - if (heap.size() == k && heap.peek().getDistance() > distance) + if (heap.size() >= k && heap.peek().getDistance() > distance) heap.remove(); if (heap.size() < k) @@ -110,7 +110,7 @@ public static void tryToAddIntoHeap(Queue> heap, int k, public static void tryToAddIntoHeapUseSimilarity(Queue> heap, int k, LabeledVector dataPnt, double sim) { if (dataPnt != null) { - if (heap.size() == k && heap.peek().getDistance() < sim) + if (heap.size() >= k && heap.peek().getDistance() < sim) heap.remove(); if (heap.size() < k) @@ -139,12 +139,12 @@ public static void tryToAddIntoHeap(Queue> heap, int k, } public static void tryToAddIntoHeap(Queue> heap, int k, Vector pnt, - Collection> dataPnts, DistanceMeasure distanceMeasure) { + Collection> dataPnts) { if (dataPnts != null) { for (PointWithDistance dataPnt : dataPnts) { if (dataPnt != null) { double distance = dataPnt.getDistance(); - if (heap.size() == k && heap.peek().getDistance() > distance) + if (heap.size() >= k && heap.peek().getDistance() > distance) heap.remove(); if (heap.size() < k) diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/BallTreeSpatialIndex.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/BallTreeSpatialIndex.java index 5280a6ea4bcb8..619e61fcd6610 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/BallTreeSpatialIndex.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/BallTreeSpatialIndex.java @@ -17,7 +17,6 @@ package org.apache.ignite.ml.knn.utils.indices; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -27,7 +26,6 @@ import org.apache.ignite.ml.knn.utils.PointWithDistance; import org.apache.ignite.ml.math.distances.DistanceMeasure; import org.apache.ignite.ml.math.primitives.vector.Vector; -import org.apache.ignite.ml.math.primitives.vector.VectorUtils; import org.apache.ignite.ml.structures.LabeledVector; import static org.apache.ignite.ml.knn.utils.PointWithDistanceUtil.transformToListOrdered; @@ -40,356 +38,159 @@ * @param Label type. */ public class BallTreeSpatialIndex implements SpatialIndex { - /** Number of points in a leaf. */ - private static final int MAX_LEAF_SIZE = 42; - - /** Margin used to identify center of Balls during data points split. */ - private static final double SPLIT_BALL_MARGIN = 0.2; - /** Distance measure. */ private final DistanceMeasure distanceMeasure; - /** Root node of Ball tree. */ - private final TreeNode root; + /** Root node of the KD tree. */ + private TreeNode root; /** - * Constructs a new instance of Ball tree spatial index. + * Constructs a new instance of KD tree spatial index. To construct KD tree a "randomized" approach is uses, all + * nodes are inserted into the tree sequentially without any additional computations and re-balancing. * - * @param data Data. + * @param data Data points. * @param distanceMeasure Distance measure. */ public BallTreeSpatialIndex(List> data, DistanceMeasure distanceMeasure) { this.distanceMeasure = distanceMeasure; - root = buildTree(data); + + data.forEach(dataPnt -> root = add(root, dataPnt)); } /** {@inheritDoc} */ @Override public Collection> findKClosest(int k, Vector pnt) { - Queue> heap = new PriorityQueue<>(k,Comparator.reverseOrder()); + if (k <= 0) + throw new IllegalArgumentException("Number of neighbours should be positive."); - root.findKClosest(pnt, heap, k); + Queue> heap = new PriorityQueue<>(k,Comparator.reverseOrder()); - return heap; - } + findKClosest(pnt, root, 0, heap, k); - /** - * Builds Ball tree. - * - * @param data Data points. - * @return Ball tree root node. - */ - private TreeNode buildTree(List> data) { - Vector center = calculateCenter(data); - - return buildTree(data, center, calculateRadius(data, center)); + return heap; } /** - * Builds Ball tree using specified {@code center} and {@code radius} as parameters of current tree node. + * Updates collection of closest points processing specified KD tree node. * - * @param data Data points. - * @param center Center of the current tree node. - * @param radius Radius of the current tree node. - * @return Ball tree node. + * @param pnt Point to calculate distance to. + * @param node KD tree node. + * @param splitDim Split dimension that corresponds to current KD tree level. + * @param heap Heap with closest points. + * @param k Number of closest points to be collected. */ - private TreeNode buildTree(List> data, Vector center, double radius) { - if (data.size() <= MAX_LEAF_SIZE) - return new TreeLeafNode(center, radius, data); - - Vector leftCenter = calculateCenter(data); - Vector rightCenter = leftCenter.copy(); - - int bestDimForSplit = calculateBestDimForSplit(data); - double min = calculateMin(data, bestDimForSplit); - double max = calculateMax(data, bestDimForSplit); - - leftCenter.set(bestDimForSplit, min + (max - min) * SPLIT_BALL_MARGIN); - rightCenter.set(bestDimForSplit, min + (max - min) * (1 - SPLIT_BALL_MARGIN)); - - List> leftBallPnts = new ArrayList<>(); - List> rightBallPnts = new ArrayList<>(); - - splitPoints(data, leftCenter, rightCenter, leftBallPnts, rightBallPnts); - - data.clear(); // Help GC to collect unused list. - - return new TreeInnerNode( - center, - radius, - buildTree(leftBallPnts, leftCenter, calculateRadius(leftBallPnts, leftCenter)), - buildTree(rightBallPnts, rightCenter, calculateRadius(rightBallPnts, rightCenter)) + private void findKClosest(Vector pnt, TreeNode node, int splitDim, Queue> heap, int k) { + if (node == null) + return; + + tryToAddIntoHeap(heap, k, node.val, distanceMeasure.compute(pnt, node.val.features())); + + double pntPrj = pnt.get(splitDim); + double splitPrj = node.val.get(splitDim); + + TreeNode primaryBranch = pntPrj > splitPrj ? node.right : node.left; + TreeNode secondaryBranch = primaryBranch == node.right ? node.left : node.right; + + findKClosestInSplittedSpace( + pnt, + primaryBranch, + secondaryBranch, + (splitDim + 1) % pnt.size(), + Math.abs(pntPrj - splitPrj), + heap, + k ); } /** - * Splits list of data points on two parts: {@code leftBallPnts} and {@code rightBallPnts} so that all points in - * {@code leftBallPnts} are closer to left center than to right center and all points in {@code rightBallPnts} are - * closer to right center than to left center. - * - * @param dataPnts Data points. - * @param leftCenter Left center. - * @param rightCenter Right center. - * @param leftBallPnts Left ball points (out parameter). - * @param rightBallPnts Right ball points (out parameter). - */ - private void splitPoints(List> dataPnts, Vector leftCenter, Vector rightCenter, - List> leftBallPnts, List> rightBallPnts) { - for (LabeledVector dataPnt : dataPnts) { - double distToLeftCenter = distanceMeasure.compute(leftCenter, dataPnt.features()); - double distToRightCenter = distanceMeasure.compute(rightCenter, dataPnt.features()); - - List> targetBallPnts = distToLeftCenter < distToRightCenter ? leftBallPnts : rightBallPnts; - targetBallPnts.add(dataPnt); - } - } - - /** - * Calculates radius of a ball (max distance from center to data point). + * Updates collection of closest points looking into primary branch and if distance to plane is less then distance + * to the most distant point within closest point looks into secondary branch as well. * - * @param data Data points. - * @param center Center of a ball. - * @return Radius of a ball. + * @param pnt Point to calculate distance to. + * @param primaryBrach Primary branch ({@code pnt} belongs to this subtree). + * @param secondaryBranch Secondary branch ({@code pnt} doesn't belong to this subtree). + * @param splitDim Split dimension that corresponds to current KD tree level. + * @param distToPlane Distance to split plane. + * @param heap Heap with closest points. + * @param k Number of closest points to be collected. */ - private double calculateRadius(List> data, Vector center) { - double radius = 0; + private void findKClosestInSplittedSpace(Vector pnt, TreeNode primaryBrach, TreeNode secondaryBranch, int splitDim, + double distToPlane, Queue> heap, int k) { - for (LabeledVector dataPnt : data) { - double distance = distanceMeasure.compute(center, dataPnt.features()); - radius = Math.max(radius, distance); - } + findKClosest(pnt, primaryBrach, splitDim, heap, k); - return radius; + // If the distance to the most distant element in the heap is less than distance to the plane we need to process + // the secondary branch as well. + if (heap.size() < k || distToPlane < heap.peek().getDistance()) + findKClosest(pnt, secondaryBranch, splitDim, heap, k); } /** - * Calculates center of the group of data points using mean values across all dimensions. + * Adds element into an existing or not existing KDTree. * - * @param data Data points. - * @return Center of the group of points. + * @param root Root node of KDTree or {@code null}. + * @param val Value to be added. + * @return Root node of KDTree. */ - private Vector calculateCenter(List> data) { - if (data.isEmpty()) - return null; + private TreeNode add(TreeNode root, LabeledVector val) { + if (root == null) + return new TreeNode(val); - double[] center = new double[data.get(0).size()]; - for (int dim = 0; dim < center.length; dim++) - center[dim] = calculateMean(data, dim); + addIntoExistingTree(root, val); - return VectorUtils.of(center); + return root; } /** - * Calculates best dimension for split space on two balls. + * Adds element into an existing KD tree. * - * @param data Data points. - * @return Dimension. + * @param node Root node of KD tree. + * @param pnt Point to be added. */ - private int calculateBestDimForSplit(List> data) { - if (data.isEmpty()) - return -1; + private void addIntoExistingTree(TreeNode node, LabeledVector pnt) { + int splitDim = 0; - double bestStd = 0; - int bestDim = -1; + while (true) { + if (pnt.get(splitDim) > node.val.get(splitDim)) { + if (node.right == null) { + node.right = new TreeNode(pnt); + break; + } - for (int dim = 0; dim < data.get(0).size(); dim++) { - double std = calculateStd(data, dim); - if (std > bestStd) { - bestStd = std; - bestDim = dim; + node = node.right; } - } - - return bestDim; - } - - /** - * Calculates max value for the list of data points and specified dimension. - * - * @param data Data points. - * @param dim Dimension. - * @return Max value. - */ - private double calculateMax(List> data, int dim) { - double max = Double.NEGATIVE_INFINITY; - - for (LabeledVector dataPnt : data) - max = Math.max(max, dataPnt.get(dim)); - - return max; - } - - /** - * Calculates min value for the list of data points and specified dimension. - * - * @param data Data points. - * @param dim Dimension. - * @return Min value. - */ - private double calculateMin(List> data, int dim) { - double min = Double.POSITIVE_INFINITY; - - for (LabeledVector dataPnt : data) - min = Math.min(min, dataPnt.get(dim)); - - return min; - } - - /** - * Calculates standard deviation for the list of data points and specified dimension. - * - * @param data Data points. - * @param dim Dimension. - * @return Standard deviation. - */ - private double calculateStd(List> data, int dim) { - double res = 0; - - double mean = calculateMean(data, dim); - for (LabeledVector dataPnt : data) - res += Math.pow(dataPnt.get(dim) - mean, 2); - - return Math.sqrt(res / data.size()); - } - - /** - * Calculates mean value for the list of data points and specified dimension. - * - * @param data Data points. - * @param dim Dimension. - * @return Mean value. - */ - private double calculateMean(List> data, int dim) { - double res = 0; - - for (LabeledVector dataPnt : data) - res += dataPnt.get(dim); - - return res / data.size(); - } - - /** - * Ball tree node. - */ - private abstract class TreeNode { - /** Center of the ball. */ - private final Vector center; - - /** Radius of the ball. */ - private final double radius; - - /** - * Constructs a new instance of Ball tree node. - * - * @param center Center of the ball. - * @param radius Radius of the ball. - */ - TreeNode(Vector center, double radius) { - this.center = center; - this.radius = radius; - } - - /** - * Finds {@code k} closest elements the the specified point and adds them into {@code heap}. - * - * @param pnt Point to be used to calculate distance to other points. - * @param heap Heap with closest points. - * @param k Number of closest points to be collected. - */ - abstract void findKClosest(Vector pnt, Queue> heap, int k); + else { + if (node.left == null) { + node.left = new TreeNode(pnt); + break; + } - /** */ - public Vector getCenter() { - return center; - } + node = node.left; + } - /** */ - public double getRadius() { - return radius; + splitDim = (splitDim + 1) % pnt.size(); } } /** - * Inner node of Ball tree that contains two children nodes. + * Binary tree node with {@code val}, {@code left} and {@code right} children. */ - private final class TreeInnerNode extends TreeNode { - /** Left child node. */ - private final TreeNode left; - - /** Right child node. */ - private final TreeNode right; - - /** - * Constructs a new instance of Ball tree inner node. - * - * @param center Center of the ball. - * @param radius Radius of the ball. - */ - TreeInnerNode(Vector center, double radius, TreeNode left, TreeNode right) { - super(center, radius); - this.left = left; - this.right = right; - } - - /** {@inheritDoc} */ - @Override void findKClosest(Vector pnt, Queue> heap, int k) { - double distToLeftCenter = computeDistToCenter(pnt, left); - double distToRightCenter = computeDistToCenter(pnt, right); - - TreeNode primaryBranch = distToLeftCenter > distToRightCenter ? right : left; - TreeNode secondaryBranch = primaryBranch == right ? left : right; - - if (primaryBranch != null) - primaryBranch.findKClosest(pnt, heap, k); - - // If the distance to the most distant element in the heap is less than distance to the plane we need to process - // the secondary branch as well. - if (secondaryBranch != null) { - double distToSecondaryBall = computeDistToCenter(pnt, secondaryBranch) - secondaryBranch.getRadius(); - if (heap.size() < k || distToSecondaryBall < heap.peek().getDistance()) - secondaryBranch.findKClosest(pnt, heap, k); - } - } + private final class TreeNode { + /** Value. */ + private final LabeledVector val; - /** - * Computed distance from point to center of Ball tree node. - * - * @param pnt Point to be used to calculate distance to other points. - * @param node Ball tree node. - * @return Distance from point to center of Ball tree node. - */ - private double computeDistToCenter(Vector pnt, TreeNode node) { - if (node == null) - return Double.MAX_VALUE; + /** Left child. */ + private TreeNode left; - return distanceMeasure.compute(pnt, node.getCenter()); - } - } - - /** - * Leaf node of Ball tree that contains an array of points that owned by the leaf. - */ - private final class TreeLeafNode extends TreeNode { - /** Array of points owned by the leaf. */ - private final List> points; + /** Right child. */ + private TreeNode right; /** - * Constructs a new instance of Ball tree leaf node. + * Constructs a new instance of binary tree node. * - * @param center Center of the ball. - * @param radius Radius of the ball. - * @param points List of points owned by the leaf. + * @param val value. */ - TreeLeafNode(Vector center, double radius, List> points) { - super(center, radius); - this.points = points; - } - - /** {@inheritDoc} */ - @Override void findKClosest(Vector pnt, Queue> heap, int k) { - for (LabeledVector dataPnt : points) { - double distance = distanceMeasure.compute(pnt, dataPnt.features()); - tryToAddIntoHeap(heap, k, dataPnt, distance); - } + TreeNode(LabeledVector val) { + this.val = val; } } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/KDTreeSpatialIndex.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/KDTreeSpatialIndex.java index 74b140f0d98f7..8ad8d28ff284b 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/KDTreeSpatialIndex.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/utils/indices/KDTreeSpatialIndex.java @@ -17,6 +17,8 @@ package org.apache.ignite.ml.knn.utils.indices; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -26,171 +28,394 @@ import org.apache.ignite.ml.knn.utils.PointWithDistance; import org.apache.ignite.ml.math.distances.DistanceMeasure; import org.apache.ignite.ml.math.primitives.vector.Vector; +import org.apache.ignite.ml.math.primitives.vector.VectorUtils; import org.apache.ignite.ml.structures.LabeledVector; import static org.apache.ignite.ml.knn.utils.PointWithDistanceUtil.transformToListOrdered; import static org.apache.ignite.ml.knn.utils.PointWithDistanceUtil.tryToAddIntoHeap; /** - * KD tree based implementation of {@link SpatialIndex}. Asymptotic runtime complexity of finding {@code k} closest + * KDBall tree based implementation of {@link SpatialIndex}. Asymptotic runtime complexity of finding {@code k} closest * elements is {@code O(log(n)*k)}, but it degrades on high dimensional data. * * @param Label type. */ public class KDTreeSpatialIndex implements SpatialIndex { + /** Number of points in a leaf. */ + private static final int MAX_LEAF_SIZE = 42; + + /** Margin used to identify center of Balls during data points split. */ + private static final double SPLIT_BALL_MARGIN = 0.2; + /** Distance measure. */ private final DistanceMeasure distanceMeasure; - /** Root node of the KD tree. */ - private TreeNode root; + /** Root node of Ball tree. */ + private final TreeNode root; /** - * Constructs a new instance of KD tree spatial index. To construct KD tree a "randomized" approach is uses, all - * nodes are inserted into the tree sequentially without any additional computations and re-balancing. + * Constructs a new instance of Ball tree spatial index. * - * @param data Data points. + * @param data Data. * @param distanceMeasure Distance measure. */ public KDTreeSpatialIndex(List> data, DistanceMeasure distanceMeasure) { this.distanceMeasure = distanceMeasure; - - data.forEach(dataPnt -> root = add(root, dataPnt)); + root = buildTree(data); } /** {@inheritDoc} */ @Override public Collection> findKClosest(int k, Vector pnt) { - if (k <= 0) - throw new IllegalArgumentException("Number of neighbours should be positive."); + Queue> heap = new PriorityQueue<>(k,Comparator.reverseOrder()); - Queue> heap = new PriorityQueue<>(k,Comparator.reverseOrder()); - - findKClosest(pnt, root, 0, heap, k); + root.findKClosest(pnt, heap, k); return heap; } /** - * Updates collection of closest points processing specified KD tree node. + * Builds Ball tree. + * + * @param data Data points. + * @return Ball tree root node. + */ + private TreeNode buildTree(List> data) { + Vector center = calculateCenter(data); + + return buildTree(data, center, calculateRadius(data, center)); + } + + /** + * Builds Ball tree using specified {@code center} and {@code radius} as parameters of current tree node. * - * @param pnt Point to calculate distance to. - * @param node KD tree node. - * @param splitDim Split dimension that corresponds to current KD tree level. - * @param heap Heap with closest points. - * @param k Number of closest points to be collected. + * @param data Data points. + * @param center Center of the current tree node. + * @param radius Radius of the current tree node. + * @return Ball tree node. */ - private void findKClosest(Vector pnt, TreeNode node, int splitDim, Queue> heap, int k) { - if (node == null) - return; - - tryToAddIntoHeap(heap, k, node.val, distanceMeasure.compute(pnt, node.val.features())); - - double pntPrj = pnt.get(splitDim); - double splitPrj = node.val.get(splitDim); - - TreeNode primaryBranch = pntPrj > splitPrj ? node.right : node.left; - TreeNode secondaryBranch = primaryBranch == node.right ? node.left : node.right; - - findKClosestInSplittedSpace( - pnt, - primaryBranch, - secondaryBranch, - (splitDim + 1) % pnt.size(), - Math.abs(pntPrj - splitPrj), - heap, - k + private TreeNode buildTree(List> data, Vector center, double radius) { + if (data.size() <= MAX_LEAF_SIZE) + return new TreeLeafNode(center, radius, data); + + Vector leftCenter = calculateCenter(data); + Vector rightCenter = leftCenter.copy(); + + int bestDimForSplit = calculateBestDimForSplit(data); + double[] seq = calculateMid(data, bestDimForSplit); + double mid = seq[(seq.length+1)/2]; + double mid1 = seq[(int)(seq.length* SPLIT_BALL_MARGIN)]; + double mid2 = seq[(int)(seq.length* (1 - SPLIT_BALL_MARGIN))]; + + leftCenter.set(bestDimForSplit, mid1); + rightCenter.set(bestDimForSplit, mid2); + + List> leftBallPnts = new ArrayList<>(); + List> rightBallPnts = new ArrayList<>(); + + splitPoints(data, leftCenter, rightCenter, leftBallPnts, rightBallPnts); + + // add@byron + if(rightBallPnts.size()==0 || leftBallPnts.size()==0) { + return new TreeLeafNode(center, radius, data); + } + + data.clear(); // Help GC to collect unused list. + + return new TreeInnerNode( + center, + radius, + buildTree(leftBallPnts, leftCenter, calculateRadius(leftBallPnts, leftCenter)), + buildTree(rightBallPnts, rightCenter, calculateRadius(rightBallPnts, rightCenter)) ); } /** - * Updates collection of closest points looking into primary branch and if distance to plane is less then distance - * to the most distant point within closest point looks into secondary branch as well. + * Splits list of data points on two parts: {@code leftBallPnts} and {@code rightBallPnts} so that all points in + * {@code leftBallPnts} are closer to left center than to right center and all points in {@code rightBallPnts} are + * closer to right center than to left center. + * + * @param dataPnts Data points. + * @param leftCenter Left center. + * @param rightCenter Right center. + * @param leftBallPnts Left ball points (out parameter). + * @param rightBallPnts Right ball points (out parameter). + */ + private void splitPoints(List> dataPnts, Vector leftCenter, Vector rightCenter, + List> leftBallPnts, List> rightBallPnts) { + for (LabeledVector dataPnt : dataPnts) { + double distToLeftCenter = distanceMeasure.compute(leftCenter, dataPnt.features()); + double distToRightCenter = distanceMeasure.compute(rightCenter, dataPnt.features()); + + List> targetBallPnts = distToLeftCenter < distToRightCenter ? leftBallPnts : rightBallPnts; + targetBallPnts.add(dataPnt); + } + } + + /** + * Calculates radius of a ball (max distance from center to data point). * - * @param pnt Point to calculate distance to. - * @param primaryBrach Primary branch ({@code pnt} belongs to this subtree). - * @param secondaryBranch Secondary branch ({@code pnt} doesn't belong to this subtree). - * @param splitDim Split dimension that corresponds to current KD tree level. - * @param distToPlane Distance to split plane. - * @param heap Heap with closest points. - * @param k Number of closest points to be collected. + * @param data Data points. + * @param center Center of a ball. + * @return Radius of a ball. */ - private void findKClosestInSplittedSpace(Vector pnt, TreeNode primaryBrach, TreeNode secondaryBranch, int splitDim, - double distToPlane, Queue> heap, int k) { + private double calculateRadius(List> data, Vector center) { + double radius = 0; - findKClosest(pnt, primaryBrach, splitDim, heap, k); + for (LabeledVector dataPnt : data) { + double distance = distanceMeasure.compute(center, dataPnt.features()); + radius = Math.max(radius, distance); + } - // If the distance to the most distant element in the heap is less than distance to the plane we need to process - // the secondary branch as well. - if (heap.size() < k || distToPlane < heap.peek().getDistance()) - findKClosest(pnt, secondaryBranch, splitDim, heap, k); + return radius; } /** - * Adds element into an existing or not existing KDTree. + * Calculates center of the group of data points using mean values across all dimensions. * - * @param root Root node of KDTree or {@code null}. - * @param val Value to be added. - * @return Root node of KDTree. + * @param data Data points. + * @return Center of the group of points. */ - private TreeNode add(TreeNode root, LabeledVector val) { - if (root == null) - return new TreeNode(val); + private Vector calculateCenter(List> data) { + if (data.isEmpty()) + return null; - addIntoExistingTree(root, val); + double[] center = new double[data.get(0).size()]; + for (int dim = 0; dim < center.length; dim++) + center[dim] = calculateMean(data, dim); - return root; + return VectorUtils.of(center); } /** - * Adds element into an existing KD tree. + * Calculates best dimension for split space on two balls. * - * @param node Root node of KD tree. - * @param pnt Point to be added. + * @param data Data points. + * @return Dimension. */ - private void addIntoExistingTree(TreeNode node, LabeledVector pnt) { - int splitDim = 0; + private int calculateBestDimForSplit(List> data) { + if (data.isEmpty()) + return -1; - while (true) { - if (pnt.get(splitDim) > node.val.get(splitDim)) { - if (node.right == null) { - node.right = new TreeNode(pnt); - break; - } + double bestStd = 0; + int bestDim = -1; - node = node.right; + for (int dim = 0; dim < data.get(0).size(); dim++) { + double std = calculateStd(data, dim); + if (std > bestStd) { + bestStd = std; + bestDim = dim; } - else { - if (node.left == null) { - node.left = new TreeNode(pnt); - break; - } + } - node = node.left; - } + return bestDim; + } + + /** + * Calculates max value for the list of data points and specified dimension. + * + * @param data Data points. + * @param dim Dimension. + * @return Max value. + */ + private double calculateMax(List> data, int dim) { + double max = Double.NEGATIVE_INFINITY; + + for (LabeledVector dataPnt : data) + max = Math.max(max, dataPnt.get(dim)); + + return max; + } + + /** + * Calculates min value for the list of data points and specified dimension. + * + * @param data Data points. + * @param dim Dimension. + * @return Min value. + */ + private double calculateMin(List> data, int dim) { + double min = Double.POSITIVE_INFINITY; + + for (LabeledVector dataPnt : data) + min = Math.min(min, dataPnt.get(dim)); + + return min; + } + + /** + * Calculates standard deviation for the list of data points and specified dimension. + * + * @param data Data points. + * @param dim Dimension. + * @return Standard deviation. + */ + private double calculateStd(List> data, int dim) { + double res = 0; - splitDim = (splitDim + 1) % pnt.size(); + double mean = calculateMean(data, dim); + for (LabeledVector dataPnt : data) + res += Math.pow(dataPnt.get(dim) - mean, 2); + + return Math.sqrt(res / data.size()); + } + + /** + * Calculates mean value for the list of data points and specified dimension. + * + * @param data Data points. + * @param dim Dimension. + * @return Mean value. + */ + private double calculateMean(List> data, int dim) { + double res = 0; + + for (LabeledVector dataPnt : data) + res += dataPnt.get(dim); + + return res / data.size(); + } + + /** + * Calculates mean value for the list of data points and specified dimension. + * + * @param data Data points. + * @param dim Dimension. + * @return Mean value. + */ + private double[] calculateMid(List> data, int dim) { + double[] res = new double[data.size()]; + int i = 0; + for (LabeledVector dataPnt : data) { + res[i] = dataPnt.get(dim); + i++; } + Arrays.sort(res); + return res; } /** - * Binary tree node with {@code val}, {@code left} and {@code right} children. + * Ball tree node. */ - private final class TreeNode { - /** Value. */ - private final LabeledVector val; + private abstract class TreeNode { + /** Center of the ball. */ + private final Vector center; - /** Left child. */ - private TreeNode left; + /** Radius of the ball. */ + private final double radius; - /** Right child. */ - private TreeNode right; + /** + * Constructs a new instance of Ball tree node. + * + * @param center Center of the ball. + * @param radius Radius of the ball. + */ + TreeNode(Vector center, double radius) { + this.center = center; + this.radius = radius; + } /** - * Constructs a new instance of binary tree node. + * Finds {@code k} closest elements the the specified point and adds them into {@code heap}. * - * @param val value. + * @param pnt Point to be used to calculate distance to other points. + * @param heap Heap with closest points. + * @param k Number of closest points to be collected. */ - TreeNode(LabeledVector val) { - this.val = val; + abstract void findKClosest(Vector pnt, Queue> heap, int k); + + /** */ + public Vector getCenter() { + return center; + } + + /** */ + public double getRadius() { + return radius; + } + } + + /** + * Inner node of Ball tree that contains two children nodes. + */ + private final class TreeInnerNode extends TreeNode { + /** Left child node. */ + private final TreeNode left; + + /** Right child node. */ + private final TreeNode right; + + /** + * Constructs a new instance of Ball tree inner node. + * + * @param center Center of the ball. + * @param radius Radius of the ball. + */ + TreeInnerNode(Vector center, double radius, TreeNode left, TreeNode right) { + super(center, radius); + this.left = left; + this.right = right; + } + + /** {@inheritDoc} */ + @Override void findKClosest(Vector pnt, Queue> heap, int k) { + double distToLeftCenter = computeDistToCenter(pnt, left); + double distToRightCenter = computeDistToCenter(pnt, right); + + TreeNode primaryBranch = distToLeftCenter > distToRightCenter ? right : left; + TreeNode secondaryBranch = primaryBranch == right ? left : right; + + if (primaryBranch != null) + primaryBranch.findKClosest(pnt, heap, k); + + // If the distance to the most distant element in the heap is less than distance to the plane we need to process + // the secondary branch as well. + if (secondaryBranch != null) { + double distToSecondaryBall = computeDistToCenter(pnt, secondaryBranch) - secondaryBranch.getRadius(); + if (heap.size() < k || distToSecondaryBall < heap.peek().getDistance()) + secondaryBranch.findKClosest(pnt, heap, k); + } + } + + /** + * Computed distance from point to center of Ball tree node. + * + * @param pnt Point to be used to calculate distance to other points. + * @param node Ball tree node. + * @return Distance from point to center of Ball tree node. + */ + private double computeDistToCenter(Vector pnt, TreeNode node) { + if (node == null) + return Double.MAX_VALUE; + + return distanceMeasure.compute(pnt, node.getCenter()); + } + } + + /** + * Leaf node of Ball tree that contains an array of points that owned by the leaf. + */ + private final class TreeLeafNode extends TreeNode { + /** Array of points owned by the leaf. */ + private final List> points; + + /** + * Constructs a new instance of Ball tree leaf node. + * + * @param center Center of the ball. + * @param radius Radius of the ball. + * @param points List of points owned by the leaf. + */ + TreeLeafNode(Vector center, double radius, List> points) { + super(center, radius); + this.points = points; + } + + /** {@inheritDoc} */ + @Override void findKClosest(Vector pnt, Queue> heap, int k) { + for (LabeledVector dataPnt : points) { + double distance = distanceMeasure.compute(pnt, dataPnt.features()); + tryToAddIntoHeap(heap, k, dataPnt, distance); + } } } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/matrix/AbstractMatrix.java index ae1025444452a..80cc6e3c5a981 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/matrix/AbstractMatrix.java @@ -862,7 +862,8 @@ else if (mean > threshold + iv) @Override public int hashCode() { int res = 1; res = res * 37 + sto.hashCode(); - res = res * 37 + meta.hashCode(); + // remove@byron + //-res = res * 37 + meta.hashCode(); return res; } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/vector/impl/DelegatingVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/vector/impl/DelegatingVector.java index 9c5adaeb2a1e5..a6a6c20efe404 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/vector/impl/DelegatingVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/primitives/vector/impl/DelegatingVector.java @@ -399,10 +399,10 @@ public Vector getVector() { @Override public int hashCode() { int res = 1; - res = res * 37 + meta.hashCode(); - res = res * 37 + dlg.hashCode(); + //res = res * 37 + meta.hashCode(); + //res = res * 37 + dlg.hashCode(); - return res; + return dlg.hashCode(); } /** {@inheritDoc} */ @@ -414,7 +414,7 @@ public Vector getVector() { return false; DelegatingVector that = (DelegatingVector)o; - - return meta.equals(that.meta) && dlg.equals(that.dlg); + + return dlg.equals(that.dlg); } } diff --git a/modules/mongodb-relay/.factorypath b/modules/mongodb-relay/.factorypath index 1e5332e741ef9..c6a4362f9bca6 100644 --- a/modules/mongodb-relay/.factorypath +++ b/modules/mongodb-relay/.factorypath @@ -1,13 +1,12 @@ - - + + - @@ -71,7 +70,7 @@ - + diff --git a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBackend.java b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBackend.java index 0c7bc79b39f43..591d65db0ddac 100644 --- a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBackend.java +++ b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBackend.java @@ -1,11 +1,14 @@ package de.bwaldvogel.mongo.backend.ignite; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.mongo.MongoPluginConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +16,8 @@ import de.bwaldvogel.mongo.MongoDatabase; import de.bwaldvogel.mongo.backend.AbstractMongoBackend; import de.bwaldvogel.mongo.backend.Utils; +import de.bwaldvogel.mongo.bson.Document; +import de.bwaldvogel.mongo.bson.ObjectId; import de.bwaldvogel.mongo.exception.MongoServerException; @@ -84,6 +89,25 @@ protected MongoDatabase openOrCreateDatabase(String databaseName) { protected Set listDatabaseNames() { return Ignition.allGrids().stream().map(Ignite::name).map(n -> n==null?"default":n).collect(Collectors.toSet()); } + + @Override + protected Document getServerDescription(){ + long topV = admin.cluster().topologyVersion(); + List hostSet = new ArrayList<>(); + StringBuilder primary = new StringBuilder(""); + for(ClusterNode node: admin.cluster().nodes()) { + hostSet.addAll(node.addresses()); + if(node.isLocal()) { + node.addresses().forEach(a->{ if(primary.length()==0) primary.append(a);}); + } + } + Document response = super.getServerDescription(); + ObjectId processId = new ObjectId(admin.cluster().id().toString().replaceAll("-", "").substring(0,24)); + //response.append("topologyVersion", processId); + //response.append("hosts", hostSet); + //response.put("primary",primary); + return response; + } @Override public void close() { diff --git a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBinaryCollection.java b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBinaryCollection.java index 8115331dd38c6..b1401c0d7999b 100644 --- a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBinaryCollection.java +++ b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteBinaryCollection.java @@ -14,6 +14,7 @@ import java.util.stream.StreamSupport; import javax.cache.Cache; +import javax.cache.Cache.Entry; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; @@ -30,6 +31,7 @@ import com.google.common.collect.Sets; import de.bwaldvogel.mongo.backend.AbstractMongoCollection; +import de.bwaldvogel.mongo.backend.CloseableIterator; import de.bwaldvogel.mongo.backend.CollectionOptions; import de.bwaldvogel.mongo.backend.ComposeKeyValue; import de.bwaldvogel.mongo.backend.CursorRegistry; @@ -40,6 +42,7 @@ import de.bwaldvogel.mongo.backend.QueryResult; import de.bwaldvogel.mongo.backend.Utils; import de.bwaldvogel.mongo.backend.ignite.util.BinaryObjectMatch; +import de.bwaldvogel.mongo.backend.ignite.util.TransformerUtil; import de.bwaldvogel.mongo.bson.Document; import de.bwaldvogel.mongo.exception.BadValueException; import de.bwaldvogel.mongo.exception.DuplicateKeyError; @@ -279,39 +282,24 @@ else if(indexResult.size()>1) { @Override - protected QueryResult matchDocuments(Document query, Document orderBy, int numberToSkip, int numberToReturn, - int batchSize, Document fieldSelector) { - Iterable list = this.matchDocuments(query, orderBy, numberToSkip, numberToReturn); - Stream documentStream = StreamSupport.stream(list.spliterator(), false); + protected QueryResult matchDocuments(Document query, Document orderBy, + int numberToSkip, int numberToReturn, int batchSize, Document fieldSelector) { + CloseableIterator list = this.matchDocuments(query, orderBy, numberToSkip, numberToReturn); + Stream documentStream = StreamSupport.stream(list.toSpliterators(), false); + documentStream.onClose(list::close); return matchDocumentsFromStream(documentStream, query, orderBy, numberToSkip, numberToReturn, batchSize, fieldSelector); } - protected Iterable matchDocuments(Document query, Document orderBy, int numberToSkip, int numberToReturn) { - List matchedDocuments = new ArrayList<>(); + protected CloseableIterator matchDocuments(Document query, Document orderBy, int numberToSkip, int numberToReturn) { IgniteBiPredicate filter = new BinaryObjectMatch(query,this.idField); ScanQuery scan = new ScanQuery<>(query.isEmpty()? null: filter); - QueryCursor> cursor = dataMap.query(scan); - //Iterator> it = cursor.iterator(); - for (Cache.Entry entry: cursor) { - Document document = objectToDocument(entry.getKey(),entry.getValue(),this.idField); - matchedDocuments.add(document); - } + QueryCursor> cursor = dataMap.query(scan); - //-sortDocumentsInMemory(matchedDocuments, orderBy); - return matchedDocuments; - } - - protected void sortDocumentsInMemory(List documents, Document orderBy) { - DocumentComparator documentComparator = deriveComparator(orderBy); - if (documentComparator != null) { - documents.sort(documentComparator); - } else if (isNaturalDescending(orderBy)) { - Collections.reverse(documents); - } + return TransformerUtil.map(cursor,this.idField); } @@ -361,7 +349,7 @@ protected Stream> streamAllDocumentsWithPosition() ScanQuery scan = new ScanQuery<>(); QueryCursor> cursor = dataMap.query(scan); - //Iterator> it = cursor.iterator(); + return StreamSupport.stream(cursor.spliterator(),false).map(entry -> new DocumentWithPosition<>(objectToDocument(entry.getKey(),entry.getValue(),this.idField), entry.getKey())); } diff --git a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteDatabase.java b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteDatabase.java index 56fd422e0c231..e2575afe94b89 100644 --- a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteDatabase.java +++ b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteDatabase.java @@ -98,8 +98,8 @@ public MongoCollection resolveCollection(String collectionName, boolean } else { return super.resolveCollection(collectionName, throwIfNotFound); } - } - + } + @Override protected Index openOrCreateUniqueIndex(String collectionName,String indexName, List keys, boolean sparse) { @@ -178,6 +178,8 @@ protected Iterable listCollectionNamespaces() { .map(n->databaseName+'.'+n) .collect(Collectors.toList()); } + + @Override protected MongoCollection openOrCreateCollection(String collectionName, CollectionOptions options) { diff --git a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteVectorIndex.java b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteVectorIndex.java index 39ba25c436b7b..7ba294e7da8ee 100644 --- a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteVectorIndex.java +++ b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/IgniteVectorIndex.java @@ -93,8 +93,7 @@ public class IgniteVectorIndex extends Index { /** Distance measure. */ protected DistanceMeasure distanceMeasure; - /** Index type. */ - private SpatialIndexType idxType = SpatialIndexType.KD_TREE; + private String indexType = SpatialIndexType.BALL_TREE.name(); private String idField = "_id"; @@ -136,7 +135,7 @@ public IgniteVectorIndex(GridKernalContext ctx, IgniteBinaryCollection collectio this.distanceMeasure = new CosineSimilarity(); if(sparse) { - idxType = SpatialIndexType.ARRAY; + indexType = SpatialIndexType.ARRAY.name(); } Document options = keys.get(0).textOptions(); @@ -162,15 +161,14 @@ else if(!similarity.isBlank()) { } } - String indexType = options.getOrDefault("indexType", "").toString(); + String indexType = options.getOrDefault("indexType", "").toString().toUpperCase(); if(!indexType.isBlank()) { if(indexType.startsWith("ANN") || indexType.startsWith("IVF")) { defaultANN = true; } - else { - idxType = SpatialIndexType.valueOf(indexType.toUpperCase()); - } + this.indexType = indexType; } + // 句向量模型 embeddingModelName = (String)options.getOrDefault("modelId", "chinese"); if(embeddingModelName.equals("chinese")) { @@ -218,6 +216,20 @@ public KNNClassificationModel knnModel(){ if(this.knnModel==null) { synchronized(this){ if(this.knnModel==null) { + /** Index type. */ + SpatialIndexType idxType = SpatialIndexType.KD_TREE; + try { + if(indexType.startsWith("ANN") || indexType.startsWith("IVF")) { + defaultANN = true; + } + else { + idxType = SpatialIndexType.valueOf(indexType.toUpperCase()); + } + } + catch(Exception e) { + + } + EmbeddingIntCoordObjectLabelVectorizer vectorizer = new EmbeddingIntCoordObjectLabelVectorizer(); CacheBasedDatasetBuilder datasetBuilder = new CacheBasedDatasetBuilder<>(ctx.grid(), vecIndex); @@ -227,7 +239,7 @@ public KNNClassificationModel knnModel(){ new KNNPartitionDataBuilder<>(vectorizer, idxType, distanceMeasure), environment ); - this.knnModel = new KNNClassificationModel(knnDataset,distanceMeasure,this.K, false); + this.knnModel = new KNNClassificationModel(knnDataset,distanceMeasure,this.K, false); } } } @@ -428,6 +440,7 @@ public Iterable getPositions(Document query) { List positions = getVectorTextList(indexKey,queriedKey); query.remove(indexKey.getKey()); all.addAll(positions); + searchKey = searchKey.copyFrom(n, null); n++; continue; } @@ -445,7 +458,7 @@ public Iterable getPositions(Document query) { searchKey = searchKey.copyFrom(n, keyObj); query.remove(indexKey.getKey()); } - else if (expression.contains(QueryOperator.RNN_VECTOR.getValue())) { + else if (expression.contains(QueryOperator.KNN_VECTOR.getValue())) { searchKey = searchKey.copyFrom(n, keyObj); query.remove(indexKey.getKey()); } @@ -605,6 +618,14 @@ else if(opt.containsKey("$text")) { if(opt.containsKey("$max")) { scoreMax = Float.parseFloat(opt.get("$max").toString()); } + + if(opt.containsKey("$indexType")) { + String indexType = opt.get("$indexType").toString().toUpperCase(); + if(!indexType.equals(this.indexType)) { + this.close(); + this.indexType = indexType; + } + } } Vector vec = this.computeValueEmbedding(obj); @@ -670,6 +691,14 @@ else if(opt.containsKey("$text")) { if(opt.containsKey("$max")) { scoreMax = Float.parseFloat(opt.get("$max").toString()); } + + if(opt.containsKey("$indexType")) { + String indexType = opt.get("$indexType").toString().toUpperCase(); + if(!indexType.equals(this.indexType)) { + this.close(); + this.indexType = indexType; + } + } } Vector vec = this.computeValueEmbedding(obj); diff --git a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/util/DocumentUtil.java b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/util/DocumentUtil.java index 10164e8c399c2..8e66242e4654e 100644 --- a/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/util/DocumentUtil.java +++ b/modules/mongodb-relay/src/main/java/de/bwaldvogel/mongo/backend/ignite/util/DocumentUtil.java @@ -138,7 +138,7 @@ public static Document objectToDocument(Object key,Object obj,String idField){ doc.append(idField, key); doc.append("_data", obj); return doc; - } + } else if(obj instanceof BinaryObject) { BinaryObject bobj = (BinaryObject) obj; return binaryObjectToDocument(key,bobj,idField); @@ -302,17 +302,16 @@ else if(bobj instanceof BinaryArray) { return bin.deserialize(); } String typeName = bobj.type().typeName(); - if(typeName.equals("Document") || typeName.equals("SerializationProxy") || typeName.startsWith("Bson")) { + if(typeName.equals("Document") || typeName.equals("SerializationProxy")) { return bobj.deserialize(); } fields = bobj.type().fieldNames(); - if(fields==null || fields.size()<=2) { + if(fields==null || fields.size()<=1) { return bobj.deserialize(); - } - + } } - catch(Exception e) { + catch(BinaryObjectException e) { fields = bobj.type().fieldNames(); } @@ -467,19 +466,19 @@ public static F readOtherBinaryField(Object buf,int hdr) { case GridBinaryMarshaller.DATE: { val = buf.getClass()==Date.class? buf: null; - if(val==null) { - + if(val==null) { DateFormat[] dateFormats = { - new SimpleDateFormat("yyyy-MM-dd\\'T\\'HH:mm:ss.SSS"), - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"), - new SimpleDateFormat("yyyy-MM-dd HH:mm"), - new SimpleDateFormat("yyyy-MM-dd") - }; + new SimpleDateFormat("yyyy-MM-dd\\'T\\'HH:mm:ss.SSS"), + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"), + new SimpleDateFormat("yyyy-MM-dd HH:mm"), + new SimpleDateFormat("yyyy-MM-dd") + }; for(int i=0;i CloseableIterator map(final Iterator> iterator, Function,E> function){ + return new CloseableIterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public void remove() { + iterator.remove(); + } + + @Override + public E next() { + return function.apply(iterator.next()); + } + + @Override + public void close() { + CloseableIterator.closeIterator(iterator); + } + }; + } + + public static CloseableIterator map(final QueryCursor> cursor,final String idField){ + final Iterator> iterator = cursor.iterator(); + + return new CloseableIterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public void remove() { + iterator.remove(); + } + + @Override + public Document next() { + Cache.Entry entry = iterator.next(); + Document document = objectToDocument(entry.getKey(),entry.getValue(),idField); + return document; + } + + @Override + public void close() { + cursor.close(); + } + }; + } + +} diff --git a/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/SimpleTest.java b/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/SimpleTest.java index 6178a23c60785..7c3f9d51c2d2d 100644 --- a/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/SimpleTest.java +++ b/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/SimpleTest.java @@ -110,7 +110,7 @@ public void testSecondarySparseFullTextIndex() throws Exception { System.out.println(doc); } - collection.updateOne(json("_id: 2"), new Document("$set", json("text: 'def',title:'这个 是 标题'"))); + collection.updateOne(json("_id: 2"), new Document("$set", json("text: 'def', title:'这个 是 标题'"))); ret = collection.find(json("title: {$text: '标题'}")); it = ret.cursor(); while(it.hasNext()) { diff --git a/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/TransactionTest.java b/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/TransactionTest.java new file mode 100644 index 0000000000000..43473a2a4308d --- /dev/null +++ b/modules/mongodb-relay/src/test/java/de/bwaldvogel/examples/TransactionTest.java @@ -0,0 +1,159 @@ +package de.bwaldvogel.examples; + + + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; + +import org.apache.ignite.internal.processors.mongo.MongoPluginConfiguration; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.IterableAssert; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.bson.BsonDocument; +import org.bson.Document; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; + +import com.mongodb.MongoWriteException; +import com.mongodb.ServerAddress; +import com.mongodb.client.ClientSession; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.IndexOptions; +import com.mongodb.session.ServerSession; + +import de.bwaldvogel.mongo.MongoServer; +import de.bwaldvogel.mongo.backend.AbstractTest; +import de.bwaldvogel.mongo.backend.TestUtils; +import de.bwaldvogel.mongo.backend.ignite.IgniteBackend; +import static de.bwaldvogel.mongo.backend.TestUtils.json; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + + +public class TransactionTest { + + private static MongoCollection collection; + private static MongoClient client; + private static MongoServer server; + + @BeforeEach + public void setUp() { + if(server!=null) { + return ; + } + server = new MongoServer(IgniteBackend.inMemory(new MongoPluginConfiguration())); + + // bind on a random local port + String serverAddress = server.bindAndGetConnectionString(); + + client = MongoClients.create(serverAddress); + + for(String name:client.getDatabase("testdb").listCollectionNames()) { + System.out.println(name); + } + collection = client.getDatabase("testdb").getCollection("testcoll"); + } + + @AfterEach + public void tearDown() { + collection.drop(); + //client.getDatabase("testdb").drop(); + //client.close(); + //server.shutdown(); + } + + @Test + void testSimpleCursor() { + int expectedCount = 20; + int batchSize = 10; + for (int i = 0; i < expectedCount; i++) { + collection.insertOne(new Document("_id", 100 + i)); + } + MongoCursor cursor = collection.find().batchSize(batchSize).cursor(); + List retrievedDocuments = new ArrayList<>(); + while (cursor.hasNext()) { + retrievedDocuments.add(cursor.next()); + } + + assertThatExceptionOfType(NoSuchElementException.class) + .isThrownBy(cursor::next) + .withMessage(null); + + } + + // see https://github.com/bwaldvogel/mongo-java-server/issues/39 + @Test + public void testBuildData() throws Exception { + + ClientSession session = client.startSession(); + + BsonDocument id = session.getServerSession().getIdentifier(); + + session.startTransaction(); + + collection.insertOne(json("_id: 1, text: 'abc'")); + collection.insertOne(json("_id: 2, text: 'def'")); + collection.insertOne(json("_id: 3, title: '标题'")); + collection.insertOne(json("_id: 4")); + collection.insertOne(json("_id: 5, text: 'def'")); + + + collection.deleteOne(json("_id: 5")); + + collection.updateOne(json("_id: 2"), new Document("$set", json("text: null"))); + collection.updateOne(json("_id: 1"), new Document("$set", json("text: 'def'"))); + + session.notifyMessageSent(); + + session.abortTransaction(); + + session.close(); + + FindIterable ret = collection.find(json("$text: {$search: 'def'}")); + MongoCursor it = ret.cursor(); + while(it.hasNext()) { + Document doc = it.next(); + System.out.println(doc); + } + + System.out.println("finish full text"); + } + + + protected static IterableAssert assertThat(Iterable actual) { + // improve assertion array by collection entire array + List values = TestUtils.toArray(actual); + return Assertions.assertThat((Iterable) values); + } + + protected static void assertMongoWriteException(ThrowingCallable callable, int expectedErrorCode, String expectedMessage) { + assertMongoWriteException(callable, expectedErrorCode, "Location" + expectedErrorCode, expectedMessage); + } + + protected static void assertMongoWriteException(ThrowingCallable callable, int expectedErrorCode, String expectedCodeName, + String expectedMessage) { + try { + callable.call(); + } + catch(MongoWriteException e) { + e.printStackTrace(); + } + catch (Throwable e1) { + e1.printStackTrace(); + + } + } + +} \ No newline at end of file diff --git a/web-console/web-console-server/pom.xml b/web-console/web-console-server/pom.xml index 8687c92e483c5..de894edfe6896 100644 --- a/web-console/web-console-server/pom.xml +++ b/web-console/web-console-server/pom.xml @@ -160,6 +160,12 @@ ${web-console.ignite.version} + + com.h2database + h2 + 1.4.197 + + org.apache.ignite ignite-log4j2 diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/CacheHolder.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/CacheHolder.java index 47d1fd4e944b7..e8da2adeb6552 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/CacheHolder.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/CacheHolder.java @@ -43,7 +43,7 @@ public class CacheHolder { protected final String cacheName; /** */ - private IgniteCache cache; + private IgniteCache cache; /** */ protected ExpiryPolicy expiryPlc; diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/NestedTransaction.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/NestedTransaction.java index c971649624dc9..5ad2aebeccdc0 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/NestedTransaction.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/NestedTransaction.java @@ -19,7 +19,7 @@ import java.util.UUID; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; -import org.apache.ignite.lang.IgniteAsyncSupport; + import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.transactions.Transaction; @@ -147,21 +147,4 @@ public NestedTransaction(Transaction delegate) { return delegate.label(); } - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public IgniteAsyncSupport withAsync() { - return delegate.withAsync(); - } - - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public boolean isAsync() { - return delegate.isAsync(); - } - - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public IgniteFuture future() { - return delegate.future(); - } } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/Table.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/Table.java index 16fd7327227a4..12a03c6d05248 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/Table.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/Table.java @@ -200,38 +200,5 @@ public void deleteAll(Set ids) { ((IgniteCache)cache()).removeAll(idxIds); } - - /** - * Index for unique constraint. - */ - public static class UniqueIndex { - /** */ - private final Function keyGenerator; - - /** */ - private final Function msgGenerator; - - /** - * Constructor. - */ - UniqueIndex(Function keyGenerator, Function msgGenerator) { - this.keyGenerator = keyGenerator; - this.msgGenerator = msgGenerator; - } - - /** - * @param val Value. - * @return Unique key. - */ - public Object key(T val) { - return keyGenerator.apply(val); - } - - /** - * @param val Value. - */ - public String message(T val) { - return msgGenerator.apply(val); - } - } + } diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/UniqueIndex.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/UniqueIndex.java new file mode 100644 index 0000000000000..5f222ee5db6bd --- /dev/null +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/db/UniqueIndex.java @@ -0,0 +1,37 @@ +package org.apache.ignite.console.db; + +import java.util.function.Function; + +/** + * Index for unique constraint. + */ +public class UniqueIndex { + /** */ + private final Function keyGenerator; + + /** */ + private final Function msgGenerator; + + /** + * Constructor. + */ + UniqueIndex(Function keyGenerator, Function msgGenerator) { + this.keyGenerator = keyGenerator; + this.msgGenerator = msgGenerator; + } + + /** + * @param val Value. + * @return Unique key. + */ + public Object key(T val) { + return keyGenerator.apply(val); + } + + /** + * @param val Value. + */ + public String message(T val) { + return msgGenerator.apply(val); + } +} diff --git a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/security/IgniteSessionRepository.java b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/security/IgniteSessionRepository.java index 42c182c5f9d47..921a157f8b6b3 100644 --- a/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/security/IgniteSessionRepository.java +++ b/web-console/web-console-server/src/main/java/org/apache/ignite/console/web/security/IgniteSessionRepository.java @@ -26,6 +26,7 @@ import org.springframework.session.MapSession; import org.springframework.session.Session; import org.springframework.session.SessionRepository; +import org.springframework.security.core.authority.SimpleGrantedAuthority; import static org.apache.ignite.console.errors.Errors.convertToDatabaseNotAvailableException; diff --git a/web-console/web-console-server/src/test/java/org/apache/ignite/console/MockConfiguration.java b/web-console/web-console-server/src/test/java/org/apache/ignite/console/MockConfiguration.java index 18825646e572e..003191b8f6769 100644 --- a/web-console/web-console-server/src/test/java/org/apache/ignite/console/MockConfiguration.java +++ b/web-console/web-console-server/src/test/java/org/apache/ignite/console/MockConfiguration.java @@ -32,7 +32,7 @@ import org.apache.ignite.console.web.socket.AgentsService; import org.apache.ignite.console.web.socket.TransitionService; import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; -import org.apache.ignite.lang.IgniteAsyncSupport; + import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.testframework.GridTestNode; @@ -252,23 +252,6 @@ private static class TransactionMock implements Transaction { @Override public @Nullable String label() { throw new UnsupportedOperationException(); } - - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public IgniteAsyncSupport withAsync() { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public boolean isAsync() { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @SuppressWarnings("deprecation") - @Override public IgniteFuture future() { - throw new UnsupportedOperationException(); - } + } }