Remove redis/tikv maindb drivers (#2254)

.github/workflows/ci.yml

@@ -183,7 +183,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        maindb_driver: ["tikv", "pg"]
+        maindb_driver: ["pg"]
         shard: [0, 1, 2]
 
     steps:
@@ -202,16 +202,11 @@ jobs:
           key: venv-${{ github.sha }}
           fail-on-cache-miss: true
 
-      - name: Install tikv
-        if: matrix.maindb_driver == 'tikv'
-        run: ./scripts/run-tikv.sh
-
       - name: Run NucliaDB tests
         # These tests can be flaky, let's retry them...
         uses: nick-fields/retry@v2
         env:
           TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }}
-          TESTING_TIKV_LOCAL: "true"
         with:
           max_attempts: 2
           retry_on: error

.github/workflows/nucliadb.yml

@@ -2,7 +2,6 @@ name: nucliadb (py)
 
 env:
   COMPONENT: nucliadb
-  TESTING_TIKV_LOCAL: "true"
   GCP_WORKLOAD_IDENTITY_PROVIDER: "projects/224545243904/locations/global/workloadIdentityPools/gh-nuclia/providers/gh-nuclia-provider"
   GCP_SERVICE_ACCOUNT: "[email protected]"
 
@@ -32,7 +31,7 @@ jobs:
       max-parallel: 6
       matrix:
         python-version: ["3.12"]
-        maindb_driver: ["tikv", "tikv_2", "tikv_3", "pg", "pg_2", "pg_3"]
+        maindb_driver: ["pg", "pg_2", "pg_3"]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -46,10 +45,6 @@ jobs:
       - name: Install the package
         run: make -C nucliadb/ install-dev
 
-      - name: Install tikv
-        if: matrix.maindb_driver == 'tikv' || matrix.maindb_driver == 'tikv_2' || matrix.maindb_driver == 'tikv_3'
-        run: sudo cp ./scripts/pingcap-com.pem /usr/local/share/ca-certificates/pingcap.crt && sudo update-ca-certificates && ./scripts/run-tikv.sh
-
       - uses: dorny/paths-filter@v2
         id: filter
         with:
@@ -120,45 +115,6 @@ jobs:
           command: |
             make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=2 --num-shards=3"
 
-      - name: Run Tikv tests 1/2
-        # These tests can be flaky, let's retry them...
-        uses: nick-fields/retry@v2
-        if: matrix.maindb_driver == 'tikv'
-        env:
-          TESTING_MAINDB_DRIVERS: tikv
-        with:
-          max_attempts: 2
-          retry_on: error
-          timeout_minutes: 10
-          command: |
-            make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=0 --num-shards=3"
-
-      - name: Run Tikv tests 2/3
-        # These tests can be flaky, let's retry them...
-        uses: nick-fields/retry@v2
-        if: matrix.maindb_driver == 'tikv_2'
-        env:
-          TESTING_MAINDB_DRIVERS: "tikv"
-        with:
-          max_attempts: 2
-          retry_on: error
-          timeout_minutes: 10
-          command: |
-            make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=1 --num-shards=3"
-
-      - name: Run Tikv tests 3/3
-        # These tests can be flaky, let's retry them...
-        uses: nick-fields/retry@v2
-        if: matrix.maindb_driver == 'tikv_3'
-        env:
-          TESTING_MAINDB_DRIVERS: "tikv"
-        with:
-          max_attempts: 2
-          retry_on: error
-          timeout_minutes: 10
-          command: |
-            make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=2 --num-shards=3"
-
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/nucliadb_ingest.yml

@@ -15,7 +15,6 @@ concurrency:
 env:
   CONTAINER_REGISTRY: europe-west4-docker.pkg.dev/nuclia-internal/nuclia
   IMAGE_NAME: ingest
-  TESTING_TIKV_LOCAL: "true"
 
 jobs:
   # Job to run tests
@@ -26,10 +25,7 @@ jobs:
       max-parallel: 2
       matrix:
         include:
-          - maindb_driver: "tikv"
-            storage_backend: "gcs"
-            python-version: "3.12"
-          - maindb_driver: "tikv"
+          - maindb_driver: "pg"
             storage_backend: "s3"
             python-version: "3.12"
           - maindb_driver: "pg"
@@ -48,10 +44,6 @@ jobs:
       - name: Install the package
         run: make -C nucliadb/ install-dev
 
-      - name: Install tikv
-        if: matrix.maindb_driver == 'tikv'
-        run: sudo cp ./scripts/pingcap-com.pem /usr/local/share/ca-certificates/pingcap.crt && sudo update-ca-certificates && ./scripts/run-tikv.sh
-
       - name: Run tests
         env:
           TESTING_MAINDB_DRIVER: ${{ matrix.maindb_driver }}

.github/workflows/nucliadb_search.yml

@@ -60,10 +60,7 @@ jobs:
       max-parallel: 3
       matrix:
         include:
-          - maindb_driver: "tikv"
-            storage_backend: "gcs"
-            python-version: "3.12"
-          - maindb_driver: "tikv"
+          - maindb_driver: "pg"
             storage_backend: "s3"
             python-version: "3.12"
           - maindb_driver: "pg"

Dockerfile

@@ -3,11 +3,6 @@ FROM python:3.12
 WORKDIR /usr/src/app
 
 # Cachable layers
-RUN ARCH="$(uname -m)"; \
-    case "$ARCH" in \
-    aarch64) pip install https://storage.googleapis.com/stashify-cdn/python/tikv_client-0.0.3-cp36-abi3-manylinux_2_31_aarch64.whl;; \
-    x86_64) pip install https://storage.googleapis.com/stashify-cdn/python/tikv_client-0.0.3-cp36-abi3-manylinux_2_31_x86_64.whl;; \
-    esac;
 RUN set -eux; \
     dpkgArch="$(dpkg --print-architecture)"; \
     case "${dpkgArch##*-}" in \

Dockerfile.rust

@@ -2,8 +2,7 @@ FROM python:3.12 as python_rust
 
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    TIKV_CLIENT_VERSION=0.0.3
+    PATH=/usr/local/cargo/bin:$PATH
 
 COPY rust-toolchain.toml /tmp
 
@@ -33,13 +32,6 @@ RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf
 
 RUN pip install maturin
 
-RUN set -eux; \
-    wget "https://github.com/tikv/client-py/archive/refs/tags/${TIKV_CLIENT_VERSION}.tar.gz"; \
-    tar -xzf ${TIKV_CLIENT_VERSION}.tar.gz; \
-    cd client-py-${TIKV_CLIENT_VERSION}; \
-    maturin build --release; \
-    cp target/wheels/tikv_client-${TIKV_CLIENT_VERSION}-*.whl /
-
 COPY . /nucliadb
 
 WORKDIR /nucliadb

Dockerfile.withbinding

@@ -44,14 +44,6 @@ RUN set -eux; \
 
 
 FROM python:3.12
-ENV TIKV_CLIENT_VERSION=0.0.3
-
-
-RUN ARCH="$(uname -m)"; \
-    case "$ARCH" in \
-    aarch64) pip install https://storage.googleapis.com/stashify-cdn/python/tikv_client-${TIKV_CLIENT_VERSION}-cp36-abi3-manylinux_2_31_aarch64.whl;; \
-    x86_64) pip install https://storage.googleapis.com/stashify-cdn/python/tikv_client-${TIKV_CLIENT_VERSION}-cp36-abi3-manylinux_2_31_x86_64.whl;; \
-    esac;
 
 RUN mkdir -p /usr/src/app
 

README.md

@@ -50,8 +50,8 @@ and inference. We do all the hard work for you.
 - Resources with multiple fields and metadata
 - Text/HTML/Markdown plain fields support
 - Field types: text, file, link, conversation
-- Storage layer support: TiKV, Redis and PostgreSQL
-- Blob support with S3-compatible API, GCS and PG drivers
+- Storage layer (PostgreSQL)
+- Blob support with S3-compatible API, GCS and Azure Blob Storage
 - Replication of index storage
 - Distributed search
 - Cloud-native

charts/nucliadb_ingest/values.yaml

@@ -45,8 +45,8 @@ serviceMonitor:
 
 # Value of ports to where this component connects to
 services:
-  # This must match the redis or tikv driver on nucliadb_shared
-  maindb: 6379 # redis = 6379 | tikv = 2379
+  # This must match the pg driver on nucliadb_shared
+  maindb: 5432
   nats: 4222
 
 running:
@@ -128,4 +128,4 @@ ingest_subscriber_workers_autoscaling:
 
 # extra_pod_annotations:
 #   what: "add annotations"
-#   where: "in the pods"
+#   where: "in the pods"

charts/nucliadb_reader/values.yaml

@@ -37,8 +37,8 @@ serving:
   metricsPort: 3030
 
 services:
-  # This must match the redis or tikv driver on nucliadb_shared
-  maindb: 6379 | 2379
+  # This must match the pg driver on nucliadb_shared
+  maindb: 5432
   nats: 4222
 
 running:
@@ -62,9 +62,8 @@ zone:
 # This configuration must match the one in ingest
 # THis is needed as reader component uses the orm module in ingest
 ingest:
-  driver: redis # tikv | redis
-  driver_redis_url: xx
-  driver_tikv_url: xx
+  driver: pg
+  driver_pg_url: xx
 
 serviceMonitor:
   enabled: False

charts/nucliadb_search/values.yaml

@@ -41,8 +41,8 @@ serving:
   metricsPort: 3030
 
 services:
-  # This must match the redis or tikv driver on nucliadb_shared
-  maindb: 6379 | 2379
+  # This must match the pg driver on nucliadb_shared
+  maindb: 5432
   nats: 4222
 
 running:

charts/nucliadb_shared/templates/nucliadb.cm.yaml

@@ -69,15 +69,8 @@ data:
 {{- end}}
   INDEX_JETSTREAM_SERVERS: {{ toJson .Values.indexing.index_jetstream_servers | quote }}
   DRIVER: {{ .Values.maindb.driver }}
-{{- if eq .Values.maindb.driver "redis" }}
-  DRIVER_REDIS_URL: {{ .Values.maindb.driver_redis_url }}
-{{- else if eq .Values.maindb.driver "pg" }}
+{{- if eq .Values.maindb.driver "pg" }}
   DRIVER_PG_URL: {{ .Values.maindb.driver_pg_url }}
-{{- else if eq .Values.maindb.driver "tikv" }}
-  DRIVER_TIKV_URL: {{ toJson .Values.maindb.driver_tikv_url | quote }}
-{{- if .Values.maindb.driver_tikv_replication_pg_url }}
-  DRIVER_TIKV_REPLICATION_PG_URL: {{ .Values.maindb.driver_tikv_replication_pg_url }}
-{{- end }}
 {{- end }}
   AUDIT_DRIVER: {{ .Values.audit.driver }}
   AUDIT_JETSTREAM_TARGET: {{ .Values.audit.audit_jetstream_target }}

charts/nucliadb_shared/values.yaml

@@ -10,9 +10,8 @@ http:
     - http://localhost:4200
 
 maindb:
-  driver: redis # tikv | redis
-  driver_redis_url:
-  driver_tikv_url:
+  driver: pg
+  driver_pg_url:
 
 storage:
   file_backend: gcs

charts/nucliadb_train/values.yaml

@@ -38,8 +38,8 @@ serving:
   grpc_port: 8080
 
 services:
-  # This must match the redis or tikv driver on nucliadb_shared
-  maindb: 2379 # redis: 6379 | tikv: 2379
+  # This must match the pg driver on nucliadb_shared
+  maindb: 5432
   nats: 4222
 
 running:

charts/nucliadb_writer/values.yaml

@@ -37,8 +37,8 @@ serving:
   metricsPort: 3030
 
 services:
-  # This must match the redis or tikv driver on nucliadb_shared
-  maindb: 6379 | 2379
+  # This must match the pg driver on nucliadb_shared
+  maindb: 5432
   nats: 4222
 
 running: