Upgrade druzhba to Python 310 (#75)

* python 310

* more official

* 310 in ci

* pytest

* pytest

Co-authored-by: ahsiungsg <[email protected]>

.env.test.docker

@@ -1,13 +1,15 @@
-AWS_ACCESS_KEY_ID
-AWS_SECRET_ACCESS_KEY
-DRUZHBA_CONFIG_DIR
-S3_BUCKET
-S3_PREFIX
-REDSHIFT_URL
-REDSHIFT_HOST
-REDSHIFT_PORT
-REDSHIFT_DATABASE
-REDSHIFT_USER
-REDSHIFT_PASSWORD
-REDSHIFT_CERT_PATH
-REDSHIFT_IAM_COPY_ROLE
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_SESSION_TOKEN=
+DRUZHBA_CONFIG_DIR=
+S3_BUCKET=
+S3_PREFIX=
+LOGLEVEL=
+REDSHIFT_URL=
+REDSHIFT_HOST=
+REDSHIFT_PORT=
+REDSHIFT_DATABASE=
+REDSHIFT_USER=
+REDSHIFT_PASSWORD=
+REDSHIFT_IAM_COPY_ROLE=
+REDSHIFT_CERT_PATH=

.env.test.sample

@@ -1,14 +1,15 @@
-export AWS_ACCESS_KEY_ID=
-export AWS_SECRET_ACCESS_KEY=
-export DRUZHBA_CONFIG_DIR=test/integration/config
-export S3_BUCKET=test-bucket
-export S3_PREFIX=pipeline
-export LOGLEVEL=INFO
-export REDSHIFT_URL=
-export REDSHIFT_HOST=
-export REDSHIFT_PORT=
-export REDSHIFT_DATABASE=
-export REDSHIFT_USER=
-export REDSHIFT_PASSWORD=
-export REDSHIFT_IAM_COPY_ROLE=
-export REDSHIFT_CERT_PATH=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_SESSION_TOKEN=
+DRUZHBA_CONFIG_DIR=
+S3_BUCKET=
+S3_PREFIX=
+LOGLEVEL=
+REDSHIFT_URL=
+REDSHIFT_HOST=
+REDSHIFT_PORT=
+REDSHIFT_DATABASE=
+REDSHIFT_USER=
+REDSHIFT_PASSWORD=
+REDSHIFT_IAM_COPY_ROLE=
+REDSHIFT_CERT_PATH=

.github/workflows/test.yml

@@ -7,17 +7,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.7
+    - name: Set up Python 3.10.4
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: '3.10.4'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e '.[test]'
-    - name: Test with nosetests
+    - name: Test with pytest
       run: |
-        python setup.py test
+        python -m pytest test/unit
     - name: Lint with pylint
       run: |
         pip install pylint

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV APP_PATH=/app \
     DEBIAN_FRONTEND=noninteractive \
@@ -8,18 +8,23 @@ ENV APP_PATH=/app \
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
     apt-get install -y --no-install-recommends \
-        build-essential postgresql mysql-client \
-        python3-pip python3.7 python3.7-dev python3.7-venv && \
+        ca-certificates git make build-essential libssl-dev zlib1g-dev \
+        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+        postgresql mysql-server && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN python3.7 -m venv /home/venv/druzhba && \
-    /home/venv/druzhba/bin/python -m pip install --upgrade pip
-
+RUN curl https://pyenv.run | bash
+ENV PATH="/root/.pyenv/bin:${PATH}"
 WORKDIR /app
-
-ENV PATH=/home/venv/druzhba/bin:$PATH
-
 COPY . .
+ARG PYTHON_VERSION=3.10.4
+RUN eval "$(pyenv init --path)" \
+    && eval "$(pyenv virtualenv-init -)" \
+    && pyenv install ${PYTHON_VERSION} \
+    && pyenv global ${PYTHON_VERSION} \
+    && pip install -e .[test]
 
-RUN pip install -e .[test]
+RUN echo 'eval "$(pyenv init --path)"' >> /root/.bashrc
+RUN echo 'eval "$(pyenv virtualenv-init -)"' >> /root/.bashrc

docker-compose.yml

@@ -2,6 +2,7 @@ version: '3.7'
 services:
   postgres:
     image: postgres:10
+    hostname: druzhba_postgres_1
     ports:
       - "5433:5432"
     networks:
@@ -15,6 +16,7 @@ services:
 
   mysql:
     image: mysql/mysql-server:5.7
+    hostname: druzhba_mysql_1
     ports:
       - "3307:3306"
     networks:
@@ -25,6 +27,9 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: 'mysql_root_password'
   test:
+    depends_on:
+      - postgres
+      - mysql
     build:
       context: .
       dockerfile: Dockerfile

druzhba/avro.py

@@ -77,7 +77,7 @@ def write_avro_file(f, results_iter, fields, table, max_size=100 * 1024 ** 2):
 
     try:
         # writer.io buffers before writing
-        while f.tell() + writer.io.tell() < max_size:
+        while f.tell() + writer.io.tell() < max_size:  # pylint: disable=no-member
             writer.write(_format_row(next(results_iter)))
             row_count += 1
     except StopIteration:

setup.cfg

@@ -1,5 +0,0 @@
-[nosetests]
-logging-level=INFO
-
-[aliases]
-test=nosetests --tests test/unit/

setup.py

@@ -12,7 +12,7 @@
     exec(f.read(), package_vars)
 
 tests_require = [
-    "nose>=1.3.7",
+    "pytest>=7.1.2",
     "mock>=2.0.0",
     "pylint==2.5.3",
 ]
@@ -41,7 +41,7 @@
     install_requires=[
         "boto3>=1.10.34",
         "botocore>=1.13.35",
-        "fastavro>=0.21.22,<0.22",
+        "fastavro>=1.6.0,<1.7.0",
         "Jinja2>=2.10",
         "psycopg2-binary>=2.7.3.2",
         "pyaml>=17.10.0",

test/integration/run_test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/bash
 
 set -u
 
@@ -30,7 +30,7 @@ if [[ ${TEST_POSTGRES} == 1 ]]; then
     echo "Setup complete, running postgres test suite"
 
     PGTEST_DATABASE_URL=postgresql://druzhba_test_user:druzhba_password@druzhba_postgres_1:5432/druzhba_test \
-        nosetests --tests test/integration/test_postgres_to_redshift.py
+        pytest test/integration/test_postgres_to_redshift.py
 
     echo "Tearing down postgres source database"
 
@@ -51,7 +51,7 @@ if [[ ${TEST_MYSQL} == 1 ]]; then
     echo "Setup complete, running mysql test suite"
 
     MYSQLTEST_DATABASE_URL=mysql://druzhba_user:druzhba_password@druzhba_mysql_1:3306/druzhba_test \
-        nosetests --tests test/integration/test_mysql_to_redshift.py
+        pytest test/integration/test_mysql_to_redshift.py
 
     echo "Tearing down mysql source database"