Merge pull request #167 from Ilhasoft/staging

Staging

.dockerignore

@@ -23,4 +23,4 @@ settings.ini
 db.sqlite3
 
 # spacy-langs
-bothub-nlp-nlu-worker/spacy-langs
+bothub-nlp-nlu-worker/spacy-langs

.gitignore

@@ -113,3 +113,7 @@ dmypy.json
 
 # IDEs
 .vscode/
+.idea/
+.DS_Store
+.idea/
+.env

Makefile

@@ -1,31 +1,28 @@
 lint:
 	@echo "${INFO}Linting...${NC}"
 	@export PIPENV_DONT_LOAD_ENV=1
-	@cd bothub-nlp \
-		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
-		&& echo "${SUCCESS}✔${NC} bothub-nlp" || echo "${DANGER}✖${NC} bothub-nlp"
 	@cd bothub-nlp-api \
 		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
 		&& echo "${SUCCESS}✔${NC} bothub-nlp-api" || echo "${DANGER}✖${NC} bothub-nlp-api"
-	@cd bothub-nlp-celery \
-		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
-		&& echo "${SUCCESS}✔${NC} bothub-nlp-celery" || echo "${DANGER}✖${NC} bothub-nlp-celery"
-	@cd bothub-nlp-nlu \
-		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
-		&& echo "${SUCCESS}✔${NC} bothub-nlp-nlu" || echo "${DANGER}✖${NC} bothub-nlp-nlu"
 	@cd bothub-nlp-nlu-worker \
 		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
 		&& echo "${SUCCESS}✔${NC} bothub-nlp-nlu-worker" || echo "${DANGER}✖${NC} bothub-nlp-nlu-worker"
 	@cd bothub-nlp-nlu-worker-on-demand \
 		&& PIPENV_DONT_LOAD_ENV=1 pipenv run lint \
 		&& echo "${SUCCESS}✔${NC} bothub-nlp-nlu-worker-on-demand" || echo "${DANGER}✖${NC} bothub-nlp-nlu-worker-on-demand"
 
-test:
-	@echo "${INFO}Testing...${NC}"
-	@cd bothub-nlp-nlu \
-		&& PIPENV_DONT_LOAD_ENV=1 SECRET_KEY=SK DJANGO_SETTINGS_MODULE="bothub.settings" pipenv run django-admin migrate \
-		&& PIPENV_DONT_LOAD_ENV=1 SECRET_KEY=SK SUPPORTED_LANGUAGES="en|pt" SEND_EMAILS=false ASYNC_TEST_TIMEOUT=30 pipenv run test \
-		&& echo "${SUCCESS}✔${NC} bothub-nlp-nlu" || echo "${DANGER}✖${NC} bothub-nlp-nlu"
+init_env:
+	@echo "${INFO}Starting init environment...${NC}"
+	@echo "SUPPORTED_LANGUAGES=en:en_core_web_md" >> .env
+	@echo "BOTHUB_ENGINE_URL=https://api.bothub.it" >> .env
+	@echo "ENGINE_PORT=8000" >> .env
+	@echo "${SUCCESS}Finish...${NC}"
+
+start_development:
+	@echo "${INFO}Starting Build all project (Docker)...${NC}"
+	@docker-compose build --build-arg DOWNLOAD_SPACY_MODELS=en:en_core_web_md
+	@docker-compose up -d
+	@echo "${SUCCESS}Finish...${NC}"
 
 
 # Utils

README.md

@@ -20,3 +20,56 @@ Check the [main Bothub project repository](https://github.com/Ilhasoft/bothub).
 ### bothub-nlp-celery (python 3.6)
 
 ### bothub-nlp-nlu (python 3.6)
+
+
+# Requirements
+
+* Python (3.6)
+* Docker
+* Docker-Compose
+
+## Development
+
+Use ```make``` commands to ```lint```, ```init_env```, ```start_development```.
+
+| Command | Description |
+|--|--|
+| make lint | Show lint warnings and errors
+| make init_env | Init file .env with variables environment
+| make start_development | Create .env with variable environment and start build docker
+
+
+## Environment Variables
+
+You can set environment variables in your OS, write on ```.env``` file or pass via Docker config.
+
+| Variable | Type | Default | Description |
+|--|--|--|--|
+| SUPPORTED_LANGUAGES | ```string```| ```en\|pt``` | Set supported languages. Separe languages using ```\|```. You can set location follow the format: ```[LANGUAGE_CODE]:[LANGUAGE_LOCATION]```.
+| BOTHUB_NLP_SENTRY_CLIENT | ```bool``` | ```None``` | Sentry Client |
+| BOTHUB_NLP_CELERY_BROKER_URL | ```string``` | ```redis://localhost:6379/0	``` | Celery Broker URL, check usage instructions in Celery Docs |
+| BOTHUB_NLP_CELERY_BACKEND_URL | ```string``` | ```BOTHUB_NLP_CELERY_BROKER_URL``` value | Celery Backend URL, check usage instructions in Celery Docs
+| BOTHUB_NLP_NLU_AGROUP_LANGUAGE_QUEUE | ```boolean``` | ```True``` | Agroup tasks by language in celery queue, if `True` there will be only one queue per language. |
+| BOTHUB_NLP_DOCKER_CLIENT_BASE_URL | `string` | `unix://var/run/docker.sock` | Docker Client Rest API URL. |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_API_PORT | `int` | `2658` | Port to server API webservice. |
+| BOTHUB_NLP_NLU_WORKER_DOCKER_IMAGE_NAME | `string` | `ilha/bothub-nlp-nlu-worker` | Bothub NLP NLU Worker Docker image. |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_DOWN_TIME | `int` | `10` | Down worker after x minutes without interaction. |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_NETWORKS | List separated by common. | `bothub-nlp` | Networks to assign in new worker service. |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_RUN_IN_WORKER_NODE | `boolean` | `False` | Run service just in Workers Nodes in Docker Swarm cluster. |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_CONFIG_FILE | `string` | `bothub-nlp-nlu-worker-on-demand.cfg` | |
+| BOTHUB_NLP_NLU_WORKER_ON_DEMAND_API_BASIC_AUTHORIZATION | `string` | `None` | Fill `api_basic_authorization` kwarg in [CeleryWorkerOnDemand](https://github.com/Ilhasoft/celery-worker-on-demand#class-celeryworkerondemand) constructor |
+| BOTHUB_NLP_AWS_S3_BUCKET_NAME | `str` |  |  |
+| BOTHUB_NLP_AWS_ACCESS_KEY_ID | `str` |  |  |
+| BOTHUB_NLP_AWS_SECRET_ACCESS_KEY | `str` |  |  |
+| BOTHUB_NLP_AWS_REGION_NAME | `str` |  |  |
+| BOTHUB_ENGINE_URL | ```string```|  ```https://api.bothub.it``` | Bothub-engine API URL
+| BOTHUB_NLP_LANGUAGE_QUEUE | `str` | en | Set language that will be loaded in celery |
+| BOTHUB_NLP_SERVICE_WORKER | `boolean` | `False` | Set true if you are running celery bothub-nlp-nlu-worker |
+
+## Docker Arguments
+
+You need to set --build-arg when you are building docker-compose
+
+| Argument | Type | Default | Description |
+|--|--|--|--|
+| DOWNLOAD_SPACY_MODELS | ```string```|  ```en:en_core_web_md``` | Set supported languages. Separe languages using ```\|```. You can set location follow the format: ```[LANGUAGE_CODE]:[LANGUAGE_LOCATION]```.

bothub-nlp-api/.flake8

@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 119
+ignore = E501,W503,E203
+exclude =
+  ./env
+  ./gunicorn.conf.py

bothub-nlp-api/Dockerfile

@@ -5,10 +5,7 @@ ENV BOTHUB_NLP_API_PORT 2657
 
 WORKDIR $WORKDIR
 
-COPY bothub-nlp ../bothub-nlp
-COPY bothub-nlp-celery ../bothub-nlp-celery
-COPY bothub-nlp-api/Pipfile .
-COPY bothub-nlp-api/Pipfile.lock .
+COPY bothub-nlp-api .
 
 RUN apk update \
     && apk add --virtual .build-dependencies --no-cache \
@@ -21,6 +18,6 @@ RUN apk update \
     && apk del .build-dependencies \
     && rm -rf /var/cache/apk/*
 
-COPY bothub-nlp-api .
+RUN chmod +x ./entrypoint.sh
 
-ENTRYPOINT [ "python", "-m", "bothub_nlp_api" ]
+ENTRYPOINT [ "./entrypoint.sh" ]

bothub-nlp-api/Pipfile

@@ -4,21 +4,29 @@ verify_ssl = true
 name = "pypi"
 
 [scripts]
-start = "python -m tornado.autoreload -m bothub_nlp_api"
 lint = "flake8"
 
 [packages]
-tornado = "==5.1.1"
 raven = "*"
-bothub-engine = {ref = "1.21.0", git = "https://github.com/Ilhasoft/bothub-engine"}
-bothub-nlp-celery = {path = "./../bothub-nlp-celery"}
-python-decouple = "*"
-bothub-nlp = {path = "./../bothub-nlp"}
+requests = "==2.20.1"
+bothub_backend = {ref = "1.0.2",git = "https://github.com/Ilhasoft/bothub-backend"}
+celery = "==4.3.0"
+gunicorn = "*"
+gevent = "*"
+bothub_nlp_celery = {ref = "0.1.5",git = "https://github.com/Ilhasoft/bothub-nlp-celery"}
+django-environ = "*"
+redis = "*"
+kombu = "==4.5.0"
+fastapi = "==0.42.0"
+uvicorn = "*"
+python-multipart = "*"
+email-validator = "*"
+dataclasses = "==0.6"
 
 [dev-packages]
-redis = "*"
 "psycopg2-binary" = "*"
 "flake8" = "*"
+black = "==19.3b0"
 
 [requires]
 python_version = "3.6"