Merge remote-tracking branch 'origin/develop' for release 0.12.0

.github/workflows/docker-image-php-fpm.yml

@@ -16,10 +16,6 @@ jobs:
     strategy:
       matrix:
         php_version: ["7.2", "7.3", "7.4", "8.0"]
-        include:
-          # IonCube and Source Guardian loaders do not yet exist for PHP 8.0
-          - php_version: "8.0"
-            php_variant: "fpm"
     steps:
     - uses: actions/checkout@v1
     - run: ./images/scripts/build.sh --push "${BUILD_GROUP}"

CHANGELOG.md

@@ -1,7 +1,23 @@
 # Change Log
 
 ## UNRELEASED [x.y.z](https://github.com/davidalger/warden/tree/x.y.z) (yyyy-mm-dd)
-[All Commits](https://github.com/davidalger/warden/compare/0.11.0..develop)
+[All Commits](https://github.com/davidalger/warden/compare/0.12.0..develop)
+
+## Version [0.12.0](https://github.com/davidalger/warden/tree/0.12.0) (2021-08-28)
+[All Commits](https://github.com/davidalger/warden/compare/0.11.0..0.12.0)
+
+**Enhancements:**
+
+* Added Elastic HQ support available at elastichq.mydomain.test for viewing Elasticsearch data ([#350](https://github.com/davidalger/warden/pull/350) by @Den4ik)
+* Update selenium to standalone latest version resolving issues with old hub/chrome image combinations ([#349](https://github.com/davidalger/warden/pull/349) by @Den4ik)
+* Environment and image build improvements ([#363](https://github.com/davidalger/warden/pull/363) by @Den4ik)
+* Introduces new redis sub-command for easy access to the running redis container ([#413](https://github.com/davidalger/warden/pull/413) by @werfu)
+* SSH known hosts will now be persisted via an additional `sshdirectory:/home/www-data/.ssh` volume on php-fpm containers ([#315](https://github.com/davidalger/warden/pull/315) by @ihor-sviziev)
+* Docker images for Nginx 1.18 are now available ([#414](https://github.com/davidalger/warden/pull/414) by @darrakurt)
+
+**Bug Fixes:**
+
+* Fixed loadEnvConfig inability to parse .env with CRLF line endings (issue [#380](https://github.com/davidalger/warden/issues/380))
 
 ## Version [0.11.0](https://github.com/davidalger/warden/tree/0.11.0) (2021-04-22)
 [All Commits](https://github.com/davidalger/warden/compare/0.10.2..0.11.0)
@@ -25,7 +41,7 @@ Huge shout out to @tdgroot who helped with updating the builds, testing, etc.
 ## Version [0.10.2](https://github.com/davidalger/warden/tree/0.10.2) (2021-04-03)
 [All Commits](https://github.com/davidalger/warden/compare/0.10.1..0.10.2)
 
-**Big Fixes:**
+**Bug Fixes:**
 
 * Fixed bug where Live Reload might not work when Varnish was disabled due to routing priorities ([#337](https://github.com/davidalger/warden/pull/337) by @alinalexandru)
 
@@ -135,7 +151,7 @@ Please note that you must pull the latest `php-fpm` images via `warden env pull`
 * Updated volume declarations for RabbitMQ and Redis services to use named volumes (avoid use of anonymous volumes)
 * Updated version of Traefik from 2.1 to 2.2
 * Updated `warden debug` to also pass `host.docker.internal` into the `php-debug` container for the `XDEBUG_REMOTE_HOST` value on WSL when Microsoft is present in `/proc/sys/kernel/osrelease` ([#196](https://github.com/davidalger/warden/pull/196) by @LeeSaferite)
-* Updated nginx configuration to pass `XDEBUG_CONNECT_BACK_HOST` as environment variable in base config allowing it to be overriden by exported env variable on all host OS envs ([#199](https://github.com/davidalger/warden/pull/199) by @LeeSaferite)
+* Updated nginx configuration to pass `XDEBUG_CONNECT_BACK_HOST` as environment variable in base config allowing it to be overridden by exported env variable on all host OS envs ([#199](https://github.com/davidalger/warden/pull/199) by @LeeSaferite)
 
 ## Version [0.7.0](https://github.com/davidalger/warden/tree/0.7.0) (2020-07-22)
 [All Commits](https://github.com/davidalger/warden/compare/0.6.0..0.7.0)
@@ -275,7 +291,7 @@ The `BYPASS_VARNISH` flag will continue to work as before but has been **depreca
 
 **Enhancements:**
 
-* Updated init routine allowing `WARDEN_HOME_DIR` and `WARDEN_COMPOSER_DIR` to be overriden via environment variables
+* Updated init routine allowing `WARDEN_HOME_DIR` and `WARDEN_COMPOSER_DIR` to be overridden via environment variables
 * Updated environment configuration to reference `WARDEN_SSL_DIR` eliminating hard-coded `~/.warden/ssl` references
 * Updated warden global docker config to reference `WARDEN_HOME_DIR` eliminating hard-coded `~/.warden` references
 * Updated `warden up` to return an error when docker is not running rather than blindly attempt to start global services

bin/warden

@@ -22,7 +22,7 @@ fi
 
 ## verify docker-compose meets version constraint
 DOCKER_COMPOSE_REQUIRE="1.25.0"
-DOCKER_COMPOSE_VERSION="$(docker-compose -v | grep -oE '[0-9\.]+' | head -n1)"
+DOCKER_COMPOSE_VERSION="$(docker-compose --version | grep -oE '[0-9\.]+' | head -n1)"
 if ! test $(version ${DOCKER_COMPOSE_VERSION}) -ge $(version ${DOCKER_COMPOSE_REQUIRE}); then
   fatal "docker-compose version should be ${DOCKER_COMPOSE_REQUIRE} or higher (${DOCKER_COMPOSE_VERSION} installed)"
 fi
@@ -38,7 +38,7 @@ declare WARDEN_PARAMS=()
 declare WARDEN_CMD_VERB=
 declare WARDEN_CMD_EXEC=
 declare WARDEN_CMD_HELP=
-declare WARDEN_CMD_ANYARGS=(svc env db sync shell debug)
+declare WARDEN_CMD_ANYARGS=(svc env db redis sync shell debug)
 
 ## parse first argument as command and determine validity
 if (( "$#" )); then

commands/blackfire.cmd

@@ -4,7 +4,7 @@
 WARDEN_ENV_PATH="$(locateEnvPath)" || exit $?
 loadEnvConfig "${WARDEN_ENV_PATH}" || exit $?
 
-## set defaults for this command which can be overriden either using exports in the user
+## set defaults for this command which can be overridden either using exports in the user
 ## profile or setting them in the .env configuration on a per-project basis
 WARDEN_ENV_BLACKFIRE_COMMAND=${WARDEN_ENV_BLACKFIRE_COMMAND:-blackfire}
 WARDEN_ENV_BLACKFIRE_CONTAINER=${WARDEN_ENV_BLACKFIRE_CONTAINER:-php-blackfire}

commands/debug.cmd

@@ -4,7 +4,7 @@
 WARDEN_ENV_PATH="$(locateEnvPath)" || exit $?
 loadEnvConfig "${WARDEN_ENV_PATH}" || exit $?
 
-## set defaults for this command which can be overriden either using exports in the user
+## set defaults for this command which can be overridden either using exports in the user
 ## profile or setting them in the .env configuration on a per-project basis
 WARDEN_ENV_DEBUG_COMMAND=${WARDEN_ENV_DEBUG_COMMAND:-bash}
 WARDEN_ENV_DEBUG_CONTAINER=${WARDEN_ENV_DEBUG_CONTAINER:-php-debug}

commands/env.cmd

@@ -12,6 +12,12 @@ fi
 ## allow return codes from sub-process to bubble up normally
 trap '' ERR
 
+## define source repository
+if [[ -f "${WARDEN_HOME_DIR}/.env" ]]; then
+  eval "$(cat "${WARDEN_HOME_DIR}/.env" | sed 's/\r$//g' | grep "^WARDEN_")"
+fi
+export WARDEN_IMAGE_REPOSITORY="${WARDEN_IMAGE_REPOSITORY:-"docker.io/wardenenv"}"
+
 ## configure environment type defaults
 if [[ ${WARDEN_ENV_TYPE} =~ ^magento ]]; then
     export WARDEN_SVC_PHP_VARIANT=-${WARDEN_ENV_TYPE}
@@ -29,7 +35,7 @@ if [[ ${WARDEN_ENV_TYPE} != local ]]; then
     WARDEN_REDIS=${WARDEN_REDIS:-1}
 
     # define bash history folder for changing permissions
-    WARDEN_CHOWN_DIR_LIST="/bash_history ${WARDEN_CHOWN_DIR_LIST:-}"
+    WARDEN_CHOWN_DIR_LIST="/bash_history /home/www-data/.ssh ${WARDEN_CHOWN_DIR_LIST:-}"
 fi
 export CHOWN_DIR_LIST=${WARDEN_CHOWN_DIR_LIST:-}
 

commands/redis.cmd

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+[[ ! ${WARDEN_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1
+
+WARDEN_ENV_PATH="$(locateEnvPath)" || exit $?
+loadEnvConfig "${WARDEN_ENV_PATH}" || exit $?
+assertDockerRunning
+
+if [[ ${WARDEN_REDIS:-1} -eq 0 ]]; then
+  fatal "Redis environment is not used (WARDEN_REDIS=0)."
+fi
+
+if [[ "${WARDEN_PARAMS[0]}" == "help" ]]; then
+  warden redis --help || exit $? && exit $?
+fi
+
+## load connection information for the redis service
+REDIS_CONTAINER=$(warden env ps -q redis)
+if [[ ! ${REDIS_CONTAINER} ]]; then
+    fatal "No container found for redis service."
+fi
+
+"${WARDEN_DIR}/bin/warden" env exec redis redis-cli "${WARDEN_PARAMS[@]}" "$@"

commands/redis.help

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+[[ ! ${WARDEN_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1
+
+WARDEN_USAGE=$(cat <<EOF
+\033[33mUsage:\033[0m
+  redis                     Launches an interactive redis session within the current project environment
+  redis COMMAND             Execute any valid Redis command within the current project environment
+
+\033[33mOptions:\033[0m
+  -h, --help        Display this help menu
+EOF
+)

commands/shell.cmd

@@ -4,7 +4,7 @@
 WARDEN_ENV_PATH="$(locateEnvPath)" || exit $?
 loadEnvConfig "${WARDEN_ENV_PATH}" || exit $?
 
-## set defaults for this command which can be overriden either using exports in the user
+## set defaults for this command which can be overridden either using exports in the user
 ## profile or setting them in the .env configuration on a per-project basis
 WARDEN_ENV_SHELL_COMMAND=${WARDEN_ENV_SHELL_COMMAND:-bash}
 WARDEN_ENV_SHELL_CONTAINER=${WARDEN_ENV_SHELL_CONTAINER:-php-fpm}

commands/usage.help

@@ -24,6 +24,7 @@ Warden version $(cat ${WARDEN_DIR}/version)
   env-init          Configure environment by adding '.env' file to the current working directory
   env               Controls an environment from any point within the root project directory
   db                Interacts with the db service on an environment (see 'warden db -h' for details)
+  redis             Interacts with the redis service on an environment (see 'warden redis -h' for details)
   install           Initializes or updates warden configuration on host machine
   shell             Launches into a shell within the current project environment
   debug             Launches debug enabled shell within current project environment

commands/vnc.cmd

@@ -9,7 +9,7 @@ if [[ ${WARDEN_SELENIUM} -ne 1 ]] || [[ ${WARDEN_SELENIUM_DEBUG} -ne 1 ]]; then
 fi
 
 WARDEN_SELENIUM_INDEX=${WARDEN_PARAMS[0]:-1}
-WARDEN_SELENIUM_VNC=${WARDEN_ENV_NAME}_${WARDEN_PARAMS[1]:-selenium-chrome}_${WARDEN_SELENIUM_INDEX}
+WARDEN_SELENIUM_VNC=${WARDEN_ENV_NAME}_${WARDEN_PARAMS[1]:-selenium}_${WARDEN_SELENIUM_INDEX}
 
 if ! which remmina >/dev/null; then
   EXPOSE_PORT=$((5900 + WARDEN_SELENIUM_INDEX))

docs/configuration/magento2-testing.md

@@ -298,7 +298,7 @@ To run Acceptance tests you need to [configure the MFTF environment](mftf.md). O
     - `MAGENTO_BASE_URL`
     - `MAGENTO_BACKEND_NAME` to your Backend path (Check with `bin/magento info:adminuri`)
     - `MAGENTO_ADMIN_USERNAME` and `MAGENTO_ADMIN_PASSWORD`
-    - `SELENIUM_HOST` (by default it is `selenium-hub`)
+    - `SELENIUM_HOST` (by default it is `selenium`)
 
    Sample configuration
    ```
@@ -309,7 +309,7 @@ To run Acceptance tests you need to [configure the MFTF environment](mftf.md). O
    BROWSER=chrome
    MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProductCatalogSearch
    ELASTICSEARCH_VERSION=7
-   SELENIUM_HOST=selenium-hub
+   SELENIUM_HOST=selenium
    ```
    More details can be found [in Magento DevDocs](https://devdocs.magento.com/mftf/docs/configuration.html).
 

docs/configuration/mftf.md

@@ -11,7 +11,7 @@ WARDEN_SELENIUM=1
 After generating MFTF configuration files (`dev/tests/acceptance/.env` generated by `vendor/bin/mftf setup:env` command), you need to provide selenium hostname:
 
 ```
-SELENIUM_HOST=selenium-hub
+SELENIUM_HOST=selenium
 BROWSER=chrome
 ```
 
@@ -43,7 +43,7 @@ To preview the process of testing, you need any **VLC** client that provides **S
 
 To preview the process in Mac OS X, you must first create an SSH tunnel to the docker instance hosting the VNC server.  That would look something like:
 
-    ssh -N -L localhost:5901:magento2_selenium-chrome_1:5900 tunnel.warden.test
+    ssh -N -L localhost:5901:magento2_selenium_1:5900 tunnel.warden.test
 
 Where `5901` is the port on your local computer you want to use to access the VNC server.  Then, using Finder you can "Go > Connect to Server" `vnc://localhost:5901`.
 

docs/environments/customizing.md

@@ -58,6 +58,17 @@ post_max_size = 25M
 session.auto_start = Off
 upload_max_filesize = 25M
 ```
+### Nginx Specific Customizations
+To override the default nginx configuration of your project, add a new file 
+`.warden/warden-env.yml` to your project root with the following content:
+```
+version: "3.5"
+services:
+  nginx:
+    volumes:
+      - ./.warden/nginx/custom.conf:/etc/nginx/default.d/custom.conf
+```
+There you can specify a custom Nginx configuration which will be included following the `.conf` files within the `/etc/nginx/available.d` directory: `include /etc/nginx/default.d/*.conf`
 
 ### Magento 1 Specific Customizations
 

docs/usage.md

@@ -36,11 +36,15 @@ Flush varnish:
 
 Connect to redis:
 
-    warden env exec redis redis-cli
+    warden redis
 
 Flush redis completely:
 
-    warden env exec -T redis redis-cli flushall
+    warden redis flushall
+
+Run redis continous stat mode
+
+    warden redis --stat
 
 Remove volumes completely:
 

environments/includes/blackfire.base.yml

@@ -12,7 +12,7 @@ x-extra_hosts: &extra_hosts
 services:
   php-blackfire:
     hostname: "${WARDEN_ENV_NAME}-php-blackfire"
-    image: docker.io/wardenenv/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}-blackfire
+    image: ${WARDEN_IMAGE_REPOSITORY}/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}-blackfire
     environment:
       - TRAEFIK_DOMAIN
       - TRAEFIK_SUBDOMAIN

environments/includes/db.base.yml

@@ -6,7 +6,7 @@ services:
 
   db:
     hostname: "${WARDEN_ENV_NAME}-mariadb"
-    image: docker.io/wardenenv/mariadb:${MARIADB_VERSION:-10.4}
+    image: ${WARDEN_IMAGE_REPOSITORY}/mariadb:${MARIADB_VERSION:-10.4}
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-app}
       - MYSQL_DATABASE=${MYSQL_DATABASE:-app}

environments/includes/elasticsearch.base.yml

@@ -2,7 +2,7 @@ version: "3.5"
 services:
   elasticsearch:
     hostname: "${WARDEN_ENV_NAME}-elasticsearch"
-    image: docker.io/wardenenv/elasticsearch:${ELASTICSEARCH_VERSION:-7.8}
+    image: ${WARDEN_IMAGE_REPOSITORY}/elasticsearch:${ELASTICSEARCH_VERSION:-7.8}
     labels:
       - traefik.enable=true
       - traefik.http.routers.${WARDEN_ENV_NAME}-elasticsearch.tls=true
@@ -15,5 +15,14 @@ services:
     volumes:
       - esdata:/usr/share/elasticsearch/data
 
+  elasticsearch-hq:
+    hostname: "${WARDEN_ENV_NAME}-elasticsearch-hq"
+    image: elastichq/elasticsearch-hq:latest
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.${WARDEN_ENV_NAME}-elasticsearch-hq.tls=true
+      - traefik.http.routers.${WARDEN_ENV_NAME}-elasticsearch-hq.rule=Host(`elastichq.${TRAEFIK_DOMAIN}`)
+      - traefik.http.services.${WARDEN_ENV_NAME}-elasticsearch-hq.loadbalancer.server.port=5000
+
 volumes:
   esdata:

environments/includes/nginx.base.yml

@@ -2,7 +2,7 @@ version: "3.5"
 services:
   nginx:
     hostname: "${WARDEN_ENV_NAME}-nginx"
-    image: docker.io/wardenenv/nginx:${NGINX_VERSION:-1.16}
+    image: ${WARDEN_IMAGE_REPOSITORY}/nginx:${NGINX_VERSION:-1.16}
     labels:
       - traefik.enable=true
       - traefik.http.routers.${WARDEN_ENV_NAME}-nginx.tls=true

environments/includes/php-fpm.base.yml

@@ -5,6 +5,7 @@ x-volumes: &volumes
   - ${WARDEN_COMPOSER_DIR}:/home/www-data/.composer:cached
   - .${WARDEN_WEB_ROOT:-}/:/var/www/html:cached
   - bashhistory:/bash_history
+  - sshdirectory:/home/www-data/.ssh
 
 x-extra_hosts: &extra_hosts
     - ${TRAEFIK_DOMAIN}:${TRAEFIK_ADDRESS:-0.0.0.0}
@@ -17,7 +18,7 @@ services:
 
   php-fpm:
     hostname: "${WARDEN_ENV_NAME}-php-fpm"
-    image: docker.io/wardenenv/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}
+    image: ${WARDEN_IMAGE_REPOSITORY}/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}
     environment:
       - TRAEFIK_DOMAIN
       - TRAEFIK_SUBDOMAIN
@@ -32,7 +33,7 @@ services:
 
   php-debug:
     hostname: "${WARDEN_ENV_NAME}-php-debug"
-    image: docker.io/wardenenv/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}-${XDEBUG_VERSION:-debug}
+    image: ${WARDEN_IMAGE_REPOSITORY}/php-fpm:${PHP_VERSION:-7.4}${WARDEN_SVC_PHP_VARIANT:-}-${XDEBUG_VERSION:-debug}
     environment:
       - TRAEFIK_DOMAIN
       - TRAEFIK_SUBDOMAIN
@@ -49,3 +50,4 @@ services:
       - php-fpm
 volumes:
   bashhistory:
+  sshdirectory:

environments/includes/rabbitmq.base.yml

@@ -2,7 +2,7 @@ version: "3.5"
 services:
   rabbitmq:
     hostname: "${WARDEN_ENV_NAME}-rabbitmq"
-    image: docker.io/wardenenv/rabbitmq:${RABBITMQ_VERSION:-3.8}
+    image: ${WARDEN_IMAGE_REPOSITORY}/rabbitmq:${RABBITMQ_VERSION:-3.8}
     labels:
       - traefik.enable=true
       - traefik.http.routers.${WARDEN_ENV_NAME}-rabbitmq.tls=true

environments/includes/redis.base.yml

@@ -2,7 +2,7 @@ version: "3.5"
 services:
   redis:
     hostname: "${WARDEN_ENV_NAME}-redis"
-    image: docker.io/wardenenv/redis:${REDIS_VERSION:-5.0}
+    image: ${WARDEN_IMAGE_REPOSITORY}/redis:${REDIS_VERSION:-5.0}
     volumes:
       - redis:/data
 

environments/includes/selenium.base.yml

@@ -1,27 +1,8 @@
 version: "3.5"
 services:
-  selenium-hub:
-    container_name: ${WARDEN_ENV_NAME}_selenium-hub
-    hostname: selenium-hub
-    image: selenium/hub:latest
-    healthcheck:
-      test: /opt/bin/check-grid.sh --host 0.0.0.0 --port 4444
-      interval: 20s
-      retries: 3
-    environment:
-      GRID_TIMEOUT: 0
-      GRID_BROWSER_TIMEOUT: 0
-
-  selenium-chrome:
-    hostname: ${WARDEN_ENV_NAME}_selenium-chrome
-    image: selenium/node-chrome${WARDEN_SELENIUM_DEBUG:-}:3.8.1
+  selenium:
+    hostname: ${WARDEN_ENV_NAME}_selenium
+    image: selenium/standalone-chrome${WARDEN_SELENIUM_DEBUG:-}:${WARDEN_SELENIUM_VERSION:-3.141.59}
     extra_hosts:
       - ${TRAEFIK_DOMAIN}:${TRAEFIK_ADDRESS:-0.0.0.0}
       - ${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}:${TRAEFIK_ADDRESS:-0.0.0.0}
-    depends_on:
-      - selenium-hub
-    environment:
-      - HUB_HOST=selenium-hub
-      - HUB_PORT=4444
-      - START_XVFB=false
-

environments/includes/varnish.base.yml

@@ -7,7 +7,7 @@ services:
 
   varnish:
     hostname: "${WARDEN_ENV_NAME}-varnish"
-    image: docker.io/wardenenv/varnish:${VARNISH_VERSION:-6.0}
+    image: ${WARDEN_IMAGE_REPOSITORY}/varnish:${VARNISH_VERSION:-6.0}
     depends_on:
       - nginx
     labels: