diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 318315b81..d374420cf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,80 +3,148 @@ # SPDX-FileCopyrightText: 2021 Sebastian Waldbauer # SPDX-License-Identifier: AGPL-3.0-or-later # -name: "Build and upload docker image" +#name: "Build and upload docker image" +# +#on: +# push: +# branches: ['develop'] +# paths-ignore: +# - '.github/**' +# +#jobs: +# docker: +# name: Building Docker image +# runs-on: ubuntu-latest +# +# steps: +# - name: Check if environment is ACT +# run: sudo chown runner:docker /var/run/docker.sock +# if: ${{ env.ACT }} +# +# - name: Checkout IntelMQ-Docker +# uses: actions/checkout@v3 +# with: +# ref: 'main' +# repository: certat/intelmq-docker +# path: ./work +# +# - name: Checkout IntelMQ-Manager +# uses: actions/checkout@v3 +# with: +# ref: 'develop' +# repository: certtools/intelmq-manager +# path: ./work/intelmq-manager +# +# - name: Checkout IntelMQ-API +# uses: actions/checkout@v3 +# with: +# ref: 'develop' +# repository: certtools/intelmq-api +# path: ./work/intelmq-api +# +# - name: Checkout IntelMQ +# uses: actions/checkout@v3 +# with: +# ref: 'develop' +# path: ./work/intelmq +# +# - name: Setup QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Setup Docker Buildx +# uses: docker/setup-buildx-action@v2 +# +# - name: Login to DockerHub +# uses: docker/login-action@v2 +# if: github.repository == 'certtools/intelmq' && !${{ env.ACT }} +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} +# +# - name: Install python build requirements +# run: | +# pip3 install mako +# +# - name: Build +# run: | +# cd work +# ./build.sh +# +# - name: Test +# run: | +# cd work +# ./test.sh +# +# - name: Publish develop version to dockerhub +# if: github.repository == 'certtools/intelmq' && !${{ env.ACT }} +# run: | +# docker tag intelmq-full:latest certat/intelmq-full:develop +# docker push certat/intelmq-full:develop + + +name: "Build and upload Docker image" on: push: - branches: ['develop'] - paths-ignore: - - '.github/**' + branches: + - develop + - merge-projects + tags: + - "*.*.*" + pull_request: + branches: + - develop jobs: docker: - name: Building Docker image runs-on: ubuntu-latest - steps: - - name: Check if environment is ACT - run: sudo chown runner:docker /var/run/docker.sock - if: ${{ env.ACT }} + - name: Checkout repository + uses: actions/checkout@v4 - - name: Checkout IntelMQ-Docker - uses: actions/checkout@v3 - with: - ref: 'main' - repository: certat/intelmq-docker - path: ./work +# - name: Docker meta +# id: meta +# uses: docker/metadata-action@v5 +# with: +# # list of Docker images to use as base name for tags +# images: | +# name/app +# ghcr.io/username/app +# # generate Docker tags based on the following events/attributes +# tags: | +# type=schedule +# type=ref,event=branch +# type=ref,event=pr +# type=semver,pattern={{version}} +# type=semver,pattern={{major}}.{{minor}} +# type=semver,pattern={{major}} +# type=sha - - name: Checkout IntelMQ-Manager - uses: actions/checkout@v3 - with: - ref: 'develop' - repository: certtools/intelmq-manager - path: ./work/intelmq-manager + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - - name: Checkout IntelMQ-API - uses: actions/checkout@v3 - with: - ref: 'develop' - repository: certtools/intelmq-api - path: ./work/intelmq-api + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Checkout IntelMQ - uses: actions/checkout@v3 - with: - ref: 'develop' - path: ./work/intelmq - - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - if: github.repository == 'certtools/intelmq' && !${{ env.ACT }} + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Install python build requirements - run: | - pip3 install mako +# - name: Login to GHCR +# if: github.event_name != 'pull_request' +# uses: docker/login-action@v3 +# with: +# registry: ghcr.io +# username: ${{ github.repository_owner }} +# password: ${{ secrets.GITHUB_TOKEN }} - - name: Build - run: | - cd work - ./build.sh - - - name: Test - run: | - cd work - ./test.sh - - - name: Publish develop version to dockerhub - if: github.repository == 'certtools/intelmq' && !${{ env.ACT }} - run: | - docker tag intelmq-full:latest certat/intelmq-full:develop - docker push certat/intelmq-full:develop + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..2e52078c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +FROM python:3.9-bullseye + +ENV LANG=C.UTF-8 +ENV PATH=/opt/venv/bin:${PATH} +ENV INTELMQ_PATHS_NO_OPT=1 + +ARG DEBIAN_FRONTEND=noninteractive + +# Install dependencides +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends python3 \ + python3-pip \ + git \ + build-essential \ + python3-dev \ + libcurl4-gnutls-dev \ + libgnutls28-dev \ + libffi-dev \ + curl \ + sudo \ + vim \ + jq \ + cron \ + libpq-dev \ + logrotate && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/www/html && \ + python3 -m venv --upgrade-deps /opt/venv + +# Copy IntelMQ complete requirements +COPY requirements.txt /tmp/requirements.txt + +# Install IntelMQ dependencies (before copying the sources -> makes for faster repeated build times) +RUN pip install -r /tmp/requirements.txt + +# Copy IntelMQ sources +COPY . /tmp/intelmq + +# Install IntelMQ +RUN pip3 install /tmp/intelmq + +# Add intelmq user +RUN useradd --system --uid 995 --create-home --home-dir /var/lib/intelmq --shell /bin/bash intelmq + +# Setup IntelMQ and clean +RUN intelmqsetup && \ + rm -rf /tmp/intelmq /tmp/requirements.txt + +# Switch to intelmq user +USER intelmq + +CMD ["intelmq", "server", "start"] \ No newline at end of file diff --git a/docker/runtime.yaml b/docker/runtime.yaml new file mode 100644 index 000000000..294eaeea9 --- /dev/null +++ b/docker/runtime.yaml @@ -0,0 +1,180 @@ +global: + destination_pipeline_broker: redis + destination_pipeline_host: redis + process_manager: intelmq + server: + enable_webgui: true + host: 0.0.0.0 + intelmq_ctl_cmd: [intelmqctl] + port: 8080 + source_pipeline_broker: redis + source_pipeline_host: redis + ssl_ca_certificate: null + statistics_database: 3 + statistics_host: redis + statistics_password: null + statistics_port: 6379 + +cymru-whois-expert: + bot_id: cymru-whois-expert + description: Cymru Whois (IP to ASN) is the bot responsible to add network information + to the events (BGP, ASN, AS Name, Country, etc..). + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.cymru_whois.expert + name: Cymru Whois + parameters: + destination_queues: + _default: [file-output-queue] + overwrite: true + redis_cache_db: 5 + redis_cache_host: redis + redis_cache_password: null + redis_cache_port: 6379 + redis_cache_ttl: 86400 + run_mode: continuous +deduplicator-expert: + bot_id: deduplicator-expert + description: Deduplicator is the bot responsible for detection and removal of duplicate + messages. Messages get cached for seconds. If found in the cache, + it is assumed to be a duplicate. + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.deduplicator.expert + name: Deduplicator + parameters: + destination_queues: + _default: [taxonomy-expert-queue] + filter_keys: raw,time.observation + filter_type: blacklist + redis_cache_db: 6 + redis_cache_host: redis + redis_cache_port: 6379 + redis_cache_ttl: 86400 + run_mode: continuous +feodo-tracker-collector: + description: Generic URL Fetcher is the bot responsible to get the report from an + URL. + enabled: true + group: Collector + module: intelmq.bots.collectors.http.collector_http + name: URL Fetcher + parameters: + destination_queues: + _default: [feodo-tracker-parser-queue] + extract_files: false + http_password: null + http_url: https://feodotracker.abuse.ch/downloads/ipblocklist.json + http_url_formatting: false + http_username: null + name: Feodo Tracker + provider: Abuse.ch + rate_limit: 86400 + ssl_client_certificate: null + run_mode: continuous +feodo-tracker-parser: + description: Parser for Feodo Tracker collector. + enabled: true + group: Parser + module: intelmq.bots.parsers.abusech.parser_feodotracker + name: Feodo Tracker Parser + parameters: + destination_queues: + _default: [deduplicator-expert-queue] + run_mode: continuous +file-output: + bot_id: file-output + description: File is the bot responsible to send events to a file. + enabled: true + group: Output + groupname: outputs + module: intelmq.bots.outputs.file.output + name: File + parameters: {file: /var/lib/intelmq/bots/file-output/events.txt, hierarchical_output: false, + single_key: null} + run_mode: continuous +gethostbyname-1-expert: + bot_id: gethostbyname-1-expert + description: fqdn2ip is the bot responsible to parsing the ip from the fqdn. + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.gethostbyname.expert + name: Gethostbyname + parameters: + destination_queues: + _default: [cymru-whois-expert-queue] + run_mode: continuous +gethostbyname-2-expert: + bot_id: gethostbyname-2-expert + description: fqdn2ip is the bot responsible to parsing the ip from the fqdn. + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.gethostbyname.expert + name: Gethostbyname + parameters: + destination_queues: + _default: [cymru-whois-expert-queue] + run_mode: continuous +spamhaus-drop-collector: + bot_id: spamhaus-drop-collector + description: '' + enabled: true + group: Collector + groupname: collectors + module: intelmq.bots.collectors.http.collector_http + name: Spamhaus Drop + parameters: + destination_queues: + _default: [spamhaus-drop-parser-queue] + http_password: null + http_url: https://www.spamhaus.org/drop/drop.txt + http_username: null + name: Drop + provider: Spamhaus + rate_limit: 3600 + ssl_client_certificate: null + run_mode: continuous +spamhaus-drop-parser: + bot_id: spamhaus-drop-parser + description: Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP, + DROPv6, and ASN-DROP reports and sanitize the information. + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.spamhaus.parser_drop + name: Spamhaus Drop + parameters: + destination_queues: + _default: [deduplicator-expert-queue] + run_mode: continuous +taxonomy-expert: + bot_id: taxonomy-expert + description: Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all + events. + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.taxonomy.expert + name: Taxonomy + parameters: + destination_queues: + _default: [url-expert-queue] + run_mode: continuous +url-expert: + bot_id: url-expert + description: Extract additional information for the URL + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.url.expert + name: url + parameters: + destination_queues: + _default: [gethostbyname-1-expert-queue, gethostbyname-2-expert-queue] + load_balance: true + overwrite: false + run_mode: continuous