diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 9e6ff9cb764..7a583c6c249 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -39,7 +39,9 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/node/setup + - uses: actions/setup-node@v3 + with: + node-version: '18' - run: yarn install - run: yarn test:appsec:ci - uses: codecov/codecov-action@v3 @@ -207,9 +209,11 @@ jobs: version: - 18 - latest + range: ['9.5.0', '11.1.4', '13.2.0', '*'] runs-on: ubuntu-latest env: PLUGINS: next + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml new file mode 100644 index 00000000000..d392f617b9b --- /dev/null +++ b/.github/workflows/datadog-static-analysis.yml @@ -0,0 +1,24 @@ +name: Datadog Static Analysis + +on: + pull_request: + push: + branches: [master] + +jobs: + static-analysis: + runs-on: ubuntu-latest + name: Datadog Static Analyzer + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Check code meets quality and security standards + id: datadog-static-analysis + uses: DataDog/datadog-static-analyzer-github-action@v1 + with: + dd_api_key: ${{ secrets.DD_STATIC_ANALYSIS_API_KEY }} + dd_app_key: ${{ secrets.DD_STATIC_ANALYSIS_APP_KEY }} + dd_service: dd-trace-js + dd_env: ci + dd_site: datadoghq.com + cpu_count: 2 diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index a2e3d02c50b..0f31bb687e0 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -15,60 +15,6 @@ concurrency: jobs: - aerospike-node-14: - runs-on: ubuntu-latest - container: - image: ubuntu:18.04 - services: - aerospike: - image: aerospike:ce-5.3.0.16 - ports: - - 3000:3000 - testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 - env: - LOG_LEVEL: DEBUG - TRACE_LANGUAGE: javascript - ENABLED_CHECKS: trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service - PORT: 9126 - ports: - - 9126:9126 - env: - PLUGINS: aerospike - SERVICES: aerospike - PACKAGE_VERSION_RANGE: '3.16.2 - 3.16.7' - DD_TEST_AGENT_URL: http://testagent:9126 - AEROSPIKE_HOST_ADDRESS: aerospike - steps: - # Needs to remain on v3 for now due to GLIBC version - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '14' - - id: pkg - run: | - content=`cat ./package.json | tr '\n' ' '` - echo "json=$content" >> $GITHUB_OUTPUT - - id: extract - run: | - version="${{fromJson(steps.pkg.outputs.json).version}}" - majorVersion=$(echo "$version" | cut -d '.' -f 1) - echo "Major Version: $majorVersion" - echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - - name: Install dependencies and run tests - if: env.MAJOR_VERSION == '3' - run: | - apt-get update && \ - apt-get install -y \ - python3 python3-pip \ - wget \ - g++ libssl1.0.0 libssl-dev zlib1g-dev && \ - npm install -g yarn - yarn install --ignore-engines - yarn test:plugins:ci - - if: always() - uses: codecov/codecov-action@v2 - aerospike-node-16: runs-on: ubuntu-latest services: @@ -222,6 +168,9 @@ jobs: - uses: codecov/codecov-action@v3 aws-sdk: + strategy: + matrix: + node-version: ['18', 'latest'] runs-on: ubuntu-latest services: localstack: @@ -262,9 +211,9 @@ jobs: - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install - - uses: ./.github/actions/node/oldest - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs @@ -327,9 +276,7 @@ jobs: runs-on: ubuntu-latest services: cassandra: - image: spotify/cassandra - env: - CASSANDRA_TOKEN: '-9223372036854775808' + image: cassandra:3-focal ports: - 9042:9042 env: @@ -365,6 +312,9 @@ jobs: - uses: codecov/codecov-action@v2 couchbase: + strategy: + matrix: + range: ['^2.6.12', '^3.0.7', '>=4.2.0'] runs-on: ubuntu-latest services: couchbase: @@ -375,6 +325,7 @@ jobs: env: PLUGINS: couchbase SERVICES: couchbase + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start @@ -458,7 +409,7 @@ jobs: runs-on: ubuntu-latest services: elasticsearch: - image: elasticsearch:7.14.0 + image: elasticsearch:7.17.22 env: discovery.type: single-node ports: @@ -622,6 +573,9 @@ jobs: - uses: codecov/codecov-action@v3 http: + strategy: + matrix: + node-version: ['18', '20', 'latest'] runs-on: ubuntu-latest env: PLUGINS: http @@ -630,11 +584,9 @@ jobs: - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install - - uses: ./.github/actions/node/18 - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/20 - - run: yarn test:plugins:ci - - uses: ./.github/actions/node/latest + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs @@ -940,9 +892,11 @@ jobs: version: - 18 - latest + range: ['9.5.0', '11.1.4', '13.2.0', '*'] runs-on: ubuntu-latest env: PLUGINS: next + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start @@ -1022,6 +976,8 @@ jobs: PLUGINS: oracledb SERVICES: oracledb DD_TEST_AGENT_URL: http://testagent:9126 + # Needed to fix issue with `actions/checkout@v3: https://github.com/actions/checkout/issues/1590 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 1f54c5a4c51..1de2caf9697 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -43,7 +43,9 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/node/setup + - uses: actions/setup-node@v3 + with: + node-version: '18' - run: yarn install - run: yarn test:profiler:ci - run: yarn test:integration:profiler diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index 258d827bdf2..d0279ea611b 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -30,11 +30,25 @@ jobs: - run: sudo sysctl -w kernel.core_pattern='|/bin/false' - run: yarn test:integration + # We'll run these separately for earlier (i.e. unsupported) versions + integration-guardrails: + strategy: + matrix: + version: [12, 14, 16] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.version }} + - run: yarn install --ignore-engines + - run: node node_modules/.bin/mocha --colors --timeout 30000 -r packages/dd-trace/test/setup/core.js integration-tests/init.spec.js + integration-ci: strategy: matrix: version: [18, latest] - framework: [cucumber, playwright, selenium] + framework: [cucumber, playwright, selenium, jest, mocha] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests @@ -78,6 +92,7 @@ jobs: version: [16, latest] # 6.7.0 is the minimum version we support cypress-version: [6.7.0, latest] + module-type: ['commonJS', 'esm'] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests @@ -95,6 +110,24 @@ jobs: env: CYPRESS_VERSION: ${{ matrix.cypress-version }} NODE_OPTIONS: '-r ./ci/init' + CYPRESS_MODULE_TYPE: ${{ matrix.module-type }} + + integration-vitest: + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - run: yarn install + - uses: actions/setup-node@v3 + with: + node-version: 20 + - run: yarn test:integration:vitest + env: + NODE_OPTIONS: '-r ./ci/init' lint: runs-on: ubuntu-latest diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index fc00326a27f..96b8ab4fc7c 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -1,12 +1,14 @@ name: Release dev release line on: - push: + pull_request_target: + types: [labeled] branches: - master jobs: dev_release: + if: ${{ github.event.label.name == 'release-dev' }} runs-on: ubuntu-latest environment: npm permissions: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 4a72cdcdb15..03b2fe21bf6 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,6 +11,19 @@ on: - cron: '00 04 * * 2-6' jobs: + build-artifacts: + runs-on: ubuntu-latest + steps: + - name: Checkout dd-trace-js + uses: actions/checkout@v4 + with: + path: dd-trace-js + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: system_tests_binaries + path: . + get-essential-scenarios: runs-on: ubuntu-latest outputs: @@ -71,31 +84,12 @@ jobs: path: artifact.tar.gz parametric: - runs-on: ubuntu-latest - env: - TEST_LIBRARY: nodejs - steps: - - name: Checkout system tests - uses: actions/checkout@v4 - with: - repository: 'DataDog/system-tests' - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Checkout dd-trace-js - uses: actions/checkout@v4 - with: - path: 'binaries/dd-trace-js' - - name: Build - run: ./build.sh -i runner - - name: Run - run: ./run.sh PARAMETRIC - - name: Compress artifact - if: ${{ always() }} - run: tar -czvf artifact.tar.gz $(ls | grep logs) - - name: Upload artifact - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: logs_parametric - path: artifact.tar.gz + needs: + - build-artifacts + uses: DataDog/system-tests/.github/workflows/run-parametric.yml@main + secrets: inherit + with: + library: nodejs + binaries_artifact: system_tests_binaries + _experimental_job_count: 8 + _experimental_job_matrix: '[1,2,3,4,5,6,7,8]' diff --git a/.github/workflows/tracing.yml b/.github/workflows/tracing.yml index cd54e25eb7e..5b76f66fad8 100644 --- a/.github/workflows/tracing.yml +++ b/.github/workflows/tracing.yml @@ -39,7 +39,10 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/node/setup + - uses: actions/setup-node@v3 + with: + cache: yarn + node-version: '18' - run: yarn install - run: yarn test:trace:core:ci - uses: codecov/codecov-action@v3 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10bfe08f0a8..d46f8284179 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,11 +5,13 @@ stages: - benchmarks-pr-comment - single-step-instrumentation-tests - manual_images + - macrobenchmarks include: - remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml - local: ".gitlab/benchmarks.yml" - local: ".gitlab/single-step-instrumentation-tests.yml" + - local: ".gitlab/macrobenchmarks.yml" variables: diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 45c3fdc99a3..2a0e61f24c4 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,5 +1,5 @@ variables: - BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-js + MICROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-js SLS_CI_IMAGE: registry.ddbuild.io/ci/serverless-tools:1 # Benchmark's env variables. Modify to tweak benchmark parameters. @@ -10,7 +10,7 @@ variables: stage: benchmarks when: on_success tags: ["runner:apm-k8s-tweaked-metal"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE interruptible: true timeout: 15m script: @@ -31,7 +31,7 @@ benchmarks-pr-comment: stage: benchmarks-pr-comment when: on_success tags: ["arch:amd64"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE script: - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-js) - bp-runner bp-runner.pr-comment.yml --debug @@ -44,7 +44,7 @@ check-big-regressions: stage: benchmarks-pr-comment when: on_success tags: ["arch:amd64"] - image: $BASE_CI_IMAGE + image: $MICROBENCHMARKS_CI_IMAGE script: - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-js) - bp-runner bp-runner.fail-on-regression.yml --debug diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml new file mode 100644 index 00000000000..4392babe28b --- /dev/null +++ b/.gitlab/macrobenchmarks.yml @@ -0,0 +1,67 @@ +.macrobenchmarks: + stage: macrobenchmarks + rules: + - if: ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_REF_NAME == "master" + when: always + - when: manual + tags: ["runner:apm-k8s-same-cpu"] + needs: [] + interruptible: true + timeout: 1h + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:js-hapi + script: + - git clone --branch js/hapi https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform + - bp-runner bp-runner.yml --debug -t + artifacts: + name: "artifacts" + when: always + paths: + - platform/artifacts/ + expire_in: 3 months + variables: + FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" + + K6_OPTIONS_WARMUP_RATE: 500 + K6_OPTIONS_WARMUP_DURATION: 1m + K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s + K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_WARMUP_MAX_VUS: 4 + + K6_OPTIONS_NORMAL_OPERATION_RATE: 300 + K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m + K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s + K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 4 + + K6_OPTIONS_HIGH_LOAD_RATE: 700 + K6_OPTIONS_HIGH_LOAD_DURATION: 3m + K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s + K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4 + K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4 + + DDTRACE_INSTALL_VERSION: "git://github.com/Datadog/dd-trace-js.git#${CI_COMMIT_SHA}" + + # Workaround: Currently we're not running the benchmarks on every PR, but GitHub still shows them as pending. + # By marking the benchmarks as allow_failure, this should go away. (This workaround should be removed once the + # benchmarks get changed to run on every PR) + allow_failure: true + + # Retry on Gitlab internal system failures + retry: + max: 2 + when: + - unknown_failure + - data_integrity_failure + - runner_system_failure + - scheduler_failure + - api_failure + +baseline: + extends: .macrobenchmarks + variables: + DD_BENCHMARKS_CONFIGURATION: baseline + +only-tracing: + extends: .macrobenchmarks + variables: + DD_BENCHMARKS_CONFIGURATION: only-tracing diff --git a/.gitlab/single-step-instrumentation-tests.yml b/.gitlab/single-step-instrumentation-tests.yml index 77cd4766d2b..704d15faba2 100644 --- a/.gitlab/single-step-instrumentation-tests.yml +++ b/.gitlab/single-step-instrumentation-tests.yml @@ -55,9 +55,9 @@ onboarding_tests: parallel: matrix: - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs] - SCENARIO: [SIMPLE_HOST_AUTO_INJECTION] + SCENARIO: [SIMPLE_HOST_AUTO_INJECTION_PROFILING] - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs-container] - SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION] + SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION_PROFILING] script: - git clone https://git@github.com/DataDog/system-tests.git system-tests - cp packaging/*.rpm system-tests/binaries diff --git a/.mocharc.yml b/.mocharc.yml new file mode 100644 index 00000000000..7646acebf54 --- /dev/null +++ b/.mocharc.yml @@ -0,0 +1,3 @@ +color: true +exit: true # TODO: Fix tests so that this is not needed. +timeout: '5000' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 294617ece08..28d493c36b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,13 @@ $ yarn ## Testing +### Prerequisites + +The `pg-native` package requires `pg_config` to be in your `$PATH` to be able to install. +Please refer to [the "Install" section](https://github.com/brianc/node-postgres/tree/master/packages/pg-native#install) of the `pg-native` documentation for how to ensure your environment is configured correctly. + +### Setup + Before running _plugin_ tests, the data stores need to be running. The easiest way to start all of them is to use the provided docker-compose configuration: @@ -96,9 +103,9 @@ $ yarn services ``` > **Note** -> The `couchbase`, `grpc` and `oracledb` instrumentations rely on native modules -> that do not compile on ARM64 devices (for example M1/M2 Mac) - their tests -> cannot be run locally on these devices. +> The `aerospike`, `couchbase`, `grpc` and `oracledb` instrumentations rely on +> native modules that do not compile on ARM64 devices (for example M1/M2 Mac) +> - their tests cannot be run locally on these devices. ### Unit Tests diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index eca2504caaa..abb5bb48c18 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -64,6 +64,7 @@ dev,sinon,BSD-3-Clause,Copyright 2010-2017 Christian Johansen dev,sinon-chai,WTFPL and BSD-2-Clause,Copyright 2004 Sam Hocevar 2012–2017 Domenic Denicola dev,tap,ISC,Copyright 2011-2022 Isaac Z. Schlueter and Contributors dev,tape,MIT,Copyright James Halliday +dev,tiktoken,MIT,Copyright (c) 2022 OpenAI, Shantanu Jain file,aws-lambda-nodejs-runtime-interface-client,Apache 2.0,Copyright 2019 Amazon.com Inc. or its affiliates. All Rights Reserved. file,profile.proto,Apache license 2.0,Copyright 2016 Google Inc. file,is-git-url,MIT,Copyright (c) 2017 Jon Schlinkert. diff --git a/docker-compose.yml b/docker-compose.yml index 24b92e2f872..fc45f699aa1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - '127.0.0.1:1521:1521' - '127.0.0.1:5500:5500' elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0 + image: elasticsearch:7.17.22 environment: - discovery.type=single-node - "ES_JAVA_OPTS=-Xms128m -Xmx128m" @@ -70,9 +70,7 @@ services: ports: - "11211:11211" cassandra: - image: spotify/cassandra - environment: - - CASSANDRA_TOKEN=-9223372036854775808 + image: cassandra:3-focal ports: - "127.0.0.1:9042:9042" limitd: diff --git a/docs/test.ts b/docs/test.ts index 7734dad4098..fe17007bf0c 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -119,6 +119,11 @@ tracer.init({ }, rasp: { enabled: true + }, + stackTrace: { + enabled: true, + maxStackTraces: 5, + maxDepth: 42 } } }); @@ -134,6 +139,11 @@ tracer.init({ redactionEnabled: true, redactionNamePattern: 'password', redactionValuePattern: 'bearer' + }, + appsec: { + standalone: { + enabled: true + } } } }) @@ -217,6 +227,7 @@ const elasticsearchOptions: plugins.elasticsearch = { const awsSdkOptions: plugins.aws_sdk = { service: 'test', splitByAwsService: false, + batchPropagationEnabled: false, hooks: { request: (span?: Span, response?) => {}, }, @@ -353,6 +364,8 @@ tracer.use('sharedb'); tracer.use('sharedb', sharedbOptions); tracer.use('tedious'); tracer.use('undici'); +tracer.use('vitest'); +tracer.use('vitest', { service: 'vitest-service' }); tracer.use('winston'); tracer.use('express', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 07bc2cd29e3..2f462dd93e7 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -4,7 +4,7 @@ declare const exporters: { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', MOCHA_WORKER: 'mocha_worker' } diff --git a/index.d.ts b/index.d.ts index 4184a015fda..34c5021b752 100644 --- a/index.d.ts +++ b/index.d.ts @@ -198,6 +198,7 @@ interface Plugins { "sharedb": tracer.plugins.sharedb; "tedious": tracer.plugins.tedious; "undici": tracer.plugins.undici; + "vitest": tracer.plugins.vitest; "winston": tracer.plugins.winston; } @@ -556,6 +557,19 @@ declare namespace tracer { */ redactionValuePattern?: string } + + appsec?: { + /** + * Configuration of Standalone ASM mode + */ + standalone?: { + /** + * Whether to enable Standalone ASM. + * @default false + */ + enabled?: boolean + } + } }; /** @@ -700,6 +714,25 @@ declare namespace tracer { * @default false */ enabled?: boolean + }, + /** + * Configuration for stack trace reporting + */ + stackTrace?: { + /** Whether to enable stack trace reporting. + * @default true + */ + enabled?: boolean, + + /** Specifies the maximum number of stack traces to be reported. + * @default 2 + */ + maxStackTraces?: number, + + /** Specifies the maximum depth of a stack trace to be reported. + * @default 32 + */ + maxDepth?: number, } }; @@ -1190,6 +1223,13 @@ declare namespace tracer { */ splitByAwsService?: boolean; + /** + * Whether to inject all messages during batch AWS SQS, Kinesis, and SNS send operations. Normal + * behavior is to inject the first message in batch send operations. + * @default false + */ + batchPropagationEnabled?: boolean; + /** * Hooks to run before spans are finished. */ @@ -1524,7 +1564,7 @@ declare namespace tracer { /** * This plugin automatically instruments the - * [jest](https://github.com/facebook/jest) module. + * [jest](https://github.com/jestjs/jest) module. */ interface jest extends Integration {} @@ -1807,6 +1847,12 @@ declare namespace tracer { */ interface undici extends HttpClient {} + /** + * This plugin automatically instruments the + * [vitest](https://github.com/vitest-dev/vitest) module. + */ + interface vitest extends Integration {} + /** * This plugin patches the [winston](https://github.com/winstonjs/winston) * to automatically inject trace identifiers in log records when the diff --git a/init.js b/init.js index 328e287f506..7ddc8dbe91e 100644 --- a/init.js +++ b/init.js @@ -2,8 +2,26 @@ const path = require('path') const Module = require('module') +const telemetry = require('./packages/dd-trace/src/telemetry/init-telemetry') +const semver = require('semver') + +function isTrue (envVar) { + return ['1', 'true', 'True'].includes(envVar) +} + +// eslint-disable-next-line no-console +let log = { info: isTrue(process.env.DD_TRACE_DEBUG) ? console.log : () => {} } +if (semver.satisfies(process.versions.node, '>=16')) { + const Config = require('./packages/dd-trace/src/config') + log = require('./packages/dd-trace/src/log') + + // eslint-disable-next-line no-new + new Config() // we need this to initialize the logger +} let initBailout = false +let clobberBailout = false +const forced = isTrue(process.env.DD_INJECT_FORCE) if (process.env.DD_INJECTION_ENABLED) { // If we're running via single-step install, and we're not in the app's @@ -19,13 +37,34 @@ if (process.env.DD_INJECTION_ENABLED) { if (resolvedInApp) { const ourselves = path.join(__dirname, 'index.js') if (ourselves !== resolvedInApp) { + clobberBailout = true + } + } + + // If we're running via single-step install, and the runtime doesn't match + // the engines field in package.json, then we should not initialize the tracer. + if (!clobberBailout) { + const { engines } = require('./package.json') + const version = process.versions.node + if (!semver.satisfies(version, engines.node)) { initBailout = true + telemetry([ + { name: 'abort', tags: ['reason:incompatible_runtime'] }, + { name: 'abort.runtime', tags: [] } + ]) + log.info('Aborting application instrumentation due to incompatible_runtime.') + log.info(`Found incompatible runtime nodejs ${version}, Supported runtimes: nodejs ${engines.node}.`) + if (forced) { + log.info('DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing.') + } } } } -if (!initBailout) { +if (!clobberBailout && (!initBailout || forced)) { const tracer = require('.') tracer.init() module.exports = tracer + telemetry('complete', [`injection_forced:${forced && initBailout ? 'true' : 'false'}`]) + log.info('Application instrumentation bootstrapping complete') } diff --git a/initialize.mjs b/initialize.mjs index e7b33de492b..777f45cc046 100644 --- a/initialize.mjs +++ b/initialize.mjs @@ -44,9 +44,12 @@ export async function getSource (...args) { } if (isMainThread) { - await import('./init.js') - const { register } = await import('node:module') - if (register) { - register('./loader-hook.mjs', import.meta.url) - } + // Need this IIFE for versions of Node.js without top-level await. + (async () => { + await import('./init.js') + const { register } = await import('node:module') + if (register) { + register('./loader-hook.mjs', import.meta.url) + } + })() } diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js deleted file mode 100644 index 4368f761cd0..00000000000 --- a/integration-tests/ci-visibility.spec.js +++ /dev/null @@ -1,2804 +0,0 @@ -'use strict' - -const { fork, exec } = require('child_process') -const path = require('path') - -const { assert } = require('chai') -const getPort = require('get-port') - -const { - createSandbox, - getCiVisAgentlessConfig, - getCiVisEvpProxyConfig -} = require('./helpers') -const { FakeCiVisIntake } = require('./ci-visibility-intake') - -const { - TEST_CODE_COVERAGE_ENABLED, - TEST_ITR_SKIPPING_ENABLED, - TEST_ITR_TESTS_SKIPPED, - TEST_CODE_COVERAGE_LINES_PCT, - TEST_SUITE, - TEST_STATUS, - TEST_SKIPPED_BY_ITR, - TEST_ITR_SKIPPING_TYPE, - TEST_ITR_SKIPPING_COUNT, - TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN, - TEST_SOURCE_FILE, - TEST_IS_NEW, - TEST_IS_RETRY, - TEST_EARLY_FLAKE_ENABLED, - TEST_NAME, - JEST_DISPLAY_NAME, - TEST_EARLY_FLAKE_ABORT_REASON, - TEST_COMMAND, - TEST_MODULE, - MOCHA_IS_PARALLEL, - TEST_SOURCE_START -} = require('../packages/dd-trace/src/plugins/util/test') -const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') - -const hookFile = 'dd-trace/loader-hook.mjs' - -const mochaCommonOptions = { - name: 'mocha', - expectedStdout: '2 passing', - extraStdout: 'end event: can add event listeners to mocha' -} - -const jestCommonOptions = { - name: 'jest', - dependencies: ['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'], - expectedStdout: 'Test Suites: 2 passed', - expectedCoverageFiles: [ - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ] -} - -const testFrameworks = [ - { - ...mochaCommonOptions, - testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], - expectedCoverageFiles: [ - 'ci-visibility/run-mocha.js', - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ], - runTestsWithCoverageCommand: './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js', - type: 'commonJS' - }, - { - ...jestCommonOptions, - testFile: 'ci-visibility/run-jest.js', - runTestsWithCoverageCommand: 'node ./ci-visibility/run-jest.js', - type: 'commonJS' - } -] - -// TODO: add ESM tests -testFrameworks.forEach(({ - name, - dependencies, - testFile, - expectedStdout, - extraStdout, - expectedCoverageFiles, - runTestsWithCoverageCommand, - type -}) => { - describe(`${name} ${type}`, () => { - let receiver - let childProcess - let sandbox - let cwd - let startupTestFile - let testOutput = '' - - before(async function () { - // add an explicit timeout to make esm tests less flaky - this.timeout(50000) - sandbox = await createSandbox(dependencies, true) - cwd = sandbox.folder - startupTestFile = path.join(cwd, testFile) - }) - - after(async function () { - await sandbox.remove() - }) - - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) - - afterEach(async () => { - childProcess.kill() - testOutput = '' - await receiver.stop() - }) - - if (name === 'mocha') { - it('does not change mocha config if CI Visibility fails to init', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) - - const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) - - // `runMocha` is only executed when using the CLI, which is where we modify mocha config - // if CI Visibility is init - childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { - cwd, - env: { - ...restEnvVars, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'error', - DD_SITE: '= invalid = url' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - assert.include(testOutput, 'Invalid URL') - assert.include(testOutput, '1 passing') // we only run one file here - done() - }) - }).timeout(50000) - - it('works with parallel mode', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const sessionEventContent = events.find(event => event.type === 'test_session_end').content - const moduleEventContent = events.find(event => event.type === 'test_module_end').content - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') - assert.equal( - sessionEventContent.test_session_id.toString(10), - moduleEventContent.test_session_id.toString(10) - ) - suites.forEach(({ - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) - }) - - tests.forEach(({ - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) - assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') - assert.exists(metrics[TEST_SOURCE_START]) - }) - }) - - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - eventsPromise.then(() => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude( - testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' - ) - done() - }).catch(done) - }) - }) - - it('works with parallel mode when run with the cli', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const sessionEventContent = events.find(event => event.type === 'test_session_end').content - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') - assert.equal(suites.length, 2) - assert.equal(tests.length, 2) - }) - childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude( - testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' - ) - done() - }).catch(done) - }) - }) - - it('does not blow up when workerpool is used outside of a test', (done) => { - childProcess = exec('node ./ci-visibility/run-workerpool.js', { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', (code) => { - assert.include(testOutput, 'result 7') - assert.equal(code, 0) - done() - }) - }) - } - - if (name === 'jest') { - it('works when sharding', (done) => { - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { - const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') - assert.equal(testSuiteEvents.length, 3) - const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) - - assert.includeMembers(testSuites, - [ - 'ci-visibility/sharding-test/sharding-test-5.js', - 'ci-visibility/sharding-test/sharding-test-4.js', - 'ci-visibility/sharding-test/sharding-test-1.js' - ] - ) - - const testSession = events.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - - // We run the second shard - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-2.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-3.js' - } - } - ]) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '2/2' - }, - stdio: 'inherit' - } - ) - - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { - const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') - - // The suites for this shard are to be skipped - assert.equal(testSuiteEvents.length, 2) - - testSuiteEvents.forEach(testSuite => { - assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') - assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') - }) - - const testSession = secondShardEvents - .payload - .events - .find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) - - done() - }) - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '1/2' - }, - stdio: 'inherit' - } - ) - }) - it('does not crash when jest is badly initialized', (done) => { - childProcess = fork('ci-visibility/run-jest-bad-init.js', { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, expectedStdout) - done() - }) - }) - it('does not crash when jest uses jest-jasmine2', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - OLD_RUNNER: 1, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - done() - }) - }) - describe('when jest is using workers to run tests in parallel', () => { - it('reports tests when using the agent', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - childProcess = fork(testFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { - const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) - assert.equal(testSpans.length, 2) - const spanTypes = testSpans.map(span => span.type) - assert.includeMembers(spanTypes, ['test']) - assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) - done() - }).catch(done) - }) - - it('reports tests when using agentless', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - - it('reports tests when using evp proxy', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) - .then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - }) - it('reports timeout error message', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true, - TESTS_TO_RUN: 'timeout-test/timeout-test.js' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') - done() - }) - }) - it('reports parsing errors in the test file', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - assert.equal(suites.length, 2) - - const resourceNames = suites.map(suite => suite.content.resource) - - assert.includeMembers(resourceNames, [ - 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', - 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' - ]) - suites.forEach(suite => { - assert.equal(suite.content.meta[TEST_STATUS], 'fail') - assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') - }) - }) - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'test-parsing-error/parsing-error' - }, - stdio: 'pipe' - }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not report total code coverage % if user has not configured coverage manually', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DISABLE_CODE_COVERAGE: '1' - }, - stdio: 'inherit' - } - ) - }) - it('reports total code coverage % even when ITR is disabled', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('works with --forceExit and logs a warning', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - assert.include(testOutput, "Jest's '--forceExit' flag has been passed") - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end') - const testModule = events.find(event => event.type === 'test_module_end') - const testSuites = events.filter(event => event.type === 'test_suite_end') - const tests = events.filter(event => event.type === 'test') - - assert.exists(testSession) - assert.exists(testModule) - assert.equal(testSuites.length, 2) - assert.equal(tests.length, 2) - }) - // Needs to run with the CLI if we want --forceExit to work - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - it('does not hang if server is not available and logs an error', (done) => { - // Very slow intake - receiver.setWaitingTime(30000) - // Needs to run with the CLI if we want --forceExit to work - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - const EXPECTED_FORCE_EXIT_LOG_MESSAGE = "Jest's '--forceExit' flag has been passed" - const EXPECTED_TIMEOUT_LOG_MESSAGE = 'Timeout waiting for the tracer to flush' - childProcess.on('exit', () => { - assert.include( - testOutput, - EXPECTED_FORCE_EXIT_LOG_MESSAGE, - `"${EXPECTED_FORCE_EXIT_LOG_MESSAGE}" log message is not in test output: ${testOutput}` - ) - assert.include( - testOutput, - EXPECTED_TIMEOUT_LOG_MESSAGE, - `"${EXPECTED_TIMEOUT_LOG_MESSAGE}" log message is not in the test output: ${testOutput}` - ) - done() - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - it('intelligent test runner can skip when using a custom test sequencer', (done) => { - receiver.setSettings({ - itr_enabled: true, - tests_skipping: true - }) - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testEvents = events.filter(event => event.type === 'test') - // no tests end up running (suite is skipped) - assert.equal(testEvents.length, 0) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - - const skippedSuite = events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - CUSTOM_TEST_SEQUENCER: './ci-visibility/jest-custom-test-sequencer.js', - TEST_SHARD: '2/2' - }, - stdio: 'inherit' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', () => { - assert.include(testOutput, 'Running shard with a custom sequencer') - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('grabs the jest displayName config and sets tag in tests and suites', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 4) // two per display name - const nodeTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'node') - assert.equal(nodeTests.length, 2) - - const standardTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'standard') - assert.equal(standardTests.length, 2) - - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - assert.equal(suites.length, 4) - - const nodeSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'node') - assert.equal(nodeSuites.length, 2) - - const standardSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'standard') - assert.equal(standardSuites.length, 2) - }) - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('works with multi project setup and test skipping with intelligent test runner', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - // suites for both projects in the multi-project config are reported as skipped - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - - const skippedSuites = testSuites.filter( - suite => suite.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ) - assert.equal(skippedSuites.length, 2) - - skippedSuites.forEach(skippedSuite => { - assert.equal(skippedSuite.meta[TEST_STATUS], 'skip') - assert.equal(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') - }) - }) - - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('calculates executable lines even if there have been skipped suites', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test-total-code-coverage/test-skipped.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - - // Before https://github.com/DataDog/dd-trace-js/pull/4336, this would've been 100% - // The reason is that skipping jest's `addUntestedFiles`, we would not see unexecuted lines. - // In this cause, these would be from the `unused-dependency.js` file. - // It is 50% now because we only cover 1 out of 2 files (`used-dependency.js`). - assert.propertyVal(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT, 50) - }) - - childProcess = exec( - runTestsWithCoverageCommand, // Requirement: the user must've opted in to code coverage - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'ci-visibility/test-total-code-coverage/test-', - COLLECT_COVERAGE_FROM: '**/test-total-code-coverage/**' - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(done).catch(done) - }) - }) - } - const reportingOptions = ['agentless', 'evp proxy'] - - reportingOptions.forEach(reportingOption => { - context(`early flake detection when reporting by ${reportingOption}`, () => { - it('retries new tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - // TODO: maybe check in stdout for the "Retried by Datadog" - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - // no other tests are considered new - const oldTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' - ) - oldTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) - }) - assert.equal(oldTests.length, 1) - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') - }) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('handles parameterized tests as a single unit', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] - } - }) - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - - const parameterizedTestFile = name === 'mocha' ? 'mocha-parameterized.js' : 'test-parameterized.js' - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === `ci-visibility/test-early-flake-detection/${parameterizedTestFile}` - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - // Each parameter is repeated independently - const testsForFirstParameter = tests.filter(test => test.resource === - `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 1` - ) - - const testsForSecondParameter = tests.filter(test => test.resource === - `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 2` - ) - - assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) - - // all but one have been retried - assert.equal( - testsForFirstParameter.length - 1, - testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length - ) - - assert.equal( - testsForSecondParameter.length - 1, - testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length - ) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/test.js', - `./test-early-flake-detection/${parameterizedTestFile}` - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - // new tests are not detected - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN, - DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('retries flaky tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/occasionally-failing-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - tests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, - // based on the global counter in the test file - const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) - assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) - // Test name does not change - retriedTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') - }) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/occasionally-failing-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/occasionally-failing-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not retry new tests that are skipped', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/skipped-and-todo-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const newSkippedTests = tests.filter( - test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' - ) - assert.equal(newSkippedTests.length, 1) - assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) - - if (name === 'jest') { - const newTodoTests = tests.filter( - test => test.meta[TEST_NAME] === 'ci visibility todo will not be retried' - ) - assert.equal(newTodoTests.length, 1) - assert.notProperty(newTodoTests[0].meta, TEST_IS_RETRY) - } - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/skipped-and-todo-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/skipped-and-todo-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('handles spaces in test names', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - // Tests from ci-visibility/test/skipped-and-todo-test will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ - 'no describe can do stuff', - 'describe trailing space ' - ] - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 2) - - const resourceNames = tests.map(test => test.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', - 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' - ] - ) - - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - // no new tests - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/weird-test-names' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/weird-test-names.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not run EFD if the known tests request fails', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - receiver.setKnownTestsResponseCode(500) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(tests.length, 2) - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => done()).catch(done) - }) - }) - it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/occasionally-failing-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - tests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, - // based on the global counter in the test file - const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) - assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) - // Test name does not change - retriedTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') - }) - }) - - const command = name === 'jest' - ? 'node ./node_modules/jest/bin/jest --config config-jest.js' - : 'node ./node_modules/mocha/bin/mocha ci-visibility/test-early-flake-detection/occasionally-failing-test*' - - childProcess = exec( - command, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' - }, - stdio: 'inherit' - } - ) - - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', (exitCode) => { - if (name === 'jest') { - assert.include(testOutput, '2 failed, 2 passed') - } else { - assert.include(testOutput, '2 passing') - assert.include(testOutput, '2 failing') - } - assert.equal(exitCode, 0) - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - if (name === 'jest') { - it('does not run early flake detection on snapshot tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test-early-flake-detection/jest-snapshot.js will be considered new - // but we don't retry them because they have snapshots - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(tests.length, 1) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - - assert.equal(retriedTests.length, 0) - - // we still detect that it's new - const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 1) - }) - - childProcess = exec(runTestsWithCoverageCommand, { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot', - CI: '1' // needs to be run as CI so snapshots are not written - }, - stdio: 'inherit' - }) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('bails out of EFD if the percentage of new tests is too high', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test* will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 1 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 2) - - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - // no new tests - assert.equal(newTests.length, 0) - }) - - childProcess = exec(runTestsWithCoverageCommand, { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'test/ci-visibility-test' - }, - stdio: 'inherit' - }) - - childProcess.on('exit', () => { - eventsPromise.then(() => done()).catch(done) - }) - }) - - it('works with jsdom', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - // no other tests are considered new - const oldTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' - ) - oldTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) - }) - assert.equal(oldTests.length, 1) - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') - }) - }) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'test/ci-visibility-test', - ENABLE_JSDOM: true, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - } - }) - }) - - it('can run tests and report spans', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - - const areAllTestSpans = testSpans.every(span => span.name === `${name}.test`) - assert.isTrue(areAllTestSpans) - - assert.include(testOutput, expectedStdout) - - if (extraStdout) { - assert.include(testOutput, extraStdout) - } - // Can read DD_TAGS - testSpans.forEach(testSpan => { - assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') - assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') - }) - - testSpans.forEach(testSpan => { - assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) - assert.exists(testSpan.metrics[TEST_SOURCE_START]) - }) - - done() - }) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: type === 'esm' ? `-r dd-trace/ci/init --loader=${hookFile}` : '-r dd-trace/ci/init', - DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] - - envVarSettings.forEach(envVar => { - context(`when ${envVar}=false`, () => { - it('does not report spans but still runs tests', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - [envVar]: 'false' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - }) - context('when no ci visibility init is used', () => { - it('does not crash', (done) => { - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude(testOutput, 'Uncaught error outside test suite') - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - - describe('agentless', () => { - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not init if DD_API_KEY is not set', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, - NODE_OPTIONS: '-r dd-trace/ci/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + - 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + - 'so dd-trace will not be initialized.' - ) - done() - }) - }) - - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/git/repository/search_commits' - ) - const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, - eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - }) - - it('can report code coverage', (done) => { - let testOutput - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') - - const [coveragePayload] = codeCovRequest.payload - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - const [coveragePayload] = coverageRequest.payload - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-unskippable.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 3) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const passedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' - ) - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ) - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ) - // It does not mark as unskippable if there is no docblock - assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') - assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-run.js', - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 3) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const passedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' - ) - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ).content - const nonSkippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ).content - - // It does not mark as unskippable if there is no docblock - assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') - assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - - assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') - assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - // it was not forced to run because it wasn't going to be skipped - assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-run.js', - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - - describe('evp proxy', () => { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - }) - - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('can report git metadata', (done) => { - const infoRequestPromise = receiver.payloadReceived(({ url }) => url === '/info') - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits' - ) - const packFileRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/packfile' - ) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - infoRequestPromise, - searchCommitsRequestPromise, - packFileRequestPromise, - eventsRequestPromise - ]).then(([infoRequest, searchCommitsRequest, packfileRequest, eventsRequest]) => { - assert.notProperty(infoRequest.headers, 'dd-api-key') - - assert.notProperty(searchCommitsRequest.headers, 'dd-api-key') - assert.propertyVal(searchCommitsRequest.headers, 'x-datadog-evp-subdomain', 'api') - - assert.notProperty(packfileRequest.headers, 'dd-api-key') - assert.propertyVal(packfileRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - - it('can report code coverage', (done) => { - let testOutput - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') - assert.propertyVal(libraryConfigRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = codeCovRequest.payload - assert.notProperty(codeCovRequest.headers, 'dd-api-key') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable' - ) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.notProperty(skippableRequest.headers, 'dd-api-key') - assert.propertyVal(skippableRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = coverageRequest.payload - assert.notProperty(coverageRequest.headers, 'dd-api-key') - assert.propertyVal(coverageRequest.headers, 'x-datadog-evp-subdomain', 'citestcov-intake') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.notProperty(eventsRequest.headers, 'dd-api-key') - assert.propertyVal(eventsRequest.headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not skip tests if git metadata upload fails', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - }) -}) diff --git a/integration-tests/ci-visibility/features-retry/flaky.feature b/integration-tests/ci-visibility/features-retry/flaky.feature new file mode 100644 index 00000000000..c24e4a61319 --- /dev/null +++ b/integration-tests/ci-visibility/features-retry/flaky.feature @@ -0,0 +1,4 @@ +Feature: Farewell + Scenario: Say flaky + When the greeter says flaky + Then I should have heard "flaky" diff --git a/integration-tests/ci-visibility/features-retry/support/steps.js b/integration-tests/ci-visibility/features-retry/support/steps.js new file mode 100644 index 00000000000..50da213fb75 --- /dev/null +++ b/integration-tests/ci-visibility/features-retry/support/steps.js @@ -0,0 +1,15 @@ +const assert = require('assert') +const { When, Then } = require('@cucumber/cucumber') + +let globalCounter = 0 + +Then('I should have heard {string}', function (expectedResponse) { + assert.equal(this.whatIHeard, expectedResponse) +}) + +When('the greeter says flaky', function () { + if (++globalCounter < 3) { + throw new Error('Not good enough!') + } + this.whatIHeard = 'flaky' +}) diff --git a/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js b/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js new file mode 100644 index 00000000000..317b97f4175 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-max-failures/failing-test-and-another-test.js @@ -0,0 +1,17 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test('should work with failing tests', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello Warld' + ]) +}) + +test('does not crash afterwards', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello World' + ]) +}) diff --git a/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js b/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js new file mode 100644 index 00000000000..de08821128d --- /dev/null +++ b/integration-tests/ci-visibility/test-flaky-test-retries/eventually-passing-test.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') + +let counter = 0 + +describe('test-flaky-test-retries', () => { + it('can retry failed tests', () => { + expect(++counter).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/sum.mjs b/integration-tests/ci-visibility/vitest-tests/sum.mjs new file mode 100644 index 00000000000..f1c6520acbd --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/sum.mjs @@ -0,0 +1,3 @@ +export function sum (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs new file mode 100644 index 00000000000..a97f95e0df1 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs @@ -0,0 +1,26 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + beforeEach(() => { + throw new Error('failed before each') + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + afterEach(() => { + throw new Error('failed after each') + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs new file mode 100644 index 00000000000..f2df345a87f --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs @@ -0,0 +1,29 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +let preparedValue = 1 + +describe('test-visibility-failed-suite-first-describe', () => { + beforeEach(() => { + preparedValue = 2 + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + expect(preparedValue).to.equal(2) + }) +}) + +describe('test-visibility-failed-suite-second-describe', () => { + afterEach(() => { + preparedValue = 1 + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs new file mode 100644 index 00000000000..c2cf93431d8 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs @@ -0,0 +1,32 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.skip('can skip', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.todo('can todo', () => { + expect(sum(1, 2)).to.equal(3) + }) + // eslint-disable-next-line + test('can programmatic skip', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 5fa6372e8c8..ba7aec01fea 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -777,6 +777,7 @@ versions.forEach(version => { }) }) }) + context('early flake detection', () => { it('retries new tests', (done) => { const NUM_RETRIES_EFD = 3 @@ -1033,6 +1034,56 @@ versions.forEach(version => { }) }) }) + + if (version === 'latest') { // flaky test retries only supported from >=8.0.0 + context('flaky test retries', () => { + it('can retry failed tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures and 1 passed attempt + assert.equal(tests.length, 3) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 1) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 2) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + } }) }) }) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 3018fcd6ce0..5442f74b055 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -39,7 +39,7 @@ const version = process.env.CYPRESS_VERSION const hookFile = 'dd-trace/loader-hook.mjs' const NUM_RETRIES_EFD = 3 -const moduleType = [ +const moduleTypes = [ { type: 'commonJS', testCommand: function commandWithSuffic (version) { @@ -51,9 +51,9 @@ const moduleType = [ type: 'esm', testCommand: `node --loader=${hookFile} ./cypress-esm-config.mjs` } -] +].filter(moduleType => !process.env.CYPRESS_MODULE_TYPE || process.env.CYPRESS_MODULE_TYPE === moduleType.type) -moduleType.forEach(({ +moduleTypes.forEach(({ type, testCommand }) => { @@ -87,8 +87,7 @@ moduleType.forEach(({ }) beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() + receiver = await new FakeCiVisIntake().start() }) afterEach(async () => { @@ -117,7 +116,8 @@ moduleType.forEach(({ env: { ...restEnvVars, CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, - DD_SITE: '= invalid = url' + DD_SITE: '= invalid = url', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -130,7 +130,7 @@ moduleType.forEach(({ }) childProcess.on('exit', () => { assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, '3 of 4 failed') + assert.include(testOutput, '1 of 1 failed') done() }) }) @@ -351,7 +351,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -383,7 +384,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -399,6 +401,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ code_coverage: false, @@ -428,7 +431,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -440,6 +444,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('can skip tests received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -498,7 +503,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -509,6 +515,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.setSettings({ code_coverage: true, @@ -535,6 +542,7 @@ moduleType.forEach(({ event.content.resource === 'cypress/e2e/other.cy.js.context passes' ) assert.exists(notSkippedTest) + assert.equal(notSkippedTest.content.meta[TEST_STATUS], 'pass') }, 25000) const { @@ -548,7 +556,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/other.cy.js' }, stdio: 'pipe' } @@ -560,6 +569,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if suite is marked as unskippable', (done) => { receiver.setSettings({ code_coverage: true, @@ -621,7 +631,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -633,6 +644,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('only sets forced to run if test was going to be skipped by ITR', (done) => { receiver.setSettings({ code_coverage: true, @@ -689,7 +701,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -701,6 +714,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('sets _dd.ci.itr.tests_skipped to false if the received test is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -741,7 +755,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -752,6 +767,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in tests', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) @@ -775,7 +791,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -816,7 +833,8 @@ moduleType.forEach(({ env: { ...restEnvVars, CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, - CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1' + CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -841,7 +859,7 @@ moduleType.forEach(({ assert.equal(testSuiteEvents.length, 4) const testEvents = events.filter(event => event.type === 'test') assert.equal(testEvents.length, 9) - }) + }, 30000) const { NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress @@ -880,7 +898,7 @@ moduleType.forEach(({ assert.equal(testSuiteEvents.length, 4) const testEvents = events.filter(event => event.type === 'test') assert.equal(testEvents.length, 9) - }) + }, 30000) const { NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress @@ -980,6 +998,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1041,6 +1060,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not retry tests that are skipped', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1097,6 +1117,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ itr_enabled: false, diff --git a/integration-tests/cypress/plugins-old/index.js b/integration-tests/cypress/plugins-old/index.js index 6e7415900a9..52fc380ba09 100644 --- a/integration-tests/cypress/plugins-old/index.js +++ b/integration-tests/cypress/plugins-old/index.js @@ -1,6 +1,27 @@ +const ddAfterRun = require('dd-trace/ci/cypress/after-run') +const ddAfterSpec = require('dd-trace/ci/cypress/after-spec') + module.exports = (on, config) => { if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) { require('cypress-fail-fast/plugin')(on, config) } + if (process.env.SPEC_PATTERN) { + config.testFiles = process.env.SPEC_PATTERN.replace('cypress/e2e/', '') + } + if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) { + on('after:run', (...args) => { + // do custom stuff + // and call after-run at the end + return ddAfterRun(...args) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) { + on('after:spec', (...args) => { + // do custom stuff + // and call after-spec at the end + return ddAfterSpec(...args) + }) + } require('dd-trace/ci/cypress/plugin')(on, config) + return config } diff --git a/integration-tests/esbuild/basic-test.js b/integration-tests/esbuild/basic-test.js index 20f53708b44..dc41b4efa53 100755 --- a/integration-tests/esbuild/basic-test.js +++ b/integration-tests/esbuild/basic-test.js @@ -1,11 +1,5 @@ #!/usr/bin/env node -// TODO: add support for Node.js v14.17+ and v16.0+ -if (Number(process.versions.node.split('.')[0]) < 16) { - console.error(`Skip esbuild test for node@${process.version}`) // eslint-disable-line no-console - process.exit(0) -} - const tracer = require('../../').init() // dd-trace const assert = require('assert') diff --git a/integration-tests/helpers.js b/integration-tests/helpers.js index b8972540e1f..98695c60156 100644 --- a/integration-tests/helpers.js +++ b/integration-tests/helpers.js @@ -7,15 +7,16 @@ const msgpack = require('msgpack-lite') const codec = msgpack.createCodec({ int64: true }) const EventEmitter = require('events') const childProcess = require('child_process') -const { fork } = childProcess +const { fork, spawn } = childProcess const exec = promisify(childProcess.exec) const http = require('http') -const fs = require('fs/promises') +const fs = require('fs') const os = require('os') const path = require('path') const rimraf = promisify(require('rimraf')) const id = require('../packages/dd-trace/src/id') const upload = require('multer')() +const assert = require('assert') const hookFile = 'dd-trace/loader-hook.mjs' @@ -79,7 +80,7 @@ class FakeAgent extends EventEmitter { // but it't not guaranteed to be the last one (so, expectedMessageCount would not be helpful). // It can still fail if it takes longer than `timeout` duration or if none pass the assertions (timeout still called) assertMessageReceived (fn, timeout, expectedMessageCount = 1, resolveAtFirstSuccess) { - timeout = timeout || 5000 + timeout = timeout || 30000 let resultResolve let resultReject let msgCount = 0 @@ -119,7 +120,7 @@ class FakeAgent extends EventEmitter { } assertTelemetryReceived (fn, timeout, requestType, expectedMessageCount = 1) { - timeout = timeout || 5000 + timeout = timeout || 30000 let resultResolve let resultReject let msgCount = 0 @@ -162,6 +163,95 @@ class FakeAgent extends EventEmitter { } } +async function runAndCheckOutput (filename, cwd, expectedOut) { + const proc = spawn('node', [filename], { cwd, stdio: 'pipe' }) + const pid = proc.pid + let out = await new Promise((resolve, reject) => { + proc.on('error', reject) + let out = Buffer.alloc(0) + proc.stdout.on('data', data => { + out = Buffer.concat([out, data]) + }) + proc.stderr.pipe(process.stdout) + proc.on('exit', () => resolve(out.toString('utf8'))) + setTimeout(() => { + if (proc.exitCode === null) proc.kill() + }, 1000) // TODO this introduces flakiness. find a better way to end the process. + }) + if (typeof expectedOut === 'function') { + expectedOut(out) + } else { + if (process.env.DD_TRACE_DEBUG) { + // Debug adds this, which we don't care about in these tests + out = out.replace('Flushing 0 metrics via HTTP\n', '') + } + assert.strictEqual(out, expectedOut) + } + return pid +} + +// This is set by the useSandbox function +let sandbox + +// This _must_ be used with the useSandbox function +async function runAndCheckWithTelemetry (filename, expectedOut, ...expectedTelemetryPoints) { + const cwd = sandbox.folder + const cleanup = telemetryForwarder(expectedTelemetryPoints) + const pid = await runAndCheckOutput(filename, cwd, expectedOut) + const msgs = await cleanup() + if (expectedTelemetryPoints.length === 0) { + // assert no telemetry sent + try { + assert.deepStrictEqual(msgs.length, 0) + } catch (e) { + // This console.log is useful for debugging telemetry. Plz don't remove. + // eslint-disable-next-line no-console + console.error('Expected no telemetry, but got:\n', msgs.map(msg => JSON.stringify(msg[1].points)).join('\n')) + throw e + } + return + } + let points = [] + for (const [telemetryType, data] of msgs) { + assert.strictEqual(telemetryType, 'library_entrypoint') + assert.deepStrictEqual(data.metadata, meta(pid)) + points = points.concat(data.points) + } + let expectedPoints = getPoints(...expectedTelemetryPoints) + // We now have to sort both the expected and actual telemetry points. + // This is because data can come in in any order. + // We'll just contatenate all the data together for each point and sort them. + points = points.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + expectedPoints = expectedPoints.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + assert.strictEqual(points, expectedPoints) + + function getPoints (...args) { + const expectedPoints = [] + let currentPoint = {} + for (const arg of args) { + if (!currentPoint.name) { + currentPoint.name = 'library_entrypoint.' + arg + } else { + currentPoint.tags = arg.split(',') + expectedPoints.push(currentPoint) + currentPoint = {} + } + } + return expectedPoints + } + + function meta (pid) { + return { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: require('../package.json').version, + pid: Number(pid) + } + } +} + function spawnProc (filename, options = {}, stdioHandler) { const proc = fork(filename, { ...options, stdio: 'pipe' }) return new Promise((resolve, reject) => { @@ -205,9 +295,9 @@ async function createSandbox (dependencies = [], isGitRepo = false, // We might use NODE_OPTIONS to init the tracer. We don't want this to affect this operations const { NODE_OPTIONS, ...restOfEnv } = process.env - await fs.mkdir(folder) - await exec(`yarn pack --filename ${out}`) // TODO: cache this - await exec(`yarn add ${allDependencies.join(' ')}`, { cwd: folder, env: restOfEnv }) + fs.mkdirSync(folder) + await exec(`yarn pack --filename ${out}`, { env: restOfEnv }) // TODO: cache this + await exec(`yarn add ${allDependencies.join(' ')} --ignore-engines`, { cwd: folder, env: restOfEnv }) for (const path of integrationTestsPaths) { if (process.platform === 'win32') { @@ -229,7 +319,7 @@ async function createSandbox (dependencies = [], isGitRepo = false, if (isGitRepo) { await exec('git init', { cwd: folder }) - await fs.writeFile(path.join(folder, '.gitignore'), 'node_modules/', { flush: true }) + fs.writeFileSync(path.join(folder, '.gitignore'), 'node_modules/', { flush: true }) await exec('git config user.email "john@doe.com"', { cwd: folder }) await exec('git config user.name "John Doe"', { cwd: folder }) await exec('git config commit.gpgsign false', { cwd: folder }) @@ -245,6 +335,54 @@ async function createSandbox (dependencies = [], isGitRepo = false, } } +function telemetryForwarder (expectedTelemetryPoints) { + process.env.DD_TELEMETRY_FORWARDER_PATH = + path.join(__dirname, 'telemetry-forwarder.sh') + process.env.FORWARDER_OUT = path.join(__dirname, `forwarder-${Date.now()}.out`) + + let retries = 0 + + const tryAgain = async function () { + retries += 1 + await new Promise(resolve => setTimeout(resolve, 100)) + return cleanup() + } + + const cleanup = function () { + let msgs + try { + msgs = fs.readFileSync(process.env.FORWARDER_OUT, 'utf8').trim().split('\n') + } catch (e) { + if (expectedTelemetryPoints.length && e.code === 'ENOENT' && retries < 10) { + return tryAgain() + } + return [] + } + for (let i = 0; i < msgs.length; i++) { + const [telemetryType, data] = msgs[i].split('\t') + if (!data && retries < 10) { + return tryAgain() + } + let parsed + try { + parsed = JSON.parse(data) + } catch (e) { + if (!data && retries < 10) { + return tryAgain() + } + throw new SyntaxError(`error parsing data: ${e.message}\n${data}`) + } + msgs[i] = [telemetryType, parsed] + } + fs.unlinkSync(process.env.FORWARDER_OUT) + delete process.env.FORWARDER_OUT + delete process.env.DD_TELEMETRY_FORWARDER_PATH + return msgs + } + + return cleanup +} + async function curl (url, useHttp2 = false) { if (typeof url === 'object') { if (url.then) { @@ -313,14 +451,43 @@ async function spawnPluginIntegrationTestProc (cwd, serverFile, agentPort, stdio }, stdioHandler) } +function useEnv (env) { + before(() => { + Object.assign(process.env, env) + }) + after(() => { + for (const key of Object.keys(env)) { + delete process.env[key] + } + }) +} + +function useSandbox (...args) { + before(async () => { + sandbox = await createSandbox(...args) + }) + after(() => { + const oldSandbox = sandbox + sandbox = undefined + return oldSandbox.remove() + }) +} +function sandboxCwd () { + return sandbox.folder +} + module.exports = { FakeAgent, spawnProc, + runAndCheckWithTelemetry, createSandbox, curl, curlAndAssertMessage, getCiVisAgentlessConfig, getCiVisEvpProxyConfig, checkSpansForServiceName, - spawnPluginIntegrationTestProc + spawnPluginIntegrationTestProc, + useEnv, + useSandbox, + sandboxCwd } diff --git a/integration-tests/init.spec.js b/integration-tests/init.spec.js index f90968fc8c6..f516d8b40d8 100644 --- a/integration-tests/init.spec.js +++ b/integration-tests/init.spec.js @@ -1,107 +1,180 @@ +const semver = require('semver') const { - createSandbox, - spawnProc + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd } = require('./helpers') -const { assert } = require('chai') const path = require('path') +const fs = require('fs') +const { DD_MAJOR } = require('../version') const DD_INJECTION_ENABLED = 'tracing' +const DD_INJECT_FORCE = 'true' +const DD_TRACE_DEBUG = 'true' -let cwd, proc, sandbox +const telemetryAbort = ['abort', 'reason:incompatible_runtime', 'abort.runtime', ''] +const telemetryForced = ['complete', 'injection_forced:true'] +const telemetryGood = ['complete', 'injection_forced:false'] -async function runTest (cwd, file, env, expected) { - return new Promise((resolve, reject) => { - spawnProc(path.join(cwd, file), { cwd, env, silent: true }, data => { - try { - assert.strictEqual(data.toString(), expected) - resolve() - } catch (e) { - reject(e) - } - }).then(subproc => { - proc = subproc - }) - }) -} +const { engines } = require('../package.json') +const supportedRange = engines.node +const currentVersionIsSupported = semver.satisfies(process.versions.node, supportedRange) function testInjectionScenarios (arg, filename, esmWorks = false) { - context('when dd-trace is not in the app dir', () => { - const NODE_OPTIONS = `--no-warnings --${arg} ${path.join(__dirname, '..', filename)}` - it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS }, 'true\n') - }) - it('should not initialize the tracer, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') - }) - it('should initialize instrumentation, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS }, 'true\n') - }) - it('should not initialize instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if no DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS }, `${esmWorks}\n`) + if (!currentVersionIsSupported) return + const doTest = (file, ...args) => testFile(file, ...args) + context('preferring app-dir dd-trace', () => { + context('when dd-trace is not in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} ${path.join(__dirname, '..', filename)}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should not initialize the tracer', () => doTest('init/trace.js', 'false\n')) + it('should not initialize instrumentation', () => doTest('init/instrument.js', 'false\n')) + it('should not initialize ESM instrumentation', () => doTest('init/instrument.mjs', 'false\n')) + }) }) - it('should not initialize ESM instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') + context('when dd-trace in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} dd-trace/${filename}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', ...telemetryGood)) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', ...telemetryGood)) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`, ...telemetryGood)) + }) }) }) - context('when dd-trace in the app dir', () => { - const NODE_OPTIONS = `--no-warnings --${arg} dd-trace/${filename}` - it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS }, 'true\n') - }) - it('should initialize the tracer, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n') - }) - it('should initialize instrumentation, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS }, 'true\n') - }) - it('should initialize instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n') - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if no DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS }, `${esmWorks}\n`) - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS, DD_INJECTION_ENABLED }, `${esmWorks}\n`) - }) +} + +function testRuntimeVersionChecks (arg, filename) { + context('runtime version check', () => { + const NODE_OPTIONS = `--${arg} dd-trace/${filename}` + const doTest = (...args) => testFile('init/trace.js', ...args) + const doTestForced = async (...args) => { + Object.assign(process.env, { DD_INJECT_FORCE }) + try { + await testFile('init/trace.js', ...args) + } finally { + delete process.env.DD_INJECT_FORCE + } + } + + if (!currentVersionIsSupported) { + context('when node version is less than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => + doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should not initialize the tracer', () => doTest('false\n', ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => doTestForced('true\n', ...telemetryForced)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should not initialize the tracer', () => + doTest(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ +>=${DD_MAJOR === 4 ? '16' : '18'}. +false +`, ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ +>=${DD_MAJOR === 4 ? '16' : '18'}. +DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing. +Application instrumentation bootstrapping complete +true +`, ...telemetryForced)) + }) + }) + }) + } else { + context('when node version is more than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should initialize the tracer', () => doTest('true\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('true\n', ...telemetryGood)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should initialize the tracer', () => + doTest('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + }) + }) + }) + } }) } +function stubTracerIfNeeded () { + if (!currentVersionIsSupported) { + before(() => { + // Stub out the tracer in the sandbox, since it will not likely load properly. + // We're only doing this on versions we don't support, since the forcing + // action results in undefined behavior in the tracer. + fs.writeFileSync( + path.join(sandboxCwd(), 'node_modules/dd-trace/index.js'), + 'exports.init = () => { Object.assign(global, { _ddtrace: true }) }' + ) + }) + } +} + describe('init.js', () => { - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - }) - afterEach(() => { - proc && proc.kill() - }) - after(() => { - return sandbox.remove() - }) + useSandbox() + stubTracerIfNeeded() testInjectionScenarios('require', 'init.js', false) + testRuntimeVersionChecks('require', 'init.js') }) -describe('initialize.mjs', () => { - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - }) - afterEach(() => { - proc && proc.kill() - }) - after(() => { - return sandbox.remove() - }) +// ESM is not supportable prior to Node.js 12 +if (semver.satisfies(process.versions.node, '>=12')) { + describe('initialize.mjs', () => { + useSandbox() + stubTracerIfNeeded() - context('as --loader', () => { - testInjectionScenarios('loader', 'initialize.mjs', true) - }) - if (Number(process.versions.node.split('.')[0]) >= 18) { - context('as --import', () => { - testInjectionScenarios('import', 'initialize.mjs', true) + context('as --loader', () => { + testInjectionScenarios('loader', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') }) - } -}) + if (Number(process.versions.node.split('.')[0]) >= 18) { + context('as --import', () => { + testInjectionScenarios('import', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') + }) + } + }) +} diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js new file mode 100644 index 00000000000..4f9d01dbf8b --- /dev/null +++ b/integration-tests/jest/jest.spec.js @@ -0,0 +1,2117 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + JEST_DISPLAY_NAME, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SOURCE_START +} = require('../../packages/dd-trace/src/plugins/util/test') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const testFile = 'ci-visibility/run-jest.js' +const expectedStdout = 'Test Suites: 2 passed' +const expectedCoverageFiles = [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' +] +const runTestsWithCoverageCommand = 'node ./ci-visibility/run-jest.js' + +// TODO: add ESM tests +describe('jest CommonJS', () => { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'jest.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + + // Can read DD_TAGS + tests.forEach(testEvent => { + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + }) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('works when sharding', (done) => { + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { + const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') + assert.equal(testSuiteEvents.length, 3) + const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) + + assert.includeMembers(testSuites, + [ + 'ci-visibility/sharding-test/sharding-test-5.js', + 'ci-visibility/sharding-test/sharding-test-4.js', + 'ci-visibility/sharding-test/sharding-test-1.js' + ] + ) + + const testSession = events.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + + // We run the second shard + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-2.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-3.js' + } + } + ]) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { + const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') + + // The suites for this shard are to be skipped + assert.equal(testSuiteEvents.length, 2) + + testSuiteEvents.forEach(testSuite => { + assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') + assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + + const testSession = secondShardEvents + .payload + .events + .find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) + + done() + }) + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '1/2' + }, + stdio: 'inherit' + } + ) + }) + + it('does not crash when jest is badly initialized', (done) => { + childProcess = fork('ci-visibility/run-jest-bad-init.js', { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.include(testOutput, expectedStdout) + done() + }) + }) + + it('does not crash when jest uses jest-jasmine2', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + OLD_RUNNER: 1, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + done() + }) + }) + + context('when jest is using workers to run tests in parallel', () => { + it('reports tests when using the old agents', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + childProcess = fork(testFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { + const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) + assert.equal(testSpans.length, 2) + const spanTypes = testSpans.map(span => span.type) + assert.includeMembers(spanTypes, ['test']) + assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) + done() + }).catch(done) + }) + + it('reports tests when using agentless', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { + const eventTypes = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + .map(event => event.type) + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + done() + }).catch(done) + }) + + it('reports tests when using evp proxy', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) + .then(eventsRequests => { + const eventTypes = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + .map(event => event.type) + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + done() + }).catch(done) + }) + }) + + it('reports timeout error message', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true, + TESTS_TO_RUN: 'timeout-test/timeout-test.js' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') + done() + }) + }) + + it('reports parsing errors in the test file', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + assert.equal(suites.length, 2) + + const resourceNames = suites.map(suite => suite.content.resource) + + assert.includeMembers(resourceNames, [ + 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', + 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' + ]) + suites.forEach(suite => { + assert.equal(suite.content.meta[TEST_STATUS], 'fail') + assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') + }) + }) + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-parsing-error/parsing-error' + }, + stdio: 'pipe' + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not report total code coverage % if user has not configured coverage manually', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DISABLE_CODE_COVERAGE: '1' + }, + stdio: 'inherit' + } + ) + }) + + it('reports total code coverage % even when ITR is disabled', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('works with --forceExit and logs a warning', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + assert.include(testOutput, "Jest's '--forceExit' flag has been passed") + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end') + const testModule = events.find(event => event.type === 'test_module_end') + const testSuites = events.filter(event => event.type === 'test_suite_end') + const tests = events.filter(event => event.type === 'test') + + assert.exists(testSession) + assert.exists(testModule) + assert.equal(testSuites.length, 2) + assert.equal(tests.length, 2) + }) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not hang if server is not available and logs an error', (done) => { + // Very slow intake + receiver.setWaitingTime(30000) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + const EXPECTED_FORCE_EXIT_LOG_MESSAGE = "Jest's '--forceExit' flag has been passed" + const EXPECTED_TIMEOUT_LOG_MESSAGE = 'Timeout waiting for the tracer to flush' + childProcess.on('exit', () => { + assert.include( + testOutput, + EXPECTED_FORCE_EXIT_LOG_MESSAGE, + `"${EXPECTED_FORCE_EXIT_LOG_MESSAGE}" log message is not in test output: ${testOutput}` + ) + assert.include( + testOutput, + EXPECTED_TIMEOUT_LOG_MESSAGE, + `"${EXPECTED_TIMEOUT_LOG_MESSAGE}" log message is not in the test output: ${testOutput}` + ) + done() + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('grabs the jest displayName config and sets tag in tests and suites', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 4) // two per display name + const nodeTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeTests.length, 2) + + const standardTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardTests.length, 2) + + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(suites.length, 4) + + const nodeSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeSuites.length, 2) + + const standardSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardSuites.length, 2) + }) + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed test suites: 1. Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test/fail-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, expectedCoverageFiles) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('can skip when using a custom test sequencer', (done) => { + receiver.setSettings({ + itr_enabled: true, + tests_skipping: true + }) + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testEvents = events.filter(event => event.type === 'test') + // no tests end up running (suite is skipped) + assert.equal(testEvents.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + + const skippedSuite = events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + CUSTOM_TEST_SEQUENCER: './ci-visibility/jest-custom-test-sequencer.js', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + assert.include(testOutput, 'Running shard with a custom sequencer') + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works with multi project setup and test skipping', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // suites for both projects in the multi-project config are reported as skipped + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + + const skippedSuites = testSuites.filter( + suite => suite.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ) + assert.equal(skippedSuites.length, 2) + + skippedSuites.forEach(skippedSuite => { + assert.equal(skippedSuite.meta[TEST_STATUS], 'skip') + assert.equal(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('calculates executable lines even if there have been skipped suites', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test-total-code-coverage/test-skipped.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + + // Before https://github.com/DataDog/dd-trace-js/pull/4336, this would've been 100% + // The reason is that skipping jest's `addUntestedFiles`, we would not see unexecuted lines. + // In this cause, these would be from the `unused-dependency.js` file. + // It is 50% now because we only cover 1 out of 2 files (`used-dependency.js`). + assert.propertyVal(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT, 50) + }) + + childProcess = exec( + runTestsWithCoverageCommand, // Requirement: the user must've opted in to code coverage + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-total-code-coverage/test-', + COLLECT_COVERAGE_FROM: '**/test-total-code-coverage/**' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(done).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles parameterized tests as a single unit', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const parameterizedTestFile = 'test-parameterized.js' + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === `ci-visibility/test-early-flake-detection/${parameterizedTestFile}` + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 1` + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 2` + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test-early-flake-detection/test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test', + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries flaky tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/occasionally-failing-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not retry new tests that are skipped', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + + const newTodoTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility todo will not be retried' + ) + assert.equal(newTodoTests.length, 1) + assert.notProperty(newTodoTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/skipped-and-todo-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles spaces in test names', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/weird-test-names' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js', + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 failed, 2 passed') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run early flake detection on snapshot tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/jest-snapshot.js will be considered new + // but we don't retry them because they have snapshots + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + + // we still detect that it's new + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot', + CI: '1' // needs to be run as CI so snapshots are not written + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test* will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 1 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('works with jsdom', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), // use agentless for this test, just for variety + TESTS_TO_RUN: 'test/ci-visibility-test', + ENABLE_JSDOM: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js new file mode 100644 index 00000000000..16ebeaff4b3 --- /dev/null +++ b/integration-tests/mocha/mocha.spec.js @@ -0,0 +1,1607 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + TEST_COMMAND, + TEST_MODULE, + MOCHA_IS_PARALLEL, + TEST_SOURCE_START +} = require('../../packages/dd-trace/src/plugins/util/test') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js' +const testFile = 'ci-visibility/run-mocha.js' +const expectedStdout = '2 passing' +const extraStdout = 'end event: can add event listeners to mocha' + +describe('mocha CommonJS', function () { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'mocha.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + if (extraStdout) { + assert.include(testOutput, extraStdout) + } + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + assert.include(testOutput, extraStdout) + + // Can read DD_TAGS + tests.forEach(testEvent => { + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + }) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('does not change mocha config if CI Visibility fails to init', (done) => { + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report tests') + done(error) + }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) + + const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) + + // `runMocha` is only executed when using the CLI, which is where we modify mocha config + // if CI Visibility is init + childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { + cwd, + env: { + ...restEnvVars, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'error', + DD_SITE: '= invalid = url' + }, + stdio: 'pipe' + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + assert.include(testOutput, 'Invalid URL') + assert.include(testOutput, '1 passing') // we only run one file here + done() + }) + }) + + it('works with parallel mode', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal( + sessionEventContent.test_session_id.toString(10), + moduleEventContent.test_session_id.toString(10) + ) + suites.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + }) + + tests.forEach(({ + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + assert.exists(metrics[TEST_SOURCE_START]) + }) + }) + + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('works with parallel mode when run with the cli', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal(suites.length, 2) + assert.equal(tests.length, 2) + }) + + childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('does not blow up when workerpool is used outside of a test', (done) => { + childProcess = exec('node ./ci-visibility/run-workerpool.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', (code) => { + assert.include(testOutput, 'result 7') + assert.equal(code, 0) + done() + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/fail-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + let testOutput + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, + [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' + ] + ) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // coverage report + assert.include(testOutput, 'Lines ') + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('handles parameterized tests as a single unit', (done) => { + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test-early-flake-detection/mocha-parameterized.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 1' + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 2' + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/test.js', + './test-early-flake-detection/mocha-parameterized.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]), + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('retries flaky tests', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/occasionally-failing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('does not retry new tests that are skipped', (done) => { + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/skipped-and-todo-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('handles spaces in test names', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/weird-test-names.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('does not run EFD if the known tests request fails', (done) => { + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/mocha/bin/mocha ci-visibility/test-early-flake-detection/occasionally-failing-test*', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 passing') + assert.include(testOutput, '2 failing') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('flaky test retries', () => { + it('retries failed tests automatically', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-flaky-test-retries/eventually-passing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 3) // two failed retries and then the pass + + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedAttempts.length, 2) + + // The first attempt is not marked as a retry + const retriedFailure = failedAttempts.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedFailure.length, 1) + + const passedAttempt = tests.find(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedAttempt.meta[TEST_IS_RETRY], 'true') + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/opentelemetry.spec.js b/integration-tests/opentelemetry.spec.js index 4a93b08fc15..006cc219bd0 100644 --- a/integration-tests/opentelemetry.spec.js +++ b/integration-tests/opentelemetry.spec.js @@ -96,7 +96,6 @@ describe('opentelemetry', () => { assert.strictEqual(payload.request_type, 'generate-metrics') const metrics = payload.payload - assert.strictEqual(metrics.namespace, 'tracers') const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') @@ -107,7 +106,7 @@ describe('opentelemetry', () => { }, true) }) - it('should capture telemetry if both DD and OTEL env vars are set ', () => { + it('should capture telemetry if both DD and OTEL env vars are set', () => { proc = fork(join(cwd, 'opentelemetry/basic.js'), { cwd, env: { @@ -121,15 +120,15 @@ describe('opentelemetry', () => { OTEL_LOG_LEVEL: 'debug', DD_TRACE_SAMPLE_RATE: '0.5', OTEL_TRACES_SAMPLER: 'traceidratio', - OTEL_TRACES_SAMPLER_ARG: '0.1', + OTEL_TRACES_SAMPLER_ARG: '1.0', DD_TRACE_ENABLED: 'true', OTEL_TRACES_EXPORTER: 'none', DD_RUNTIME_METRICS_ENABLED: 'true', OTEL_METRICS_EXPORTER: 'none', DD_TAGS: 'foo:bar,baz:qux', - OTEL_RESOURCE_ATTRIBUTES: 'foo=bar1,baz=qux1', - DD_TRACE_PROPAGATION_STYLE: 'datadog', - OTEL_PROPAGATORS: 'datadog,tracecontext', + OTEL_RESOURCE_ATTRIBUTES: 'foo+bar13baz+qux1', + DD_TRACE_PROPAGATION_STYLE: 'datadog, tracecontext', + OTEL_PROPAGATORS: 'datadog, tracecontext', OTEL_LOGS_EXPORTER: 'none', OTEL_SDK_DISABLED: 'false' } @@ -144,58 +143,142 @@ describe('opentelemetry', () => { const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') const otelInvalid = metrics.series.filter(({ metric }) => metric === 'otel.env.invalid') - - assert.strictEqual(otelHiding.length, 8) - assert.strictEqual(otelInvalid.length, 1) + assert.strictEqual(otelHiding.length, 9) + assert.strictEqual(otelInvalid.length, 0) assert.deepStrictEqual(otelHiding[0].tags, [ - 'DD_TRACE_LOG_LEVEL', 'OTEL_LOG_LEVEL', + 'config.datadog:DD_TRACE_LOG_LEVEL', 'config.opentelemetry:OTEL_LOG_LEVEL', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[1].tags, [ - 'DD_TRACE_PROPAGATION_STYLE', 'OTEL_PROPAGATORS', + 'config.datadog:DD_TRACE_PROPAGATION_STYLE', 'config.opentelemetry:OTEL_PROPAGATORS', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[2].tags, [ - 'DD_SERVICE', 'OTEL_SERVICE_NAME', + 'config.datadog:DD_SERVICE', 'config.opentelemetry:OTEL_SERVICE_NAME', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[3].tags, [ - 'DD_TRACE_SAMPLE_RATE', 'OTEL_TRACES_SAMPLER', - 'OTEL_TRACES_SAMPLER_ARG', `version:${process.version}` + 'config.datadog:DD_TRACE_SAMPLE_RATE', 'config.opentelemetry:OTEL_TRACES_SAMPLER', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[4].tags, [ - 'DD_TRACE_ENABLED', 'OTEL_TRACES_EXPORTER', + 'config.datadog:DD_TRACE_SAMPLE_RATE', 'config.opentelemetry:OTEL_TRACES_SAMPLER_ARG', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[5].tags, [ - 'DD_RUNTIME_METRICS_ENABLED', 'OTEL_METRICS_EXPORTER', + 'config.datadog:DD_TRACE_ENABLED', 'config.opentelemetry:OTEL_TRACES_EXPORTER', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[6].tags, [ - 'DD_TAGS', 'OTEL_RESOURCE_ATTRIBUTES', + 'config.datadog:DD_RUNTIME_METRICS_ENABLED', 'config.opentelemetry:OTEL_METRICS_EXPORTER', `version:${process.version}` ]) assert.deepStrictEqual(otelHiding[7].tags, [ - 'DD_TRACE_OTEL_ENABLED', 'OTEL_SDK_DISABLED', + 'config.datadog:DD_TAGS', 'config.opentelemetry:OTEL_RESOURCE_ATTRIBUTES', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelHiding[8].tags, [ + 'config.datadog:DD_TRACE_OTEL_ENABLED', 'config.opentelemetry:OTEL_SDK_DISABLED', `version:${process.version}` ]) for (const metric of otelHiding) { assert.strictEqual(metric.points[0][1], 1) } + }, true) + }) + + it('should capture telemetry when OTEL env vars are invalid', () => { + proc = fork(join(cwd, 'opentelemetry/basic.js'), { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TRACE_OTEL_ENABLED: 1, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + TIMEOUT: 1500, + OTEL_SERVICE_NAME: 'otel_service', + OTEL_LOG_LEVEL: 'foo', + OTEL_TRACES_SAMPLER: 'foo', + OTEL_TRACES_SAMPLER_ARG: 'foo', + OTEL_TRACES_EXPORTER: 'foo', + OTEL_METRICS_EXPORTER: 'foo', + OTEL_RESOURCE_ATTRIBUTES: 'foo', + OTEL_PROPAGATORS: 'foo', + OTEL_LOGS_EXPORTER: 'foo', + OTEL_SDK_DISABLED: 'foo' + } + }) + + return check(agent, proc, timeout, ({ payload }) => { + assert.strictEqual(payload.request_type, 'generate-metrics') + + const metrics = payload.payload + + assert.strictEqual(metrics.namespace, 'tracers') + + const otelHiding = metrics.series.filter(({ metric }) => metric === 'otel.env.hiding') + const otelInvalid = metrics.series.filter(({ metric }) => metric === 'otel.env.invalid') - assert.deepStrictEqual(otelInvalid[0].points[0][1], 1) + assert.strictEqual(otelHiding.length, 1) + assert.strictEqual(otelInvalid.length, 8) + + assert.deepStrictEqual(otelHiding[0].tags, [ + 'config.datadog:DD_TRACE_OTEL_ENABLED', 'config.opentelemetry:OTEL_SDK_DISABLED', + `version:${process.version}` + ]) assert.deepStrictEqual(otelInvalid[0].tags, [ - 'OTEL_LOGS_EXPORTER', + 'config.datadog:DD_TRACE_LOG_LEVEL', 'config.opentelemetry:OTEL_LOG_LEVEL', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[1].tags, [ + 'config.datadog:DD_TRACE_SAMPLE_RATE', + 'config.opentelemetry:OTEL_TRACES_SAMPLER', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[2].tags, [ + 'config.datadog:DD_TRACE_SAMPLE_RATE', + 'config.opentelemetry:OTEL_TRACES_SAMPLER_ARG', + `version:${process.version}` + ]) + assert.deepStrictEqual(otelInvalid[3].tags, [ + 'config.datadog:DD_TRACE_ENABLED', 'config.opentelemetry:OTEL_TRACES_EXPORTER', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[4].tags, [ + 'config.datadog:DD_RUNTIME_METRICS_ENABLED', + 'config.opentelemetry:OTEL_METRICS_EXPORTER', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[5].tags, [ + 'config.datadog:DD_TRACE_OTEL_ENABLED', 'config.opentelemetry:OTEL_SDK_DISABLED', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[6].tags, [ + 'config.opentelemetry:OTEL_LOGS_EXPORTER', + `version:${process.version}` + ]) + + assert.deepStrictEqual(otelInvalid[7].tags, [ + 'config.datadog:DD_TRACE_PROPAGATION_STYLE', + 'config.opentelemetry:OTEL_PROPAGATORS', `version:${process.version}` ]) + + for (const metric of otelInvalid) { + assert.strictEqual(metric.points[0][1], 1) + } }, true) }) diff --git a/integration-tests/package-guardrails.spec.js b/integration-tests/package-guardrails.spec.js new file mode 100644 index 00000000000..8fff27db3b2 --- /dev/null +++ b/integration-tests/package-guardrails.spec.js @@ -0,0 +1,95 @@ +const { + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd +} = require('./helpers') +const path = require('path') +const fs = require('fs') +const assert = require('assert') + +const NODE_OPTIONS = '--require dd-trace/init.js' +const DD_TRACE_DEBUG = 'true' +const DD_INJECTION_ENABLED = 'tracing' +const DD_LOG_LEVEL = 'error' + +describe('package guardrails', () => { + useEnv({ NODE_OPTIONS }) + const runTest = (...args) => + testFile('package-guardrails/index.js', ...args) + + context('when package is out of range', () => { + useSandbox(['bluebird@1.0.0']) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'abort.integration', 'integration:bluebird,integration_version:1.0.0' + )) + }) + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG }) + it('should not instrument the package', () => + runTest(`Application instrumentation bootstrapping complete +Found incompatible integration version: bluebird@1.0.0 +false +`)) + }) + }) + + context('when package is in range', () => { + context('when bluebird is 2.9.0', () => { + useSandbox(['bluebird@2.9.0']) + it('should instrument the package', () => runTest('true\n')) + }) + context('when bluebird is 3.7.2', () => { + useSandbox(['bluebird@3.7.2']) + it('should instrument the package', () => runTest('true\n')) + }) + }) + + context('when package errors out', () => { + useSandbox(['bluebird']) + before(() => { + const file = path.join(sandboxCwd(), 'node_modules/dd-trace/packages/datadog-instrumentations/src/bluebird.js') + fs.writeFileSync(file, ` +const { addHook } = require('./helpers/instrument') + +addHook({ name: 'bluebird', versions: ['*'] }, Promise => { + throw new ReferenceError('this is a test error') + return Promise +}) + `) + }) + + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'error', 'error_type:ReferenceError,integration:bluebird,integration_version:3.7.2' + )) + }) + + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG, DD_LOG_LEVEL }) + it('should not instrument the package', () => + runTest( + log => { + assert.ok(log.includes(` +Error during ddtrace instrumentation of application, aborting. +ReferenceError: this is a test error + at `)) + assert.ok(log.includes('\nfalse\n')) + })) + }) + }) +}) diff --git a/integration-tests/package-guardrails/index.js b/integration-tests/package-guardrails/index.js new file mode 100644 index 00000000000..efaf37abcd8 --- /dev/null +++ b/integration-tests/package-guardrails/index.js @@ -0,0 +1,8 @@ +'use strict' + +const P = require('bluebird') + +const isWrapped = P.prototype._then.toString().includes('AsyncResource') + +// eslint-disable-next-line no-console +console.log(isWrapped) diff --git a/integration-tests/playwright.config.js b/integration-tests/playwright.config.js index 7b57f6b4183..34b0a69a859 100644 --- a/integration-tests/playwright.config.js +++ b/integration-tests/playwright.config.js @@ -1,7 +1,7 @@ // Playwright config file for integration tests const { devices } = require('@playwright/test') -module.exports = { +const config = { baseURL: process.env.PW_BASE_URL, testDir: process.env.TEST_DIR || './ci-visibility/playwright-tests', timeout: Number(process.env.TEST_TIMEOUT) || 30000, @@ -17,3 +17,9 @@ module.exports = { ], testMatch: '**/*-test.js' } + +if (process.env.MAX_FAILURES) { + config.maxFailures = Number(process.env.MAX_FAILURES) +} + +module.exports = config diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 6cb8de941b5..aaf51f31d79 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -489,5 +489,32 @@ versions.forEach((version) => { }) }) } + + it('does not crash when maxFailures=1 and there is an error', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'failing-test-and-another-test.js.should work with failing tests', + 'failing-test-and-another-test.js.does not crash afterwards' + ]) + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + MAX_FAILURES: 1, + TEST_DIR: './ci-visibility/playwright-tests-max-failures' + }, + stdio: 'pipe' + } + ) + }) }) }) diff --git a/integration-tests/profiler/profiler.spec.js b/integration-tests/profiler/profiler.spec.js index 0803e2e9ae4..3b26448a2b1 100644 --- a/integration-tests/profiler/profiler.spec.js +++ b/integration-tests/profiler/profiler.spec.js @@ -110,7 +110,7 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args } }) - await processExitPromise(proc, 5000) + await processExitPromise(proc, 30000) const procEnd = BigInt(Date.now() * 1000000) const { profile, encoded } = await getLatestProfile(cwd, /^events_.+\.pprof$/) @@ -171,7 +171,7 @@ describe('profiler', () => { let oomTestFile let oomEnv let oomExecArgv - const timeout = 5000 + const timeout = 30000 before(async () => { sandbox = await createSandbox() @@ -201,7 +201,7 @@ describe('profiler', () => { } }) - await processExitPromise(proc, 5000) + await processExitPromise(proc, 30000) const procEnd = BigInt(Date.now() * 1000000) const { profile, encoded } = await getLatestProfile(cwd, /^wall_.+\.pprof$/) diff --git a/integration-tests/standalone-asm.spec.js b/integration-tests/standalone-asm.spec.js new file mode 100644 index 00000000000..d57a96f738e --- /dev/null +++ b/integration-tests/standalone-asm.spec.js @@ -0,0 +1,306 @@ +'use strict' + +const { assert } = require('chai') +const path = require('path') + +const { + createSandbox, + FakeAgent, + spawnProc, + curlAndAssertMessage, + curl +} = require('./helpers') + +describe('Standalone ASM', () => { + let sandbox, cwd, startupTestFile, agent, proc, env + + before(async () => { + sandbox = await createSandbox(['express']) + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'standalone-asm/index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + describe('enabled', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + + env = { + AGENT_PORT: agent.port, + DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED: 'true' + } + + const execArgv = [] + + proc = await spawnProc(startupTestFile, { cwd, env, execArgv }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + function assertKeep (payload, manual = true) { + const { meta, metrics } = payload + if (manual) { + assert.propertyVal(meta, 'manual.keep', 'true') + } else { + assert.notProperty(meta, 'manual.keep') + } + assert.propertyVal(meta, '_dd.p.appsec', '1') + + assert.propertyVal(metrics, '_sampling_priority_v1', 2) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + } + + function assertDrop (payload) { + const { metrics } = payload + assert.propertyVal(metrics, '_sampling_priority_v1', 0) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + assert.notProperty(metrics, '_dd.p.appsec') + } + + async function doWarmupRequests (procOrUrl, number = 3) { + for (let i = number; i > 0; i--) { + await curl(procOrUrl) + } + } + + // first req initializes the waf and reports the first appsec event adding manual.keep tag + it('should send correct headers and tags on first req', async () => { + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + assertKeep(payload[0][0]) + }) + }) + + it('should keep second req because RateLimiter allows 1 req/min and discard the next', async () => { + // 1st req kept because waf init + // 2nd req kept because it's the first one hitting RateLimiter + // next in the first minute are dropped + await doWarmupRequests(proc) + + return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + const secondReq = payload[1] + assert.isArray(secondReq) + assert.strictEqual(secondReq.length, 5) + + const { meta, metrics } = secondReq[0] + assert.notProperty(meta, 'manual.keep') + assert.notProperty(meta, '_dd.p.appsec') + + assert.propertyVal(metrics, '_sampling_priority_v1', 1) + assert.propertyVal(metrics, '_dd.apm.enabled', 0) + + assertDrop(payload[2][0]) + + assertDrop(payload[3][0]) + }) + }) + + it('should keep attack requests', async () => { + await doWarmupRequests(proc) + + const urlAttack = proc.url + '?query=1 or 1=1' + return curlAndAssertMessage(agent, urlAttack, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep sdk events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/login?user=test' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep custom sdk events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/sdk' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + assertKeep(payload[3][0]) + }) + }) + + it('should keep iast events', async () => { + await doWarmupRequests(proc) + + const url = proc.url + '/vulnerableHash' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + assert.strictEqual(payload.length, 4) + + const expressReq4 = payload[3][0] + assertKeep(expressReq4) + assert.property(expressReq4.meta, '_dd.iast.json') + assert.propertyVal(expressReq4.metrics, '_dd.iast.enabled', 1) + }) + }) + + describe('propagation', () => { + let proc2 + let port2 + + beforeEach(async () => { + const execArgv = [] + + proc2 = await spawnProc(startupTestFile, { cwd, env, execArgv }) + + port2 = parseInt(proc2.url.substring(proc2.url.lastIndexOf(':') + 1), 10) + }) + + afterEach(async () => { + proc2.kill() + }) + + // proc/drop-and-call-sdk: + // after setting a manual.drop calls to downstream proc2/sdk which triggers an appsec event + it('should keep trace even if parent prio is -1 but there is an event in the local trace', async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-after-drop-and-call-sdk?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /sdk') + assert.notStrictEqual(innerReq, undefined) + assertKeep(innerReq[0]) + }, undefined, undefined, true) + }) + + // proc/propagation-with-event triggers an appsec event and calls downstream proc2/down with no event + it('should keep if parent trace is (prio:2, _dd.p.appsec:1) but there is no event in the local trace', + async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-with-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assertKeep(innerReq[0], false) + }, undefined, undefined, true) + }) + + it('should remove parent trace data if there is no event in the local trace', async () => { + await doWarmupRequests(proc) + await doWarmupRequests(proc2) + + const url = `${proc.url}/propagation-without-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assert.notProperty(innerReq[0].meta, '_dd.p.other') + }, undefined, undefined, true) + }) + + it('should not remove parent trace data if there is event in the local trace', async () => { + await doWarmupRequests(proc) + + const url = `${proc.url}/propagation-with-event?port=${port2}` + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.propertyVal(headers, 'datadog-client-computed-stats', 'yes') + assert.isArray(payload) + + const innerReq = payload.find(p => p[0].resource === 'GET /down') + assert.notStrictEqual(innerReq, undefined) + assert.property(innerReq[0].meta, '_dd.p.other') + }, undefined, undefined, true) + }) + }) + }) + + describe('disabled', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + + env = { + AGENT_PORT: agent.port + } + + proc = await spawnProc(startupTestFile, { cwd, env }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should not add standalone related tags in iast events', () => { + const url = proc.url + '/vulnerableHash' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assert.notProperty(headers, 'datadog-client-computed-stats') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + const { meta, metrics } = payload[0][0] + assert.property(meta, '_dd.iast.json') // WEAK_HASH and XCONTENTTYPE_HEADER_MISSING reported + + assert.notProperty(meta, '_dd.p.appsec') + assert.notProperty(metrics, '_dd.apm.enabled') + }) + }) + + it('should not add standalone related tags in appsec events', () => { + const urlAttack = proc.url + '?query=1 or 1=1' + + return curlAndAssertMessage(agent, urlAttack, ({ headers, payload }) => { + assert.notProperty(headers, 'datadog-client-computed-stats') + assert.isArray(payload) + assert.strictEqual(payload.length, 1) + assert.isArray(payload[0]) + + // express.request + 4 middlewares + assert.strictEqual(payload[0].length, 5) + + const { meta, metrics } = payload[0][0] + assert.property(meta, '_dd.appsec.json') // crs-942-100 triggered + + assert.notProperty(meta, '_dd.p.appsec') + assert.notProperty(metrics, '_dd.apm.enabled') + }) + }) + }) +}) diff --git a/integration-tests/standalone-asm/index.js b/integration-tests/standalone-asm/index.js new file mode 100644 index 00000000000..e0c92d61ff1 --- /dev/null +++ b/integration-tests/standalone-asm/index.js @@ -0,0 +1,116 @@ +'use strict' + +const options = { + appsec: { + enabled: true + }, + experimental: { + iast: { + enabled: true, + requestSampling: 100 + } + } +} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.AGENT_URL) { + options.url = process.env.AGENT_URL +} + +const tracer = require('dd-trace') +tracer.init(options) + +const http = require('http') +const express = require('express') +const app = express() + +const valueToHash = 'iast-showcase-demo' +const crypto = require('crypto') + +async function makeRequest (url) { + return new Promise((resolve, reject) => { + http.get(url, function (res) { + const chunks = [] + + res.on('data', function (chunk) { + chunks.push(chunk) + }) + + res.on('end', () => { + resolve(Buffer.concat(chunks).toString('utf8')) + }) + + res.on('error', reject) + }) + }) +} + +app.get('/', (req, res) => { + res.status(200).send('hello world') +}) + +app.get('/login', (req, res) => { + tracer.appsec.trackUserLoginSuccessEvent({ id: req.query.user }) + res.status(200).send('login') +}) + +app.get('/sdk', (req, res) => { + tracer.appsec.trackCustomEvent('custom-event') + res.status(200).send('sdk') +}) + +app.get('/vulnerableHash', (req, res) => { + const result = crypto.createHash('sha1').update(valueToHash).digest('hex') + res.status(200).send(result) +}) + +app.get('/propagation-with-event', async (req, res) => { + tracer.appsec.trackCustomEvent('custom-event') + + const span = tracer.scope().active() + span.context()._trace.tags['_dd.p.other'] = '1' + + const port = req.query.port || server.address().port + const url = `http://localhost:${port}/down` + + await makeRequest(url) + + res.status(200).send('propagation-with-event') +}) + +app.get('/propagation-without-event', async (req, res) => { + const port = req.query.port || server.address().port + const url = `http://localhost:${port}/down` + + const span = tracer.scope().active() + span.context()._trace.tags['_dd.p.other'] = '1' + + await makeRequest(url) + + res.status(200).send('propagation-without-event') +}) + +app.get('/down', (req, res) => { + res.status(200).send('down') +}) + +app.get('/propagation-after-drop-and-call-sdk', async (req, res) => { + const span = tracer.scope().active() + span?.setTag('manual.drop', 'true') + + const port = req.query.port + + const url = `http://localhost:${port}/sdk` + + const sdkRes = await makeRequest(url) + + res.status(200).send(`drop-and-call-sdk ${sdkRes}`) +}) + +const server = http.createServer(app).listen(0, () => { + const port = server.address().port + process.send?.({ port }) +}) diff --git a/integration-tests/telemetry-forwarder.sh b/integration-tests/telemetry-forwarder.sh new file mode 100755 index 00000000000..5fe156993be --- /dev/null +++ b/integration-tests/telemetry-forwarder.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Implemented this in bash instead of Node.js for two reasons: +# 1. It's trivial in bash. +# 2. We're using NODE_OPTIONS in tests to init the tracer, and we don't want that for this script. + +echo "$1 $(cat -)" >> $FORWARDER_OUT diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs new file mode 100644 index 00000000000..f04d63785fd --- /dev/null +++ b/integration-tests/vitest.config.mjs @@ -0,0 +1,9 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + include: [ + 'ci-visibility/vitest-tests/test-visibility*' + ] + } +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js new file mode 100644 index 00000000000..4a3151c2b72 --- /dev/null +++ b/integration-tests/vitest/vitest.spec.js @@ -0,0 +1,137 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_STATUS, + TEST_TYPE +} = require('../../packages/dd-trace/src/plugins/util/test') + +// tested with 1.6.0 +const versions = ['latest'] + +versions.forEach((version) => { + describe(`vitest@${version}`, () => { + let sandbox, cwd, receiver, childProcess + + before(async function () { + sandbox = await createSandbox([`vitest@${version}`], true) + cwd = sandbox.folder + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + it('can run and report tests', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] + ) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + // maybe only in node@20 + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + }) +}) diff --git a/package.json b/package.json index 23176e03111..aba484be36b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "5.17.0", + "version": "5.18.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", @@ -14,34 +14,37 @@ "type:test": "cd docs && yarn && yarn test", "lint": "node scripts/check_licenses.js && eslint . && yarn audit --groups dependencies", "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services", - "test": "SERVICES=* yarn services && mocha --colors --exit --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'", - "test:appsec": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"", + "test": "SERVICES=* yarn services && mocha --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'", + "test:appsec": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" \"packages/dd-trace/test/appsec/**/*.spec.js\"", "test:appsec:ci": "nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" -- npm run test:appsec", - "test:appsec:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/appsec/**/*.@($(echo $PLUGINS)).plugin.spec.js\"", + "test:appsec:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/appsec/**/*.@($(echo $PLUGINS)).plugin.spec.js\"", "test:appsec:plugins:ci": "yarn services && nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" -- npm run test:appsec:plugins", "test:trace:core": "tap packages/dd-trace/test/*.spec.js \"packages/dd-trace/test/{ci-visibility,datastreams,encode,exporters,opentelemetry,opentracing,plugins,service-naming,telemetry}/**/*.spec.js\"", "test:trace:core:ci": "npm run test:trace:core -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/**/*.js\"", - "test:instrumentations": "mocha --colors -r 'packages/dd-trace/test/setup/mocha.js' 'packages/datadog-instrumentations/test/**/*.spec.js'", + "test:instrumentations": "mocha -r 'packages/dd-trace/test/setup/mocha.js' 'packages/datadog-instrumentations/test/**/*.spec.js'", "test:instrumentations:ci": "nyc --no-clean --include 'packages/datadog-instrumentations/src/**/*.js' -- npm run test:instrumentations", "test:core": "tap \"packages/datadog-core/test/**/*.spec.js\"", "test:core:ci": "npm run test:core -- --coverage --nyc-arg=--include=\"packages/datadog-core/src/**/*.js\"", - "test:lambda": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/lambda/**/*.spec.js\"", + "test:lambda": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/lambda/**/*.spec.js\"", "test:lambda:ci": "nyc --no-clean --include \"packages/dd-trace/src/lambda/**/*.js\" -- npm run test:lambda", - "test:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"", + "test:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"", "test:plugins:ci": "yarn services && nyc --no-clean --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS)).js\" --include \"packages/datadog-instrumentations/src/@($(echo $PLUGINS))/**/*.js\" --include \"packages/datadog-plugin-@($(echo $PLUGINS))/src/**/*.js\" -- npm run test:plugins", "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js", "test:profiler": "tap \"packages/dd-trace/test/profiling/**/*.spec.js\"", "test:profiler:ci": "npm run test:profiler -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/profiling/**/*.js\"", - "test:integration": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/*.spec.js\"", - "test:integration:cucumber": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cucumber/*.spec.js\"", - "test:integration:cypress": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", - "test:integration:playwright": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", - "test:integration:selenium": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", - "test:integration:profiler": "mocha --colors --timeout 90000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", - "test:integration:serverless": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", - "test:integration:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", - "test:unit:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\" --exclude \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", - "test:shimmer": "mocha --colors 'packages/datadog-shimmer/test/**/*.spec.js'", + "test:integration": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/*.spec.js\"", + "test:integration:cucumber": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cucumber/*.spec.js\"", + "test:integration:cypress": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", + "test:integration:jest": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/jest/*.spec.js\"", + "test:integration:mocha": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/mocha/*.spec.js\"", + "test:integration:playwright": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", + "test:integration:selenium": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", + "test:integration:vitest": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/vitest/*.spec.js\"", + "test:integration:profiler": "mocha --timeout 180000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", + "test:integration:serverless": "mocha --timeout 60000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", + "test:integration:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", + "test:unit:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\" --exclude \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", + "test:shimmer": "mocha 'packages/datadog-shimmer/test/**/*.spec.js'", "test:shimmer:ci": "nyc --no-clean --include 'packages/datadog-shimmer/src/**/*.js' -- npm run test:shimmer", "leak:core": "node ./scripts/install_plugin_modules && (cd packages/memwatch && yarn) && NODE_PATH=./packages/memwatch/node_modules node --no-warnings ./node_modules/.bin/tape 'packages/dd-trace/test/leak/**/*.js'", "leak:plugins": "yarn services && (cd packages/memwatch && yarn) && NODE_PATH=./packages/memwatch/node_modules node --no-warnings ./node_modules/.bin/tape \"packages/datadog-plugin-@($(echo $PLUGINS))/test/leak.js\"" @@ -72,7 +75,7 @@ "dependencies": { "@datadog/native-appsec": "8.0.1", "@datadog/native-iast-rewriter": "2.3.1", - "@datadog/native-iast-taint-tracking": "2.1.0", + "@datadog/native-iast-taint-tracking": "3.0.0", "@datadog/native-metrics": "^2.0.0", "@datadog/pprof": "5.3.0", "@datadog/sketches-js": "^2.1.0", @@ -81,7 +84,7 @@ "crypto-randomuuid": "^1.0.0", "dc-polyfill": "^0.1.4", "ignore": "^5.2.4", - "import-in-the-middle": "^1.7.4", + "import-in-the-middle": "^1.8.1", "int64-buffer": "^0.1.9", "istanbul-lib-coverage": "3.2.0", "jest-docblock": "^29.7.0", @@ -136,6 +139,7 @@ "sinon": "^15.2.0", "sinon-chai": "^3.7.0", "tap": "^16.3.7", - "tape": "^5.6.5" + "tape": "^5.6.5", + "tiktoken": "^1.0.15" } } diff --git a/packages/datadog-core/src/storage/async_hooks.js b/packages/datadog-core/src/storage/async_hooks.js deleted file mode 100644 index d8e71e1df2d..00000000000 --- a/packages/datadog-core/src/storage/async_hooks.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict' - -const { executionAsyncId } = require('async_hooks') -const AsyncResourceStorage = require('./async_resource') - -class AsyncHooksStorage extends AsyncResourceStorage { - constructor () { - super() - - this._resources = new Map() - } - - disable () { - super.disable() - - this._resources.clear() - } - - _createHook () { - return { - ...super._createHook(), - destroy: this._destroy.bind(this) - } - } - - _init (asyncId, type, triggerAsyncId, resource) { - super._init.apply(this, arguments) - - this._resources.set(asyncId, resource) - } - - _destroy (asyncId) { - this._resources.delete(asyncId) - } - - _executionAsyncResource () { - const asyncId = executionAsyncId() - - let resource = this._resources.get(asyncId) - - if (!resource) { - this._resources.set(asyncId, resource = {}) - } - - return resource - } -} - -module.exports = AsyncHooksStorage diff --git a/packages/datadog-core/src/storage/index.js b/packages/datadog-core/src/storage/index.js index 0d48defbc3c..e522e61ced2 100644 --- a/packages/datadog-core/src/storage/index.js +++ b/packages/datadog-core/src/storage/index.js @@ -2,13 +2,4 @@ // TODO: default to AsyncLocalStorage when it supports triggerAsyncResource -const semver = require('semver') - -// https://github.com/nodejs/node/pull/33801 -const hasJavaScriptAsyncHooks = semver.satisfies(process.versions.node, '>=14.5') - -if (hasJavaScriptAsyncHooks) { - module.exports = require('./async_resource') -} else { - module.exports = require('./async_hooks') -} +module.exports = require('./async_resource') diff --git a/packages/datadog-core/test/storage/async_hooks.spec.js b/packages/datadog-core/test/storage/async_hooks.spec.js deleted file mode 100644 index dc990ab94f4..00000000000 --- a/packages/datadog-core/test/storage/async_hooks.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -require('../setup') - -const StorageBackend = require('../../src/storage/async_hooks') -const testStorage = require('./test') - -describe('storage/async_hooks', () => { - let storage - - beforeEach(() => { - storage = new StorageBackend() - }) - - afterEach(() => { - storage.disable() - }) - - testStorage(() => storage) -}) diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 95a0e8ddd16..ce263799023 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -7,7 +7,11 @@ const hooks = require('../datadog-instrumentations/src/helpers/hooks.js') const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path') for (const hook of Object.values(hooks)) { - hook() + if (typeof hook === 'object') { + hook.fn() + } else { + hook() + } } const modulesOfInterest = new Set() diff --git a/packages/datadog-instrumentations/src/aws-sdk.js b/packages/datadog-instrumentations/src/aws-sdk.js index 8ea5552fe1a..5a2efd00681 100644 --- a/packages/datadog-instrumentations/src/aws-sdk.js +++ b/packages/datadog-instrumentations/src/aws-sdk.js @@ -20,7 +20,8 @@ function wrapRequest (send) { return innerAr.runInAsyncScope(() => { this.on('complete', innerAr.bind(response => { - channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response }) + const cbExists = typeof cb === 'function' + channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response, cbExists }) })) startCh.publish({ diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index b1a07d8781c..3e713ad89bb 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -1,11 +1,13 @@ 'use strict' const { createCoverageMap } = require('istanbul-lib-coverage') +const { NUM_FAILED_TEST_RETRIES } = require('../../dd-trace/src/plugins/util/test') const { addHook, channel, AsyncResource } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') const log = require('../../dd-trace/src/log') const testStartCh = channel('ci:cucumber:test:start') +const testRetryCh = channel('ci:cucumber:test:retry') const testFinishCh = channel('ci:cucumber:test:finish') // used for test steps too const testStepStartCh = channel('ci:cucumber:test-step:start') @@ -47,6 +49,7 @@ const patched = new WeakSet() const lastStatusByPickleId = new Map() const numRetriesByPickleId = new Map() +const numAttemptToAsyncResource = new Map() let pickleByFile = {} const pickleResultByFile = {} @@ -60,6 +63,7 @@ let isUnskippable = false let isSuitesSkippingEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 +let isFlakyTestRetriesEnabled = false let knownTests = [] let skippedSuites = [] let isSuitesSkipped = false @@ -162,47 +166,80 @@ function wrapRun (pl, isLatestVersion) { return run.apply(this, arguments) } + let numAttempt = 0 + const asyncResource = new AsyncResource('bound-anonymous-fn') - return asyncResource.runInAsyncScope(() => { - const testFileAbsolutePath = this.pickle.uri - const testSourceLine = this.gherkinDocument?.feature?.location?.line + numAttemptToAsyncResource.set(numAttempt, asyncResource) - testStartCh.publish({ - testName: this.pickle.name, - testFileAbsolutePath, - testSourceLine, - isParallel: !!process.env.CUCUMBER_WORKER_ID - }) - try { - const promise = run.apply(this, arguments) - promise.finally(() => { - const result = this.getWorstStepResult() - const { status, skipReason, errorMessage } = isLatestVersion - ? getStatusFromResultLatest(result) - : getStatusFromResult(result) + const testFileAbsolutePath = this.pickle.uri - if (lastStatusByPickleId.has(this.pickle.id)) { - lastStatusByPickleId.get(this.pickle.id).push(status) - } else { - lastStatusByPickleId.set(this.pickle.id, [status]) - } - let isNew = false - let isEfdRetry = false - if (isEarlyFlakeDetectionEnabled && status !== 'skip') { - const numRetries = numRetriesByPickleId.get(this.pickle.id) + const testSourceLine = this.gherkinDocument?.feature?.location?.line - isNew = numRetries !== undefined - isEfdRetry = numRetries > 0 + const testStartPayload = { + testName: this.pickle.name, + testFileAbsolutePath, + testSourceLine, + isParallel: !!process.env.CUCUMBER_WORKER_ID + } + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testStartPayload) + }) + try { + this.eventBroadcaster.on('envelope', (testCase) => { + // Only supported from >=8.0.0 + if (testCase?.testCaseFinished) { + const { testCaseFinished: { willBeRetried } } = testCase + if (willBeRetried) { // test case failed and will be retried + const failedAttemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) + failedAttemptAsyncResource.runInAsyncScope(() => { + testRetryCh.publish(numAttempt++ > 0) // the current span will be finished and a new one will be created + }) + + const newAsyncResource = new AsyncResource('bound-anonymous-fn') + numAttemptToAsyncResource.set(numAttempt, newAsyncResource) + + newAsyncResource.runInAsyncScope(() => { + testStartCh.publish(testStartPayload) // a new span will be created + }) } - testFinishCh.publish({ status, skipReason, errorMessage, isNew, isEfdRetry }) + } + }) + let promise + + asyncResource.runInAsyncScope(() => { + promise = run.apply(this, arguments) + }) + promise.finally(() => { + const result = this.getWorstStepResult() + const { status, skipReason, errorMessage } = isLatestVersion + ? getStatusFromResultLatest(result) + : getStatusFromResult(result) + + if (lastStatusByPickleId.has(this.pickle.id)) { + lastStatusByPickleId.get(this.pickle.id).push(status) + } else { + lastStatusByPickleId.set(this.pickle.id, [status]) + } + let isNew = false + let isEfdRetry = false + if (isEarlyFlakeDetectionEnabled && status !== 'skip') { + const numRetries = numRetriesByPickleId.get(this.pickle.id) + + isNew = numRetries !== undefined + isEfdRetry = numRetries > 0 + } + const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) + + attemptAsyncResource.runInAsyncScope(() => { + testFinishCh.publish({ status, skipReason, errorMessage, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 }) }) - return promise - } catch (err) { - errorCh.publish(err) - throw err - } - }) + }) + return promise + } catch (err) { + errorCh.publish(err) + throw err + } }) shimmer.wrap(pl.prototype, 'runStep', runStep => function () { if (!testStepStartCh.hasSubscribers) { @@ -267,6 +304,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false) { isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled + isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled if (isEarlyFlakeDetectionEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) @@ -304,6 +342,10 @@ function getWrappedStart (start, frameworkVersion, isParallel = false) { const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` + if (isFlakyTestRetriesEnabled && !this.options.retry) { + this.options.retry = NUM_FAILED_TEST_RETRIES + } + sessionAsyncResource.runInAsyncScope(() => { sessionStartCh.publish({ command, frameworkVersion }) }) diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 7bec453187a..0177744ea1c 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -11,8 +11,13 @@ const ritm = require('../../../dd-trace/src/ritm') * @param {string[]} modules list of modules to hook into * @param {Function} onrequire callback to be executed upon encountering module */ -function Hook (modules, onrequire) { - if (!(this instanceof Hook)) return new Hook(modules, onrequire) +function Hook (modules, hookOptions, onrequire) { + if (!(this instanceof Hook)) return new Hook(modules, hookOptions, onrequire) + + if (typeof hookOptions === 'function') { + onrequire = hookOptions + hookOptions = {} + } this._patched = Object.create(null) @@ -28,7 +33,7 @@ function Hook (modules, onrequire) { } this._ritmHook = ritm(modules, {}, safeHook) - this._iitmHook = iitm(modules, {}, (moduleExports, moduleName, moduleBaseDir) => { + this._iitmHook = iitm(modules, hookOptions, (moduleExports, moduleName, moduleBaseDir) => { // TODO: Move this logic to import-in-the-middle and only do it for CommonJS // modules and not ESM. In the meantime, all the modules we instrument are // CommonJS modules for which the default export is always moved to diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 0723ceabd84..94f3318fb62 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -23,6 +23,7 @@ module.exports = { '@opentelemetry/sdk-trace-node': () => require('../otel-sdk-trace'), '@redis/client': () => require('../redis'), '@smithy/smithy-client': () => require('../aws-sdk'), + '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, aerospike: () => require('../aerospike'), amqp10: () => require('../amqp10'), amqplib: () => require('../amqplib'), @@ -110,6 +111,7 @@ module.exports = { sharedb: () => require('../sharedb'), tedious: () => require('../tedious'), undici: () => require('../undici'), + vitest: { esmFirst: true, fn: () => require('../vitest') }, when: () => require('../when'), winston: () => require('../winston') } diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 0889f1e5402..20657335044 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -17,10 +17,11 @@ exports.channel = function (name) { /** * @param {string} args.name module name * @param {string[]} args.versions array of semver range strings - * @param {string} args.file path to file within package to instrument? + * @param {string} args.file path to file within package to instrument + * @param {string} args.filePattern pattern to match files within package to instrument * @param Function hook */ -exports.addHook = function addHook ({ name, versions, file }, hook) { +exports.addHook = function addHook ({ name, versions, file, filePattern }, hook) { if (typeof name === 'string') { name = [name] } @@ -29,7 +30,7 @@ exports.addHook = function addHook ({ name, versions, file }, hook) { if (!instrumentations[val]) { instrumentations[val] = [] } - instrumentations[val].push({ name: val, versions, file, hook }) + instrumentations[val].push({ name: val, versions, file, filePattern, hook }) } } diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index eba90d6a980..e45a0c0cd14 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -7,6 +7,7 @@ const Hook = require('./hook') const requirePackageJson = require('../../../dd-trace/src/require-package-json') const log = require('../../../dd-trace/src/log') const checkRequireCache = require('../check_require_cache') +const telemetry = require('../../../dd-trace/src/telemetry/init-telemetry') const { DD_TRACE_DISABLED_INSTRUMENTATIONS = '', @@ -35,22 +36,38 @@ if (DD_TRACE_DEBUG && DD_TRACE_DEBUG.toLowerCase() !== 'false') { setImmediate(checkRequireCache.checkForPotentialConflicts) } +const seenCombo = new Set() + // TODO: make this more efficient for (const packageName of names) { if (disabledInstrumentations.has(packageName)) continue - Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { + const hookOptions = {} + + let hook = hooks[packageName] + + if (typeof hook === 'object') { + hookOptions.internals = hook.esmFirst + hook = hook.fn + } + + Hook([packageName], hookOptions, (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { moduleName = moduleName.replace(pathSepExpr, '/') // This executes the integration file thus adding its entries to `instrumentations` - hooks[packageName]() + hook() if (!instrumentations[packageName]) { return moduleExports } - for (const { name, file, versions, hook } of instrumentations[packageName]) { + const namesAndSuccesses = {} + for (const { name, file, versions, hook, filePattern } of instrumentations[packageName]) { + let fullFilePattern = filePattern const fullFilename = filename(name, file) + if (fullFilePattern) { + fullFilePattern = filename(name, fullFilePattern) + } // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once // for example by instrumenting both dns and node:dns double the spans would be created @@ -58,13 +75,29 @@ for (const packageName of names) { if (!hook[HOOK_SYMBOL]) { hook[HOOK_SYMBOL] = new WeakMap() } + let matchesFile = false + + matchesFile = moduleName === fullFilename - if (moduleName === fullFilename) { + if (fullFilePattern) { + // Some libraries include a hash in their filenames when installed, + // so our instrumentation has to include a '.*' to match them for more than a single version. + matchesFile = matchesFile || new RegExp(fullFilePattern).test(moduleName) + } + + if (matchesFile) { const version = moduleVersion || getVersion(moduleBaseDir) + if (!Object.hasOwnProperty(namesAndSuccesses, name)) { + namesAndSuccesses[name] = { + success: false, + version + } + } if (matchVersion(version, versions)) { // Check if the hook already has a set moduleExport if (hook[HOOK_SYMBOL].has(moduleExports)) { + namesAndSuccesses[name].success = true return moduleExports } @@ -76,11 +109,29 @@ for (const packageName of names) { // Set the moduleExports in the hooks weakmap hook[HOOK_SYMBOL].set(moduleExports, name) } catch (e) { - log.error(e) + log.info('Error during ddtrace instrumentation of application, aborting.') + log.info(e) + telemetry('error', [ + `error_type:${e.constructor.name}`, + `integration:${name}`, + `integration_version:${version}` + ]) } + namesAndSuccesses[name].success = true } } } + for (const name of Object.keys(namesAndSuccesses)) { + const { success, version } = namesAndSuccesses[name] + if (!success && !seenCombo.has(`${name}@${version}`)) { + telemetry('abort.integration', [ + `integration:${name}`, + `integration_version:${version}` + ]) + log.info(`Found incompatible integration version: ${name}@${version}`) + seenCombo.add(`${name}@${version}`) + } + } return moduleExports }) diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index fbf8ca88a9b..9c4956eed0b 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -21,6 +21,7 @@ const { runnableWrapper, getOnTestHandler, getOnTestEndHandler, + getOnTestRetryHandler, getOnHookEndHandler, getOnFailHandler, getOnPendingHandler, @@ -37,10 +38,12 @@ let isSuitesSkipped = false let skippedSuites = [] let isEarlyFlakeDetectionEnabled = false let isSuitesSkippingEnabled = false +let isFlakyTestRetriesEnabled = false let earlyFlakeDetectionNumRetries = 0 let knownTests = [] let itrCorrelationId = '' let isForcedToRun = false +const config = {} // We'll preserve the original coverage here const originalCoverageMap = createCoverageMap() @@ -227,6 +230,12 @@ addHook({ isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled + + config.isEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled + config.isSuitesSkippingEnabled = isSuitesSkippingEnabled + config.earlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + config.isFlakyTestRetriesEnabled = isFlakyTestRetriesEnabled if (isEarlyFlakeDetectionEnabled) { knownTestsCh.publish({ @@ -317,6 +326,8 @@ addHook({ this.on('test end', getOnTestEndHandler()) + this.on('retry', getOnTestRetryHandler()) + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted this.on('hook end', getOnHookEndHandler()) @@ -401,7 +412,7 @@ addHook({ name: 'mocha', versions: ['>=5.2.0'], file: 'lib/runnable.js' -}, runnableWrapper) +}, (runnablePackage) => runnableWrapper(runnablePackage, config)) // Only used in parallel mode (--parallel flag is passed) // Used to generate suite events and receive test payloads from workers diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 254f3be5860..02e49ee810b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -3,7 +3,8 @@ const { getTestSuitePath, removeEfdStringFromTestName, - addEfdStringToTestName + addEfdStringToTestName, + NUM_FAILED_TEST_RETRIES } = require('../../../dd-trace/src/plugins/util/test') const { channel, AsyncResource } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') @@ -11,6 +12,8 @@ const shimmer = require('../../../datadog-shimmer') // test channels const testStartCh = channel('ci:mocha:test:start') const testFinishCh = channel('ci:mocha:test:finish') +// after a test has failed, we'll publish to this channel +const testRetryCh = channel('ci:mocha:test:retry') const errorCh = channel('ci:mocha:test:error') const skipCh = channel('ci:mocha:test:skip') @@ -70,6 +73,10 @@ function isMochaRetry (test) { return test._currentRetry !== undefined && test._currentRetry !== 0 } +function isLastRetry (test) { + return test._currentRetry === test._retries +} + function getTestFullName (test) { return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` } @@ -84,22 +91,34 @@ function getTestStatus (test) { return 'pass' } -function getTestAsyncResource (test) { +function getTestToArKey (test) { if (!test.fn) { - return testToAr.get(test) + return test } if (!wrappedFunctions.has(test.fn)) { - return testToAr.get(test.fn) + return test.fn } const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) + return originalFn +} + +function getTestAsyncResource (test) { + const key = getTestToArKey(test) + return testToAr.get(key) } -function runnableWrapper (RunnablePackage) { +function runnableWrapper (RunnablePackage, libraryConfig) { shimmer.wrap(RunnablePackage.prototype, 'run', run => function () { if (!testStartCh.hasSubscribers) { return run.apply(this, arguments) } + // Flaky test retries does not work in parallel mode + if (libraryConfig?.isFlakyTestRetriesEnabled) { + this.retries(NUM_FAILED_TEST_RETRIES) + } + // The reason why the wrapping logic is here is because we need to cover + // `afterEach` and `beforeEach` hooks as well. + // It can't be done in `getOnTestHandler` because it's only called for tests. const isBeforeEach = this.parent._beforeEach.includes(this) const isAfterEach = this.parent._afterEach.includes(this) @@ -135,11 +154,16 @@ function runnableWrapper (RunnablePackage) { function getOnTestHandler (isMain, newTests) { return function (test) { - if (isMochaRetry(test)) { - return - } const testStartLine = testToStartLine.get(test) const asyncResource = new AsyncResource('bound-anonymous-fn') + + // This may be a retry. If this is the case, `test.fn` is already wrapped, + // so we need to restore it. + if (wrappedFunctions.has(test.fn)) { + const originalFn = originalFns.get(test.fn) + test.fn = originalFn + wrappedFunctions.delete(test.fn) + } testToAr.set(test.fn, asyncResource) const { @@ -186,7 +210,7 @@ function getOnTestEndHandler () { // if there are afterEach to be run, we don't finish the test yet if (asyncResource && !test.parent._afterEach.length) { asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) + testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test) }) }) } } @@ -197,12 +221,17 @@ function getOnHookEndHandler () { const test = hook.ctx.currentTest if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (test._retries > 0 && !isLastRetry(test)) { + return + } if (isLastAfterEach) { const status = getTestStatus(test) const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish({ status, hasBeenRetried: isMochaRetry(test) }) + }) + } } } } @@ -226,7 +255,7 @@ function getOnFailHandler (isMain) { err.message = `${testOrHook.fullTitle()}: ${err.message}` errorCh.publish(err) // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') + testFinishCh.publish({ status: 'fail', hasBeenRetried: isMochaRetry(test) }) } else { errorCh.publish(err) } @@ -250,6 +279,20 @@ function getOnFailHandler (isMain) { } } +function getOnTestRetryHandler () { + return function (test) { + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + const isFirstAttempt = test._currentRetry === 0 + asyncResource.runInAsyncScope(() => { + testRetryCh.publish(isFirstAttempt) + }) + } + const key = getTestToArKey(test) + testToAr.delete(key) + } +} + function getOnPendingHandler () { return function (test) { const testStartLine = testToStartLine.get(test) @@ -299,6 +342,7 @@ module.exports = { testToStartLine, getOnTestHandler, getOnTestEndHandler, + getOnTestRetryHandler, getOnHookEndHandler, getOnFailHandler, getOnPendingHandler, diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index c2fa26f1504..fadd8f80a6e 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -49,3 +49,4 @@ addHook({ versions: ['>=5.2.0'], file: 'lib/runnable.js' }, runnableWrapper) +// TODO: parallel mode does not support flaky test retries, so no library config is passed. diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index c77b078bdac..baf82768681 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -249,7 +249,7 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { testAsyncResource.runInAsyncScope(() => { testFinishCh.publish({ testStatus, - steps: testResult.steps, + steps: testResult?.steps || [], error, extraTags: annotationTags, isNew: test._ddIsNew, diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js new file mode 100644 index 00000000000..0513d31a5d2 --- /dev/null +++ b/packages/datadog-instrumentations/src/vitest.js @@ -0,0 +1,303 @@ +const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +// test hooks +const testStartCh = channel('ci:vitest:test:start') +const testFinishTimeCh = channel('ci:vitest:test:finish-time') +const testPassCh = channel('ci:vitest:test:pass') +const testErrorCh = channel('ci:vitest:test:error') +const testSkipCh = channel('ci:vitest:test:skip') + +// test suite hooks +const testSuiteStartCh = channel('ci:vitest:test-suite:start') +const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') +const testSuiteErrorCh = channel('ci:vitest:test-suite:error') + +// test session hooks +const testSessionStartCh = channel('ci:vitest:session:start') +const testSessionFinishCh = channel('ci:vitest:session:finish') + +const taskToAsync = new WeakMap() + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +function isReporterPackage (vitestPackage) { + return vitestPackage.B?.name === 'BaseSequencer' +} + +// from 2.0.0 +function isReporterPackageNew (vitestPackage) { + return vitestPackage.e?.name === 'BaseSequencer' +} + +function getSessionStatus (state) { + if (state.getCountOfFailedTests() > 0) { + return 'fail' + } + if (state.pathsSet.size === 0) { + return 'skip' + } + return 'pass' +} + +// eslint-disable-next-line +// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 +function getVitestTestStatus (test, retryCount) { + if (test.result.state !== 'fail') { + if (!test.repeats) { + return 'pass' + } else if (test.repeats && (test.retry ?? 0) === retryCount) { + return 'pass' + } + } + return 'fail' +} + +function getTypeTasks (fileTasks, type = 'test') { + const typeTasks = [] + + function getTasks (tasks) { + for (const task of tasks) { + if (task.type === type) { + typeTasks.push(task) + } else if (task.tasks) { + getTasks(task.tasks) + } + } + } + + getTasks(fileTasks) + + return typeTasks +} + +function getTestName (task) { + let testName = task.name + let currentTask = task.suite + + while (currentTask) { + if (currentTask.name) { + testName = `${currentTask.name} ${testName}` + } + currentTask = currentTask.suite + } + + return testName +} + +function getSortWrapper (sort) { + return async function () { + if (!testSessionFinishCh.hasSubscribers) { + return sort.apply(this, arguments) + } + shimmer.wrap(this.ctx, 'exit', exit => async function () { + let onFinish + + const flushPromise = new Promise(resolve => { + onFinish = resolve + }) + const failedSuites = this.state.getFailedFilepaths() + let error + if (failedSuites.length) { + error = new Error(`Test suites failed: ${failedSuites.length}.`) + } + + sessionAsyncResource.runInAsyncScope(() => { + testSessionFinishCh.publish({ + status: getSessionStatus(this.state), + onFinish, + error + }) + }) + + await flushPromise + + return exit.apply(this, arguments) + }) + + return sort.apply(this, arguments) + } +} + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + file: 'dist/runners.js' +}, (vitestPackage) => { + const { VitestTestRunner } = vitestPackage + // test start (only tests that are not marked as skip or todo) + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task) { + if (!testStartCh.hasSubscribers) { + return onBeforeTryTask.apply(this, arguments) + } + const asyncResource = new AsyncResource('bound-anonymous-fn') + taskToAsync.set(task, asyncResource) + + asyncResource.runInAsyncScope(() => { + testStartCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + }) + return onBeforeTryTask.apply(this, arguments) + }) + + // test finish (only passed tests) + shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => + async function (task, { retry: retryCount }) { + if (!testFinishTimeCh.hasSubscribers) { + return onAfterTryTask.apply(this, arguments) + } + const result = await onAfterTryTask.apply(this, arguments) + + const status = getVitestTestStatus(task, retryCount) + const asyncResource = taskToAsync.get(task) + + if (asyncResource) { + // We don't finish here because the test might fail in a later hook + asyncResource.runInAsyncScope(() => { + testFinishTimeCh.publish({ status, task }) + }) + } + + return result + }) + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=2.0.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + // there are multiple index* files so we have to check the exported values + if (isReporterPackageNew(vitestPackage)) { + shimmer.wrap(vitestPackage.e.prototype, 'sort', getSortWrapper) + } + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + // there are multiple index* files so we have to check the exported values + if (isReporterPackage(vitestPackage)) { + shimmer.wrap(vitestPackage.B.prototype, 'sort', getSortWrapper) + } + + return vitestPackage +}) + +// Can't specify file because compiled vitest includes hashes in their files +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/cac.*' +}, (vitestPackage, frameworkVersion) => { + shimmer.wrap(vitestPackage, 'c', oldCreateCli => function () { + if (!testSessionStartCh.hasSubscribers) { + return oldCreateCli.apply(this, arguments) + } + sessionAsyncResource.runInAsyncScope(() => { + const processArgv = process.argv.slice(2).join(' ') + testSessionStartCh.publish({ command: `vitest ${processArgv}`, frameworkVersion }) + }) + return oldCreateCli.apply(this, arguments) + }) + + return vitestPackage +}) + +// test suite start and finish +// only relevant for workers +addHook({ + name: '@vitest/runner', + versions: ['>=1.6.0'], + file: 'dist/index.js' +}, vitestPackage => { + shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPath) { + let testSuiteError = null + if (!testSuiteStartCh.hasSubscribers) { + return startTests.apply(this, arguments) + } + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish(testPath[0]) + }) + const startTestsResponse = await startTests.apply(this, arguments) + + let onFinish = null + const onFinishPromise = new Promise(resolve => { + onFinish = resolve + }) + + const testTasks = getTypeTasks(startTestsResponse[0].tasks) + + testTasks.forEach(task => { + const testAsyncResource = taskToAsync.get(task) + const { result } = task + + if (result) { + const { state, duration, errors } = result + if (state === 'skip') { // programmatic skip + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } else if (state === 'pass') { + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } else if (state === 'fail') { + // If it's failing, we have no accurate finish time, so we have to use `duration` + let testError + + if (errors?.length) { + testError = errors[0] + } + + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testErrorCh.publish({ duration, error: testError }) + }) + } + if (errors?.length) { + testSuiteError = testError // we store the error to bubble it up to the suite + } + } + } else { // test.skip or test.todo + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } + }) + + const testSuiteResult = startTestsResponse[0].result + + if (testSuiteResult.errors?.length) { // Errors from root level hooks + testSuiteError = testSuiteResult.errors[0] + } else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks + const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite') + const failedSuites = suiteTasks.filter(task => task.result?.state === 'fail') + if (failedSuites.length && failedSuites[0].result?.errors?.length) { + testSuiteError = failedSuites[0].result.errors[0] + } + } + + if (testSuiteError) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish({ error: testSuiteError }) + }) + } + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish({ status: testSuiteResult.state, onFinish }) + }) + + // TODO: fix too frequent flushes + await onFinishPromise + + return startTestsResponse + }) + + return vitestPackage +}) diff --git a/packages/datadog-instrumentations/test/body-parser.spec.js b/packages/datadog-instrumentations/test/body-parser.spec.js index d502bc00ea6..23c7388f2dd 100644 --- a/packages/datadog-instrumentations/test/body-parser.spec.js +++ b/packages/datadog-instrumentations/test/body-parser.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') @@ -22,11 +21,9 @@ withVersions('body-parser', 'body-parser', version => { middlewareProcessBodyStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/cookie-parser.spec.js b/packages/datadog-instrumentations/test/cookie-parser.spec.js index 4137ddbef63..14afe44ba90 100644 --- a/packages/datadog-instrumentations/test/cookie-parser.spec.js +++ b/packages/datadog-instrumentations/test/cookie-parser.spec.js @@ -1,7 +1,6 @@ 'use strict' const { assert } = require('chai') -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') @@ -23,11 +22,9 @@ withVersions('cookie-parser', 'cookie-parser', version => { middlewareProcessCookieStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js index 7464f83152a..3fcf981e528 100644 --- a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js +++ b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const { channel } = require('dc-polyfill') const axios = require('axios') describe('express-mongo-sanitize', () => { @@ -25,11 +24,9 @@ describe('express-mongo-sanitize', () => { res.end() }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) diff --git a/packages/datadog-instrumentations/test/express.spec.js b/packages/datadog-instrumentations/test/express.spec.js index 88f75164be6..ff9f577bc8d 100644 --- a/packages/datadog-instrumentations/test/express.spec.js +++ b/packages/datadog-instrumentations/test/express.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -20,11 +19,9 @@ withVersions('express', 'express', version => { requestBody() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/passport-http.spec.js b/packages/datadog-instrumentations/test/passport-http.spec.js index 68b06abbe5c..10b2cd292a0 100644 --- a/packages/datadog-instrumentations/test/passport-http.spec.js +++ b/packages/datadog-instrumentations/test/passport-http.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -70,11 +69,9 @@ withVersions('passport-http', 'passport-http', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(() => { diff --git a/packages/datadog-instrumentations/test/passport-local.spec.js b/packages/datadog-instrumentations/test/passport-local.spec.js index f31c7a83230..92ffe9bb1d8 100644 --- a/packages/datadog-instrumentations/test/passport-local.spec.js +++ b/packages/datadog-instrumentations/test/passport-local.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -71,11 +70,9 @@ withVersions('passport-local', 'passport-local', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(() => { diff --git a/packages/datadog-plugin-apollo/test/index.spec.js b/packages/datadog-plugin-apollo/test/index.spec.js index 6718098ef28..5bf25e4e428 100644 --- a/packages/datadog-plugin-apollo/test/index.spec.js +++ b/packages/datadog-plugin-apollo/test/index.spec.js @@ -5,7 +5,6 @@ const agent = require('../../dd-trace/test/plugins/agent.js') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants.js') const { expectedSchema, rawExpectedSchema } = require('./naming.js') const axios = require('axios') -const getPort = require('get-port') const accounts = require('./fixtures.js') @@ -86,13 +85,15 @@ describe('Plugin', () => { gateway: setupGateway(), subscriptions: false // Disable subscriptions (not supported with Apollo Gateway) }) - getPort().then(newPort => { - port = newPort - startStandaloneServer(server, { - listen: { port } - }).then(() => {}) + + return startStandaloneServer(server, { + listen: { port: 0 } + }).then(({ url }) => { + port = new URL(url).port }) + }) + before(() => { return agent.load('apollo') }) diff --git a/packages/datadog-plugin-aws-sdk/src/base.js b/packages/datadog-plugin-aws-sdk/src/base.js index 21e4bfa47f6..21c48831f92 100644 --- a/packages/datadog-plugin-aws-sdk/src/base.js +++ b/packages/datadog-plugin-aws-sdk/src/base.js @@ -64,11 +64,17 @@ class BaseAwsSdkPlugin extends ClientPlugin { span.setTag('region', region) }) - this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response }) => { + this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response, cbExists = false }) => { const store = storage.getStore() if (!store) return const { span } = store if (!span) return + // try to extract DSM context from response if no callback exists as extraction normally happens in CB + if (!cbExists && this.serviceIdentifier === 'sqs') { + const params = response.request.params + const operation = response.request.operation + this.responseExtractDSMContext(operation, params, response.data, span) + } this.addResponseTags(span, response) this.finish(span, response, response.error) }) @@ -159,6 +165,7 @@ function normalizeConfig (config, serviceIdentifier) { return Object.assign({}, config, specificConfig, { splitByAwsService: config.splitByAwsService !== false, + batchPropagationEnabled: config.batchPropagationEnabled !== false, hooks }) } diff --git a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js index f33ec7f5dc8..62c9c9b4a6f 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +++ b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js @@ -52,7 +52,7 @@ class Kinesis extends BaseAwsSdkPlugin { // extract DSM context after as we might not have a parent-child but may have a DSM context this.responseExtractDSMContext( - request.operation, response, span || null, streamName + request.operation, request.params, response, span || null, { streamName } ) } }) @@ -100,7 +100,8 @@ class Kinesis extends BaseAwsSdkPlugin { } } - responseExtractDSMContext (operation, response, span, streamName) { + responseExtractDSMContext (operation, params, response, span, kwargs = {}) { + const { streamName } = kwargs if (!this.config.dsmEnabled) return if (operation !== 'getRecords') return if (!response || !response.Records || !response.Records[0]) return @@ -151,7 +152,12 @@ class Kinesis extends BaseAwsSdkPlugin { case 'putRecords': stream = params.StreamArn ? params.StreamArn : (params.StreamName ? params.StreamName : '') for (let i = 0; i < params.Records.length; i++) { - this.injectToMessage(span, params.Records[i], stream, i === 0) + this.injectToMessage( + span, + params.Records[i], + stream, + i === 0 || (this.config.kinesis && this.config.kinesis.batchPropagationEnabled) + ) } } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sns.js b/packages/datadog-plugin-aws-sdk/src/services/sns.js index ee5191ddabc..a88aa8bda46 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sns.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sns.js @@ -59,7 +59,12 @@ class Sns extends BaseAwsSdkPlugin { break case 'publishBatch': for (let i = 0; i < params.PublishBatchRequestEntries.length; i++) { - this.injectToMessage(span, params.PublishBatchRequestEntries[i], params.TopicArn, i === 0) + this.injectToMessage( + span, + params.PublishBatchRequestEntries[i], + params.TopicArn, + i === 0 || (this.config.sns && this.config.sns.batchPropagationEnabled) + ) } break } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sqs.js b/packages/datadog-plugin-aws-sdk/src/services/sqs.js index 769d9fc9ac6..cce27c18719 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sqs.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sqs.js @@ -23,7 +23,7 @@ class Sqs extends BaseAwsSdkPlugin { const plugin = this const contextExtraction = this.responseExtract(request.params, request.operation, response) let span - let parsedMessageAttributes + let parsedMessageAttributes = null if (contextExtraction && contextExtraction.datadogContext) { obj.needsFinish = true const options = { @@ -39,8 +39,9 @@ class Sqs extends BaseAwsSdkPlugin { this.enter(span, store) } // extract DSM context after as we might not have a parent-child but may have a DSM context + this.responseExtractDSMContext( - request.operation, request.params, response, span || null, parsedMessageAttributes || null + request.operation, request.params, response, span || null, { parsedMessageAttributes } ) }) @@ -165,7 +166,8 @@ class Sqs extends BaseAwsSdkPlugin { } } - responseExtractDSMContext (operation, params, response, span, parsedAttributes) { + responseExtractDSMContext (operation, params, response, span, kwargs = {}) { + let { parsedAttributes } = kwargs if (!this.config.dsmEnabled) return if (operation !== 'receiveMessage') return if (!response || !response.Messages || !response.Messages[0]) return @@ -188,7 +190,7 @@ class Sqs extends BaseAwsSdkPlugin { // SQS to SQS } } - if (message.MessageAttributes && message.MessageAttributes._datadog) { + if (!parsedAttributes && message.MessageAttributes && message.MessageAttributes._datadog) { parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog) } } @@ -216,7 +218,23 @@ class Sqs extends BaseAwsSdkPlugin { break case 'sendMessageBatch': for (let i = 0; i < params.Entries.length; i++) { - this.injectToMessage(span, params.Entries[i], params.QueueUrl, i === 0) + this.injectToMessage( + span, + params.Entries[i], + params.QueueUrl, + i === 0 || (this.config.sqs && this.config.sqs.batchPropagationEnabled) + ) + } + break + case 'receiveMessage': + if (!params.MessageAttributeNames) { + params.MessageAttributeNames = ['_datadog'] + } else if ( + !params.MessageAttributeNames.includes('_datadog') && + !params.MessageAttributeNames.includes('.*') && + !params.MessageAttributeNames.includes('All') + ) { + params.MessageAttributeNames.push('_datadog') } break } diff --git a/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js b/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js index 0fdc15ea537..e077c0b64b2 100644 --- a/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/integration-test/client.spec.js @@ -15,7 +15,7 @@ describe('esm', () => { withVersions('aws-sdk', ['aws-sdk'], version => { before(async function () { - this.timeout(20000) + this.timeout(60000) sandbox = await createSandbox([`'aws-sdk@${version}'`], false, [ './packages/datadog-plugin-aws-sdk/test/integration-test/*']) }) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js index 89518c45cdc..cedeb14f000 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js @@ -52,7 +52,7 @@ describe('Kinesis', function () { describe('no configuration', () => { before(() => { - return agent.load('aws-sdk', { kinesis: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load('aws-sdk', { kinesis: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) before(done => { @@ -91,6 +91,24 @@ describe('Kinesis', function () { }) }) + it('injects trace context to each message during Kinesis putRecord and batchPropagationEnabled', done => { + helpers.putTestRecords(kinesis, streamName, (err, data) => { + if (err) return done(err) + + helpers.getTestRecord(kinesis, streamName, data.Records[0], (err, data) => { + if (err) return done(err) + + for (const record in data.Records) { + const recordData = JSON.parse(Buffer.from(data.Records[record].Data).toString()) + expect(recordData).to.have.property('_datadog') + expect(recordData._datadog).to.have.property('x-datadog-trace-id') + } + + done() + }) + }) + }) + it('handles already b64 encoded data', done => { helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer.toString('base64'), (err, data) => { if (err) return done(err) diff --git a/packages/datadog-plugin-aws-sdk/test/sns.spec.js b/packages/datadog-plugin-aws-sdk/test/sns.spec.js index 27f194abc7e..293833a6009 100644 --- a/packages/datadog-plugin-aws-sdk/test/sns.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sns.spec.js @@ -7,7 +7,7 @@ const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') const { rawExpectedSchema } = require('./sns-naming') -describe('Sns', () => { +describe('Sns', function () { setup() withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { @@ -25,7 +25,8 @@ describe('Sns', () => { const snsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sns' : 'aws-sdk' const sqsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sqs' : 'aws-sdk' - const assertPropagation = done => { + let childSpansFound = 0 + const assertPropagation = (done, childSpans = 1) => { agent.use(traces => { const span = traces[0][0] @@ -37,7 +38,10 @@ describe('Sns', () => { expect(parentId).to.not.equal('0') expect(parentId).to.equal(spanId) - }).then(done, done) + childSpansFound += 1 + expect(childSpansFound).to.equal(childSpans) + childSpansFound = 0 + }, { timeoutMs: 10000 }).then(done, done) } function createResources (queueName, topicName, cb) { @@ -85,13 +89,13 @@ describe('Sns', () => { parentId = '0' spanId = '0' - return agent.load('aws-sdk', { sns: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) before(done => { process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') - tracer.use('aws-sdk', { sns: { dsmEnabled: false } }) + tracer.use('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }) createResources('TestQueue', 'TestTopic', done) }) @@ -170,6 +174,34 @@ describe('Sns', () => { }, e => e && done(e)) }) }) + + it('injects trace context to each message SNS publishBatch with batch propagation enabled', done => { + assertPropagation(done, 3) + + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, (err, data) => { + if (err) done(err) + + for (const message in data.Messages) { + const recordData = JSON.parse(data.Messages[message].Body) + expect(recordData.MessageAttributes).to.have.property('_datadog') + + const attributes = JSON.parse(Buffer.from(recordData.MessageAttributes._datadog.Value, 'base64')) + expect(attributes).to.have.property('x-datadog-trace-id') + } + }) + sns.publishBatch({ + TopicArn, + PublishBatchRequestEntries: [ + { Id: '1', Message: 'message 1' }, + { Id: '2', Message: 'message 2' }, + { Id: '3', Message: 'message 3' } + ] + }, e => e && done(e)) + }) + }) } // TODO: Figure out why this fails only in 3.0.0 @@ -261,7 +293,7 @@ describe('Sns', () => { } catch { // pass } - agent.reload('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) + agent.reload('aws-sdk', { sns: { dsmEnabled: true, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) it('injects DSM pathway hash to SNS publish span', done => { diff --git a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js index fee2a918810..9c0c3686f9b 100644 --- a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js @@ -3,6 +3,7 @@ const sinon = require('sinon') const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') +const semver = require('semver') const { rawExpectedSchema } = require('./sqs-naming') const queueName = 'SQS_QUEUE_NAME' @@ -37,8 +38,11 @@ describe('Plugin', () => { before(() => { process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sqs: { batchPropagationEnabled: true } }) - return agent.load('aws-sdk', { sqs: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load( + 'aws-sdk', { sqs: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true } + ) }) before(done => { @@ -147,6 +151,74 @@ describe('Plugin', () => { }) }) + it('should propagate the tracing context from the producer to the consumer in batch operations', (done) => { + let parentId + let traceId + + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource.startsWith('sendMessageBatch')).to.equal(true) + expect(span.meta).to.include({ + queuename: 'SQS_QUEUE_NAME' + }) + + parentId = span.span_id.toString() + traceId = span.trace_id.toString() + }) + + let batchChildSpans = 0 + agent.use(traces => { + const span = traces[0][0] + + expect(parentId).to.be.a('string') + expect(span.parent_id.toString()).to.equal(parentId) + expect(span.trace_id.toString()).to.equal(traceId) + batchChildSpans += 1 + expect(batchChildSpans).to.equal(3) + }, { timeoutMs: 2000 }).then(done, done) + + sqs.sendMessageBatch( + { + Entries: [ + { + Id: '1', + MessageBody: 'test batch propagation 1' + }, + { + Id: '2', + MessageBody: 'test batch propagation 2' + }, + { + Id: '3', + MessageBody: 'test batch propagation 3' + } + ], + QueueUrl + }, (err) => { + if (err) return done(err) + + function receiveMessage () { + sqs.receiveMessage({ + QueueUrl, + MaxNumberOfMessages: 1 + }, (err, data) => { + if (err) return done(err) + + for (const message in data.Messages) { + const recordData = data.Messages[message].MessageAttributes + expect(recordData).to.have.property('_datadog') + const traceContext = JSON.parse(recordData._datadog.StringValue) + expect(traceContext).to.have.property('x-datadog-trace-id') + } + }) + } + receiveMessage() + receiveMessage() + receiveMessage() + }) + }) + it('should run the consumer in the context of its span', (done) => { sqs.sendMessage({ MessageBody: 'test body', @@ -408,6 +480,34 @@ describe('Plugin', () => { }) }) + if (sqsClientName === 'aws-sdk' && semver.intersects(version, '>=2.3')) { + it('Should set pathway hash tag on a span when consuming and promise() was used over a callback', + async () => { + await sqs.sendMessage({ MessageBody: 'test DSM', QueueUrl: QueueUrlDsm }) + await sqs.receiveMessage({ QueueUrl: QueueUrlDsm }).promise() + + let consumeSpanMeta = {} + return new Promise((resolve, reject) => { + agent.use(traces => { + const span = traces[0][0] + + if (span.name === 'aws.request' && span.meta['aws.operation'] === 'receiveMessage') { + consumeSpanMeta = span.meta + } + + try { + expect(consumeSpanMeta).to.include({ + 'pathway.hash': expectedConsumerHash + }) + resolve() + } catch (error) { + reject(error) + } + }) + }) + }) + } + it('Should emit DSM stats to the agent when sending a message', done => { agent.expectPipelineStats(dsmStats => { let statsPointsReceived = 0 diff --git a/packages/datadog-plugin-child_process/src/index.js b/packages/datadog-plugin-child_process/src/index.js index 663b46b1183..7209f44b97a 100644 --- a/packages/datadog-plugin-child_process/src/index.js +++ b/packages/datadog-plugin-child_process/src/index.js @@ -54,7 +54,7 @@ class ChildProcessPlugin extends TracingPlugin { } this.startSpan('command_execution', { - service: this.config.service, + service: this.config.service || this._tracerConfig.service, resource: (shell === true) ? 'sh' : cmdFields[0], type: 'system', meta diff --git a/packages/datadog-plugin-child_process/test/index.spec.js b/packages/datadog-plugin-child_process/test/index.spec.js index 68eefad4e7d..4598457274e 100644 --- a/packages/datadog-plugin-child_process/test/index.spec.js +++ b/packages/datadog-plugin-child_process/test/index.spec.js @@ -34,6 +34,10 @@ describe('Child process plugin', () => { tracerStub = { startSpan: sinon.stub() } + + configStub = { + service: 'test-service' + } }) afterEach(() => { @@ -52,7 +56,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'ls', 'span.kind': undefined, 'span.type': 'system', @@ -74,7 +78,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'sh', 'span.kind': undefined, 'span.type': 'system', @@ -98,7 +102,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'echo', 'span.kind': undefined, 'span.type': 'system', @@ -123,7 +127,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'sh', 'span.kind': undefined, 'span.type': 'system', @@ -149,7 +153,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'ls', 'span.kind': undefined, 'span.type': 'system', @@ -175,7 +179,7 @@ describe('Child process plugin', () => { childOf: undefined, tags: { component: 'subprocess', - 'service.name': undefined, + 'service.name': 'test-service', 'resource.name': 'sh', 'span.kind': undefined, 'span.type': 'system', diff --git a/packages/datadog-plugin-connect/test/index.spec.js b/packages/datadog-plugin-connect/test/index.spec.js index f30b4967d44..62b64bcc8a7 100644 --- a/packages/datadog-plugin-connect/test/index.spec.js +++ b/packages/datadog-plugin-connect/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { AsyncLocalStorage } = require('async_hooks') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') @@ -45,7 +44,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -62,11 +63,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -76,7 +75,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -109,7 +108,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -119,11 +120,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -137,7 +136,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -148,11 +149,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -175,12 +174,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -196,7 +195,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -206,11 +207,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -239,12 +238,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -253,7 +252,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -263,11 +264,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -293,11 +292,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { app.use('/app', (req, res) => res.end()) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -318,10 +319,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -332,7 +331,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -342,17 +343,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -368,7 +367,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -381,13 +382,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -403,7 +402,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -416,13 +417,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -432,7 +431,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -447,13 +448,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,12 +476,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -497,7 +496,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -516,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -551,7 +550,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -561,11 +562,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -577,7 +576,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -587,13 +588,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -604,7 +603,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -614,13 +615,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -632,7 +631,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -649,11 +650,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -668,7 +667,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -687,13 +688,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -703,7 +702,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -718,13 +719,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -750,7 +749,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -762,11 +763,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -791,11 +790,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -810,7 +809,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -825,13 +826,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -841,7 +840,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -856,13 +857,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 5c9f378c802..2e77b59395b 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -195,6 +195,17 @@ class CucumberPlugin extends CiPlugin { this.enter(testSpan, store) }) + this.addSub('ci:cucumber:test:retry', (isFlakyRetry) => { + const store = storage.getStore() + const span = store.span + if (isFlakyRetry) { + span.setTag(TEST_IS_RETRY, 'true') + } + span.setTag(TEST_STATUS, 'fail') + span.finish() + finishAllTraceSpans(span) + }) + this.addSub('ci:cucumber:test-step:start', ({ resource }) => { const store = storage.getStore() const childOf = store ? store.span : store @@ -239,7 +250,15 @@ class CucumberPlugin extends CiPlugin { }) }) - this.addSub('ci:cucumber:test:finish', ({ isStep, status, skipReason, errorMessage, isNew, isEfdRetry }) => { + this.addSub('ci:cucumber:test:finish', ({ + isStep, + status, + skipReason, + errorMessage, + isNew, + isEfdRetry, + isFlakyRetry + }) => { const span = storage.getStore().span const statusTag = isStep ? 'step.status' : TEST_STATUS @@ -260,6 +279,10 @@ class CucumberPlugin extends CiPlugin { span.setTag(ERROR_MESSAGE, errorMessage) } + if (isFlakyRetry > 0) { + span.setTag(TEST_IS_RETRY, 'true') + } + span.finish() if (!isStep) { this.telemetry.ciVisEvent( diff --git a/packages/datadog-plugin-cypress/test/index.spec.js b/packages/datadog-plugin-cypress/test/index.spec.js index 6be933cf88b..67ca6387ac4 100644 --- a/packages/datadog-plugin-cypress/test/index.spec.js +++ b/packages/datadog-plugin-cypress/test/index.spec.js @@ -1,5 +1,4 @@ 'use strict' -const getPort = require('get-port') const { expect } = require('chai') const semver = require('semver') @@ -31,15 +30,16 @@ describe('Plugin', function () { let agentListenPort this.retries(2) withVersions('cypress', 'cypress', (version, moduleName) => { - beforeEach(function () { + beforeEach(() => { + return agent.load() + }) + beforeEach(function (done) { this.timeout(10000) - return agent.load().then(() => { - agentListenPort = agent.server.address().port - cypressExecutable = require(`../../../versions/cypress@${version}`).get() - return getPort().then(port => { - appPort = port - appServer.listen(appPort) - }) + agentListenPort = agent.server.address().port + cypressExecutable = require(`../../../versions/cypress@${version}`).get() + appServer.listen(0, () => { + appPort = appServer.address().port + done() }) }) afterEach(() => agent.close({ ritmReset: false })) diff --git a/packages/datadog-plugin-dns/test/index.spec.js b/packages/datadog-plugin-dns/test/index.spec.js index 98edb26ad1c..1457bb869d8 100644 --- a/packages/datadog-plugin-dns/test/index.spec.js +++ b/packages/datadog-plugin-dns/test/index.spec.js @@ -142,19 +142,19 @@ describe('Plugin', () => { expect(traces[0][0]).to.deep.include({ name: 'dns.resolve', service: 'test', - resource: 'ANY lvh.me' + resource: 'ANY localhost' }) expect(traces[0][0].meta).to.deep.include({ component: 'dns', 'span.kind': 'client', - 'dns.hostname': 'lvh.me', + 'dns.hostname': 'localhost', 'dns.rrtype': 'ANY' }) }) .then(done) .catch(done) - dns.resolveAny('lvh.me', err => err && done(err)) + dns.resolveAny('localhost', () => done()) }) it('should instrument reverse', done => { diff --git a/packages/datadog-plugin-express/test/index.spec.js b/packages/datadog-plugin-express/test/index.spec.js index 4dbc1776a61..55a608f4adf 100644 --- a/packages/datadog-plugin-express/test/index.spec.js +++ b/packages/datadog-plugin-express/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') const plugin = require('../src') @@ -45,7 +44,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const timer = setTimeout(done, 100) agent.use(() => { @@ -53,11 +53,9 @@ describe('Plugin', () => { done(new Error('Agent received an unexpected trace.')) }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -70,13 +68,13 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) }) @@ -101,7 +99,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -114,15 +114,14 @@ describe('Plugin', () => { expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) expect(spans[0].meta).to.have.property('http.method', 'GET') expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('http.route', '/user') }) .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -136,7 +135,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -153,11 +154,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -173,7 +172,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -190,11 +191,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -209,7 +208,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -245,11 +246,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -271,7 +270,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -286,11 +287,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -316,7 +315,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -330,11 +331,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -348,7 +347,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -358,11 +359,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -386,11 +387,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -410,7 +409,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -420,11 +421,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -438,7 +437,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -448,11 +449,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -467,7 +466,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -477,11 +478,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -495,7 +494,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -507,11 +508,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -534,12 +533,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -563,7 +562,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -573,11 +574,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -595,7 +594,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -605,11 +606,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -624,7 +623,9 @@ describe('Plugin', () => { res.status(200).send(error.message) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -634,11 +635,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -659,7 +658,9 @@ describe('Plugin', () => { app.use('/v1', routerA) app.use('/v1', routerB) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -669,11 +670,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -689,7 +688,9 @@ describe('Plugin', () => { res.status(200).send(req.body) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -699,11 +700,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -725,7 +724,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -735,11 +736,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -761,7 +760,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -771,11 +772,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -799,7 +798,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -809,11 +810,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -830,7 +829,9 @@ describe('Plugin', () => { app.use('/v1', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -840,11 +841,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -873,12 +872,12 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -887,7 +886,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.status(200).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -897,11 +898,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -927,11 +926,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -957,11 +956,11 @@ describe('Plugin', () => { } ) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -979,7 +978,9 @@ describe('Plugin', () => { app.use('/app', router) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -989,10 +990,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -1003,7 +1002,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1013,17 +1014,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -1038,7 +1037,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1050,13 +1051,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1072,7 +1071,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1084,13 +1085,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1100,7 +1099,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1115,13 +1116,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1133,7 +1132,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1152,13 +1153,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1170,7 +1169,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1189,13 +1190,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1206,7 +1205,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1217,11 +1218,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1260,12 +1259,46 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) + }) + }) + + it('should handle 404 errors', done => { + const app = express() + + app.use((req, res, next) => { + next() + }) + + app.get('/does-exist', (req, res) => { + res.status(200).send('hi') + }) + + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent.use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('error', 0) + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '404') + expect(spans[0].meta).to.have.property('component', 'express') + expect(spans[0].meta).to.not.have.property('http.route') + + done() }) + + axios + .get(`http://localhost:${port}/does-not-exist`, { + validateStatus: status => status === 404 + }) + .catch(done) }) }) @@ -1285,7 +1318,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1302,10 +1337,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) @@ -1320,7 +1353,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1332,10 +1367,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) }) @@ -1366,7 +1399,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1376,11 +1411,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1391,7 +1424,9 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1401,13 +1436,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1418,7 +1451,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1428,13 +1463,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -1445,7 +1478,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const spy = sinon.spy() agent @@ -1461,11 +1495,9 @@ describe('Plugin', () => { } }, 100) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/health`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/health`) + .catch(done) }) }) }) @@ -1505,11 +1537,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1524,7 +1556,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1535,10 +1569,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1553,7 +1585,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1565,13 +1599,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1587,7 +1619,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1600,13 +1634,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1618,7 +1650,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1632,13 +1666,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1648,7 +1680,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1663,13 +1697,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-express/test/leak.js b/packages/datadog-plugin-express/test/leak.js index 3c8b97b4d1c..9d13fc8f978 100644 --- a/packages/datadog-plugin-express/test/leak.js +++ b/packages/datadog-plugin-express/test/leak.js @@ -7,23 +7,22 @@ require('../../dd-trace') const test = require('tape') const express = require('../../../versions/express').get() const axios = require('axios') -const getPort = require('get-port') const profile = require('../../dd-trace/test/profile') test('express plugin should not leak', t => { - getPort().then(port => { - const app = express() + const app = express() - app.use((req, res) => { - res.status(200).send() - }) + app.use((req, res) => { + res.status(200).send() + }) + + const listener = app.listen(0, '127.0.0.1', () => { + const port = listener.address().port - const listener = app.listen(port, '127.0.0.1', () => { - profile(t, operation).then(() => listener.close()) + profile(t, operation).then(() => listener.close()) - function operation (done) { - axios.get(`http://localhost:${port}`).then(done) - } - }) + function operation (done) { + axios.get(`http://localhost:${port}`).then(done) + } }) }) diff --git a/packages/datadog-plugin-fastify/test/index.spec.js b/packages/datadog-plugin-fastify/test/index.spec.js index 920807fbd29..33b1430f98c 100644 --- a/packages/datadog-plugin-fastify/test/index.spec.js +++ b/packages/datadog-plugin-fastify/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') @@ -48,7 +47,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -66,11 +67,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -83,7 +82,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -101,11 +102,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -117,7 +116,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -135,11 +136,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) } @@ -155,12 +154,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -172,12 +171,12 @@ describe('Plugin', () => { app.get('/user', (request, reply) => reply.send()) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -187,12 +186,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -211,12 +210,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -248,12 +247,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -264,7 +263,9 @@ describe('Plugin', () => { reply.send(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -279,11 +280,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -295,11 +294,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, async () => { - await axios.get(`http://localhost:${port}/user`) - done() - }) + app.listen({ host, port: 0 }, async () => { + const port = app.server.address().port + + await axios.get(`http://localhost:${port}/user`) + done() }) }) @@ -324,11 +323,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -343,7 +342,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -359,11 +360,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -389,11 +390,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -407,7 +406,9 @@ describe('Plugin', () => { return Promise.reject(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -422,11 +423,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -442,7 +441,9 @@ describe('Plugin', () => { throw (error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -458,11 +459,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -476,7 +475,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -492,11 +493,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -515,7 +514,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -531,11 +532,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } diff --git a/packages/datadog-plugin-fetch/test/index.spec.js b/packages/datadog-plugin-fetch/test/index.spec.js index f7bd7b85889..1d322de04a4 100644 --- a/packages/datadog-plugin-fetch/test/index.spec.js +++ b/packages/datadog-plugin-fetch/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const tags = require('../../../ext/tags') const { expect } = require('chai') @@ -21,9 +20,9 @@ describe('Plugin', () => { let appListener describe('fetch', () => { - function server (app, port, listener) { + function server (app, listener) { const server = require('http').createServer(app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -54,10 +53,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/user`) }) }, rawExpectedSchema.client @@ -68,7 +65,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -84,9 +81,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -95,7 +90,7 @@ describe('Plugin', () => { app.post('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -111,9 +106,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) - }) + fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) }) }) @@ -122,7 +115,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -138,9 +131,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - }) + fetch(new globalThis.Request(`http://localhost:${port}/user`)) }) }) @@ -149,15 +140,13 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - .then(res => { - expect(res).to.have.property('status', 200) - done() - }) - .catch(done) - }) + appListener = server(app, port => { + fetch(new globalThis.Request(`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) }) }) @@ -168,7 +157,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -177,9 +166,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -193,7 +180,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -201,9 +188,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -218,7 +203,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -226,9 +211,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) - }) + fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) }) }) @@ -248,13 +231,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + Authorization: 'AWS4-HMAC-SHA256 ...' + } }) }) }) @@ -275,13 +256,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + Authorization: ['AWS4-HMAC-SHA256 ...'] + } }) }) }) @@ -302,13 +281,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - 'X-Amz-Signature': 'abc123' - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + 'X-Amz-Signature': 'abc123' + } }) }) }) @@ -329,30 +306,26 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'fetch') - }) - .then(done) - .catch(done) + let error - fetch(`http://localhost:${port}/user`).catch(err => { - error = err + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'fetch') }) + .then(done) + .catch(done) + + fetch('http://localhost:7357/user').catch(err => { + error = err }) }) @@ -363,7 +336,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -371,9 +344,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -384,7 +355,7 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -392,9 +363,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -403,7 +372,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -412,15 +381,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -431,7 +398,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -439,15 +406,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -458,7 +423,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -467,15 +432,13 @@ describe('Plugin', () => { clearTimeout(timer) }) - appListener = server(app, port, () => { - const store = storage.getStore() + const store = storage.getStore() - storage.enterWith({ noop: true }) + storage.enterWith({ noop: true }) - fetch(`http://localhost:${port}/user`).catch(() => {}) + fetch(`http://localhost:${port}/user`).catch(() => {}) - storage.enterWith(store) - }) + storage.enterWith(store) }) }) }) @@ -502,7 +465,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -510,9 +473,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -539,7 +500,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -547,9 +508,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -576,7 +535,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -584,9 +543,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -614,7 +571,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -625,13 +582,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`, { - headers: { - 'x-baz': 'qux' - } - }).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) }) }) }) @@ -662,7 +617,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('foo', '/foo') @@ -670,9 +625,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -708,10 +661,8 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) @@ -738,7 +689,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -748,9 +699,7 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) diff --git a/packages/datadog-plugin-graphql/test/index.spec.js b/packages/datadog-plugin-graphql/test/index.spec.js index 7d7aae7fb71..aa8c754f28a 100644 --- a/packages/datadog-plugin-graphql/test/index.spec.js +++ b/packages/datadog-plugin-graphql/test/index.spec.js @@ -7,7 +7,6 @@ const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/c const { expectedSchema, rawExpectedSchema } = require('./naming') const axios = require('axios') const http = require('http') -const getPort = require('get-port') const dc = require('dc-polyfill') const plugin = require('../src') @@ -231,14 +230,16 @@ describe('Plugin', () => { const yoga = graphqlYoga.createYoga({ schema }) server = http.createServer(yoga) - - getPort().then(newPort => { - port = newPort - server.listen(port) - }) }) }) + before(done => { + server.listen(0, () => { + port = server.address().port + done() + }) + }) + after(() => { server.close() return agent.close({ ritmReset: false }) diff --git a/packages/datadog-plugin-hapi/test/index.spec.js b/packages/datadog-plugin-hapi/test/index.spec.js index d3bb3a5d68a..2e67022f494 100644 --- a/packages/datadog-plugin-hapi/test/index.spec.js +++ b/packages/datadog-plugin-hapi/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') @@ -47,15 +46,13 @@ describe('Plugin', () => { if (semver.intersects(version, '>=17')) { beforeEach(() => { - return getPort() - .then(_port => { - port = _port - server = Hapi.server({ - address: 'localhost', - port - }) - return server.start() - }) + server = Hapi.server({ + address: 'localhost', + port: 0 + }) + return server.start().then(() => { + port = server.listener.address().port + }) }) afterEach(() => { @@ -63,19 +60,19 @@ describe('Plugin', () => { }) } else { beforeEach(done => { - getPort() - .then(_port => { - port = _port - - if (Hapi.Server.prototype.connection) { - server = new Hapi.Server() - server.connection({ address: 'localhost', port }) - } else { - server = new Hapi.Server('localhost', port) - } + if (Hapi.Server.prototype.connection) { + server = new Hapi.Server() + server.connection({ address: 'localhost', port }) + } else { + server = new Hapi.Server('localhost', port) + } - server.start(done) - }) + server.start(err => { + if (!err) { + port = server.listener.address().port + } + done(err) + }) }) afterEach(done => { diff --git a/packages/datadog-plugin-http/test/client.spec.js b/packages/datadog-plugin-http/test/client.spec.js index 22406986b15..42f4c8436f8 100644 --- a/packages/datadog-plugin-http/test/client.spec.js +++ b/packages/datadog-plugin-http/test/client.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const fs = require('fs') const path = require('path') @@ -28,7 +27,7 @@ describe('Plugin', () => { ['http', 'https', 'node:http', 'node:https'].forEach(pluginToBeLoaded => { const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded describe(pluginToBeLoaded, () => { - function server (app, port, listener) { + function server (app, listener) { let server if (pluginToBeLoaded === 'https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' @@ -41,7 +40,9 @@ describe('Plugin', () => { } else { server = require('node:http').createServer(app) } - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => { + listener(server.address().port) + }) return server } @@ -71,13 +72,12 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - req.end() + appListener = server(app, () => { + const port = appListener.address().port + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + req.end() }) } @@ -99,7 +99,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -115,13 +116,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -132,22 +131,20 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0]).to.not.be.undefined - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.get(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0]).to.not.be.undefined + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.get(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -156,7 +153,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -172,27 +170,25 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - appListener.on('connect', (req, clientSocket, head) => { - clientSocket.write('HTTP/1.1 200 Connection Established\r\n' + + appListener.on('connect', (req, clientSocket, head) => { + clientSocket.write('HTTP/1.1 200 Connection Established\r\n' + 'Proxy-agent: Node.js-Proxy\r\n' + '\r\n') - clientSocket.end() - appListener.close() - }) - - const req = http.request({ - protocol: `${protocol}:`, - port, - method: 'CONNECT', - hostname: 'localhost', - path: '/user' - }) + clientSocket.end() + appListener.close() + }) - req.on('connect', (res, socket) => socket.end()) - req.on('error', () => {}) - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port, + method: 'CONNECT', + hostname: 'localhost', + path: '/user' }) + + req.on('connect', (res, socket) => socket.end()) + req.on('error', () => {}) + req.end() }) }) @@ -201,7 +197,8 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -216,30 +213,28 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - appListener.on('upgrade', (req, socket, head) => { - socket.write('HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + + appListener.on('upgrade', (req, socket, head) => { + socket.write('HTTP/1.1 101 Web Socket Protocol Handshake\r\n' + 'Upgrade: WebSocket\r\n' + 'Connection: Upgrade\r\n' + '\r\n') - socket.pipe(socket) - }) - - const req = http.request({ - protocol: `${protocol}:`, - port, - hostname: 'localhost', - path: '/user', - headers: { - Connection: 'Upgrade', - Upgrade: 'websocket' - } - }) + socket.pipe(socket) + }) - req.on('upgrade', (res, socket) => socket.end()) - req.on('error', () => {}) - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port, + hostname: 'localhost', + path: '/user', + headers: { + Connection: 'Upgrade', + Upgrade: 'websocket' + } }) + + req.on('upgrade', (res, socket) => socket.end()) + req.on('error', () => {}) + req.end() }) }) @@ -249,7 +244,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -263,12 +258,9 @@ describe('Plugin', () => { port, path: '/user' } + const req = http.request(uri) - appListener = server(app, port, () => { - const req = http.request(uri) - - req.end() - }) + req.end() }) }) @@ -279,7 +271,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -288,13 +280,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user?foo=bar`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user?foo=bar`, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -307,7 +297,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -316,11 +306,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/another-path`, { path: '/user' }) + const req = http.request(`${protocol}://localhost:${port}/another-path`, { path: '/user' }) - req.end() - }) + req.end() }) }) } @@ -332,7 +320,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -347,31 +335,28 @@ describe('Plugin', () => { port, pathname: '/another-path' } + const req = http.request(uri, { path: '/user' }) - appListener = server(app, port, () => { - const req = http.request(uri, { path: '/user' }) - - req.end() - }) + req.end() }) }) - it('should support configuration as an WHATWG URL object', async () => { + it('should support configuration as an WHATWG URL object', done => { const app = express() - const port = await getPort() - const url = new URL(`${protocol}://localhost:${port}/user`) - app.get('/user', (req, res) => res.status(200).send()) + appListener = server(app, port => { + const url = new URL(`${protocol}://localhost:${port}/user`) + + app.get('/user', (req, res) => res.status(200).send()) + + agent.use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) + }).then(done, done) - appListener = server(app, port, () => { const req = http.request(url) req.end() }) - - await agent.use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) - }) }) it('should use the correct defaults when not specified', done => { @@ -381,7 +366,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/`) @@ -389,14 +374,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request({ - protocol: `${protocol}:`, - port - }) - - req.end() + const req = http.request({ + protocol: `${protocol}:`, + port }) + + req.end() }) }) @@ -407,19 +390,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.not.be.undefined - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.not.be.undefined + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`) - req.end() - }) + req.end() }) }) @@ -430,16 +411,14 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - setTimeout(() => { - res.on('data', () => done()) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + setTimeout(() => { + res.on('data', () => done()) }) - - req.end() }) + + req.end() }) }) @@ -453,19 +432,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`) - req.end() - }) + req.end() }) }) @@ -485,17 +462,15 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + headers: { + Authorization: 'AWS4-HMAC-SHA256 ...' + } }) + + req.end() }) }) @@ -515,17 +490,15 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + headers: { + Authorization: ['AWS4-HMAC-SHA256 ...'] + } }) + + req.end() }) }) @@ -545,17 +518,15 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + headers: { + 'X-Amz-Signature': 'abc123' + } }) + + req.end() }) }) @@ -575,15 +546,13 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - path: '/?X-Amz-Signature=abc123' - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + path: '/?X-Amz-Signature=abc123' }) + + req.end() }) }) @@ -593,15 +562,14 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - expect(tracer.scope().active()).to.be.null - done() - }) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + expect(tracer.scope().active()).to.be.null + done() }) + + req.end() }) }) @@ -612,20 +580,18 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - const span = tracer.scope().active() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + const span = tracer.scope().active() - res.on('data', () => {}) - res.on('end', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) + res.on('data', () => {}) + res.on('end', () => { + expect(tracer.scope().active()).to.equal(span) + done() }) - - req.end() }) + + req.end() }) }) @@ -636,42 +602,38 @@ describe('Plugin', () => { res.status(200).send('OK') }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, () => {}) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, () => {}) - req.on('response', () => { - expect(tracer.scope().active()).to.not.be.null - done() - }) - - req.end() + req.on('response', () => { + expect(tracer.scope().active()).to.not.be.null + done() }) + + req.end() }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) - - const req = http.request(`${protocol}://localhost:${port}/user`) - - req.on('error', err => { - error = err + let error + + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'http') }) + .then(done) + .catch(done) - req.end() + const req = http.request(`${protocol}://localhost:7357/user`) + + req.on('error', err => { + error = err }) + + req.end() }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -681,21 +643,19 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) @@ -706,21 +666,19 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) @@ -729,32 +687,30 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) + let error - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) - req.on('error', err => { - error = err - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.destroy() + req.on('error', err => { + error = err }) + + req.destroy() }) }) @@ -763,24 +719,22 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('abort', () => {}) + req.on('abort', () => {}) - req.abort() - }) + req.abort() }) }) @@ -789,25 +743,23 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - expect(traces[0][0].meta).to.not.have.property('http.status_code') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + expect(traces[0][0].meta).to.not.have.property('http.status_code') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.setTimeout(1) - req.end() - }) + req.setTimeout(1) + req.end() }) }) @@ -822,23 +774,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 0) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 0) + }) + .then(done) + .catch(done) - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.end() - }) + req.end() }) }).timeout(10000) @@ -852,27 +802,25 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - const options = { - agent: new http.Agent({ keepAlive: true, timeout: 5000 }) // custom agent with same default timeout - } + const options = { + agent: new http.Agent({ keepAlive: true, timeout: 5000 }) // custom agent with same default timeout + } - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, options, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, options, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) + req.on('error', () => {}) - req.end() - }) + req.end() }) }).timeout(10000) @@ -886,24 +834,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - appListener = server(app, port, async () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) + }) - req.on('error', () => {}) - req.setTimeout(5000) // match default timeout + req.on('error', () => {}) + req.setTimeout(5000) // match default timeout - req.end() - }) + req.end() }) }).timeout(10000) } @@ -924,33 +870,31 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const spans = traces[0] - expect(spans.length).to.equal(3) + agent + .use(traces => { + const spans = traces[0] + expect(spans.length).to.equal(3) + }) + .then(done) + .catch(done) + + appListener = server(app, port => { + // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts + // would be siblings and our test would only capture 1 as a false positive. + const span = tracer.startSpan('http-test') + tracer.scope().activate(span, () => { + // Test `http(s).request + const req = http.request(`${protocol}://localhost:${port}/user?test=request`, res => { + res.on('data', () => {}) }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts - // would be siblings and our test would only capture 1 as a false positive. - const span = tracer.startSpan('http-test') - tracer.scope().activate(span, () => { - // Test `http(s).request - const req = http.request(`${protocol}://localhost:${port}/user?test=request`, res => { - res.on('data', () => {}) - }) - req.end() - - // Test `http(s).get` - http.get(`${protocol}://localhost:${port}/user?test=get`, res => { - res.on('data', () => {}) - }) + req.end() - span.finish() + // Test `http(s).get` + http.get(`${protocol}://localhost:${port}/user?test=get`, res => { + res.on('data', () => {}) }) + + span.finish() }) }) }) @@ -962,19 +906,17 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('service', SERVICE_NAME) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', SERVICE_NAME) + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/abort`) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/abort`) - req.abort() - }) + req.abort() }) }) @@ -985,7 +927,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -994,17 +936,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request({ - hostname: 'localhost', - port, - path: '/user?foo=bar' - }, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request({ + hostname: 'localhost', + port, + path: '/user?foo=bar' + }, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -1015,7 +955,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][1]).to.have.property('error', 0) @@ -1025,17 +965,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - tracer.trace('test.request', (span, finish) => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - res.on('end', () => { - setTimeout(finish) - }) + tracer.trace('test.request', (span, finish) => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) + res.on('end', () => { + setTimeout(finish) }) - - req.end() }) + + req.end() }) }) }) @@ -1048,26 +986,24 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - const timer = setTimeout(done, 100) + const timer = setTimeout(done, 100) - agent - .use(() => { - done(new Error('Noop request was traced.')) - clearTimeout(timer) - }) + agent + .use(() => { + done(new Error('Noop request was traced.')) + clearTimeout(timer) + }) - appListener = server(app, port, () => { - const store = storage.getStore() + appListener = server(app, port => { + const store = storage.getStore() - storage.enterWith({ noop: true }) - const req = http.request(tracer._tracer._url.href) + storage.enterWith({ noop: true }) + const req = http.request(tracer._tracer._url.href) - req.on('error', () => {}) - req.end() + req.on('error', () => {}) + req.end() - storage.enterWith(store) - }) + storage.enterWith(store) }) }) } @@ -1096,23 +1032,21 @@ describe('Plugin', () => { res.end() } - getPort().then(port => { - appListener = server(app, port, () => { - ch.subscribe(sub) + appListener = server(app, port => { + ch.subscribe(sub) - tracer.use('http', false) + tracer.use('http', false) - const req = http.request(`${protocol}://localhost:${port}`, res => { - res.on('error', done) - res.on('data', () => {}) - res.on('end', () => done()) - }) - req.on('error', done) + const req = http.request(`${protocol}://localhost:${port}`, res => { + res.on('error', done) + res.on('data', () => {}) + res.on('end', () => done()) + }) + req.on('error', done) - tracer.use('http', true) + tracer.use('http', true) - req.end() - }) + req.end() }) }) }) @@ -1142,21 +1076,19 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('service', 'custom') - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('service', 'custom') + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) }) @@ -1192,17 +1124,15 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + headers: { + Authorization: 'AWS4-HMAC-SHA256 ...' + } }) + + req.end() }) }) }) @@ -1232,21 +1162,19 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('error', 1) - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('error', 1) + }) + .then(done) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.end() }) }) }) @@ -1276,14 +1204,12 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { serverPort = port - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + req.end() }) }, { @@ -1305,7 +1231,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -1313,13 +1239,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) - - req.end() + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) }) @@ -1351,7 +1275,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -1364,15 +1288,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const url = `${protocol}://localhost:${port}/user` - const headers = { 'x-baz': 'baz' } - const req = http.request(url, { headers }, res => { - res.on('data', () => {}) - }) - - req.end() + const url = `${protocol}://localhost:${port}/user` + const headers = { 'x-baz': 'baz' } + const req = http.request(url, { headers }, res => { + res.on('data', () => {}) }) + + req.end() }) }) @@ -1383,24 +1305,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const meta = traces[0][0].meta - - expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const meta = traces[0][0].meta - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar') + }) + .then(done) + .catch(done) - req.setHeader('x-foo', 'bar') - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.setHeader('x-foo', 'bar') + req.end() }) }) @@ -1411,24 +1331,22 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - const meta = traces[0][0].meta + agent + .use(traces => { + const meta = traces[0][0].meta - expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar1,bar2') - }) - .then(done) - .catch(done) - - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => { }) - }) + expect(meta).to.have.property(`${HTTP_REQUEST_HEADERS}.x-foo`, 'bar1,bar2') + }) + .then(done) + .catch(done) - req.setHeader('x-foo', ['bar1', 'bar2']) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => { }) }) + + req.setHeader('x-foo', ['bar1', 'bar2']) + req.end() }) }) }) @@ -1462,23 +1380,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('resource', 'GET /user') - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0]).to.have.property('resource', 'GET /user') + }) + .then(done) + .catch(done) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) + }) - req._route = '/user' + req._route = '/user' - req.end() - }) + req.end() }) }) }) @@ -1517,15 +1433,13 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - const req = http.request({ - port, - path: '/users' - }) - - req.end() + appListener = server(app, port => { + const req = http.request({ + port, + path: '/users' }) + + req.end() }) }) }) @@ -1555,23 +1469,21 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - const timer = setTimeout(done, 100) - - agent - .use(() => { - clearTimeout(timer) - done(new Error('Blocklisted requests should not be recorded.')) - }) - .catch(done) + const timer = setTimeout(done, 100) - appListener = server(app, port, () => { - const req = http.request(`${protocol}://localhost:${port}/user`, res => { - res.on('data', () => {}) - }) + agent + .use(() => { + clearTimeout(timer) + done(new Error('Blocklisted requests should not be recorded.')) + }) + .catch(done) - req.end() + appListener = server(app, port => { + const req = http.request(`${protocol}://localhost:${port}/user`, res => { + res.on('data', () => {}) }) + + req.end() }) }) }) diff --git a/packages/datadog-plugin-http/test/server.spec.js b/packages/datadog-plugin-http/test/server.spec.js index ee38d21789e..f3b5f3964ef 100644 --- a/packages/datadog-plugin-http/test/server.spec.js +++ b/packages/datadog-plugin-http/test/server.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const axios = require('axios') const { incomingHttpRequestStart } = require('../../dd-trace/src/appsec/channels') @@ -25,12 +24,6 @@ describe('Plugin', () => { } }) - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - afterEach(() => { appListener && appListener.close() app = null @@ -58,7 +51,10 @@ describe('Plugin', () => { beforeEach(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) it('should send traces to agent', (done) => { diff --git a/packages/datadog-plugin-http2/test/client.spec.js b/packages/datadog-plugin-http2/test/client.spec.js index 970569c12a5..f8d44f3ac0b 100644 --- a/packages/datadog-plugin-http2/test/client.spec.js +++ b/packages/datadog-plugin-http2/test/client.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const fs = require('fs') const path = require('path') @@ -24,7 +23,7 @@ describe('Plugin', () => { const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded const loadPlugin = pluginToBeLoaded.includes('node:') ? 'node:http2' : 'http2' describe(`http2/client, protocol ${pluginToBeLoaded}`, () => { - function server (app, port, listener) { + function server (app, listener) { let server if (pluginToBeLoaded === 'https' || pluginToBeLoaded === 'node:https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' @@ -33,7 +32,7 @@ describe('Plugin', () => { server = require(loadPlugin).createServer() } server.on('stream', app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -58,23 +57,22 @@ describe('Plugin', () => { }) const spanProducerFn = (done) => { - getPort().then(port => { - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const app = (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end() + } - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - req.end() - }) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) } @@ -99,7 +97,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -116,16 +114,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -137,7 +133,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('span.kind', 'client') @@ -146,16 +142,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({}) - .on('error', done) + const req = client.request({}) + .on('error', done) - req.end() - }) + req.end() }) }) @@ -167,7 +161,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -181,16 +175,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -202,7 +194,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -210,16 +202,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user?foo=bar' }) - req.on('error', done) + const req = client.request({ ':path': '/user?foo=bar' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -232,7 +222,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -252,21 +242,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(incorrectConfig, correctConfig) - } else { - client = http2.connect(correctConfig, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(incorrectConfig, correctConfig) + } else { + client = http2.connect(correctConfig, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -279,7 +267,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -299,21 +287,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) - } else { - client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) + } else { + client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -325,7 +311,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/`) @@ -338,16 +324,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({}) - req.on('error', done) + const req = client.request({}) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -362,7 +346,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -370,16 +354,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({}) - req.on('error', done) + const req = client.request({}) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -400,20 +382,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + Authorization: 'AWS4-HMAC-SHA256 ...' + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -434,20 +414,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + Authorization: ['AWS4-HMAC-SHA256 ...'] + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -468,20 +446,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - 'X-Amz-Signature': 'abc123' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + 'X-Amz-Signature': 'abc123' + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -502,17 +478,15 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/?X-Amz-Signature=abc123' }) - req.on('error', done) + const req = client.request({ ':path': '/?X-Amz-Signature=abc123' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -524,53 +498,49 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const span = {} + const span = {} - tracer.scope().activate(span, () => { - const req = client.request({ ':path': '/user' }) - req.on('response', (headers, flags) => { - expect(tracer.scope().active()).to.equal(span) - done() - }) + tracer.scope().activate(span, () => { + const req = client.request({ ':path': '/user' }) + req.on('response', (headers, flags) => { + expect(tracer.scope().active()).to.equal(span) + done() + }) - req.on('error', done) + req.on('error', done) - req.end() - }) + req.end() }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error + let error - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'http2') - expect(traces[0][0].metrics).to.have.property('network.destination.port', port) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'http2') + expect(traces[0][0].metrics).to.have.property('network.destination.port', 7357) + }) + .then(done) + .catch(done) - const client = http2.connect(`${protocol}://localhost:${port}`) - // eslint-disable-next-line n/handle-callback-err - .on('error', (err) => {}) + const client = http2.connect(`${protocol}://localhost:7357`) + // eslint-disable-next-line n/handle-callback-err + .on('error', (err) => {}) - const req = client.request({ ':path': '/user' }) - .on('error', (err) => { error = err }) + const req = client.request({ ':path': '/user' }) + .on('error', (err) => { error = err }) - req.end() - }) + req.end() }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -581,7 +551,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -589,16 +559,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -610,7 +578,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -618,16 +586,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -640,7 +606,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const spans = traces[0] @@ -649,24 +615,22 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts - // would be siblings and our test would only capture 1 as a false positive. - const span = tracer.startSpan('http-test') - tracer.scope().activate(span, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts + // would be siblings and our test would only capture 1 as a false positive. + const span = tracer.startSpan('http-test') + tracer.scope().activate(span, () => { + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/test-1' }) - .on('error', done) - .end() + client.request({ ':path': '/test-1' }) + .on('error', done) + .end() - client.request({ ':path': '/user?test=2' }) - .on('error', done) - .end() + client.request({ ':path': '/user?test=2' }) + .on('error', done) + .end() - span.finish() - }) + span.finish() }) }) }) @@ -697,7 +661,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -705,16 +669,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -744,7 +706,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -752,16 +714,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -786,24 +746,23 @@ describe('Plugin', () => { withNamingSchema( (done) => { - getPort().then(port => { - serverPort = port - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) - - req.end() + const app = (stream, headers) => { + stream.respond({ + ':status': 200 }) + stream.end() + } + appListener = server(app, port => { + serverPort = port + + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) + + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) }, { @@ -826,7 +785,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -834,14 +793,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -872,7 +829,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -883,14 +840,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -920,7 +875,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -930,14 +885,12 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) diff --git a/packages/datadog-plugin-http2/test/server.spec.js b/packages/datadog-plugin-http2/test/server.spec.js index c412c9ccbea..d86817b2860 100644 --- a/packages/datadog-plugin-http2/test/server.spec.js +++ b/packages/datadog-plugin-http2/test/server.spec.js @@ -1,7 +1,6 @@ 'use strict' const { EventEmitter } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { rawExpectedSchema } = require('./naming') @@ -63,12 +62,6 @@ describe('Plugin', () => { } }) - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - afterEach(() => { appListener && appListener.close() app = null @@ -96,7 +89,10 @@ describe('Plugin', () => { beforeEach(done => { const server = http2.createServer(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) it('should send traces to agent', (done) => { diff --git a/packages/datadog-plugin-koa/test/index.spec.js b/packages/datadog-plugin-koa/test/index.spec.js index f37db4acf32..2a123f18f3c 100644 --- a/packages/datadog-plugin-koa/test/index.spec.js +++ b/packages/datadog-plugin-koa/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') @@ -16,14 +15,9 @@ describe('Plugin', () => { describe('koa', () => { withVersions('koa', 'koa', version => { - let port - beforeEach(() => { tracer = require('../../dd-trace') Koa = require(`../../../versions/koa@${version}`).get() - return getPort().then(newPort => { - port = newPort - }) }) afterEach(done => { @@ -41,29 +35,31 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'handle') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'handle') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -78,29 +74,31 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'converted') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'converted') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -123,7 +121,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -151,11 +151,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -206,12 +206,12 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -232,17 +232,19 @@ describe('Plugin', () => { app .use(koaRouter.get('/user/:id', getUser)) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -269,21 +271,23 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) + agent + .use(traces => { + const spans = sort(traces[0]) + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[2]).to.have.property('resource', 'handle') - }) - .then(done) - .catch(done) + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + + expect(spans[2]).to.have.property('resource', 'handle') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -303,16 +307,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -332,16 +338,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -360,16 +368,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -390,16 +400,18 @@ describe('Plugin', () => { app.use(router1.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /public/plop') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /public/plop') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/public/plop`) .catch(done) @@ -422,18 +434,20 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/discussions/456/posts/789`) .catch(done) @@ -457,18 +471,20 @@ describe('Plugin', () => { app.use(first.routes()) app.use(second.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /first/child') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/first/child`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /first/child') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/first/child`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/first/child`) .catch(done) @@ -492,17 +508,19 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/posts/456`) .catch(done) @@ -523,17 +541,19 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -556,26 +576,28 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) - expect(spans[1].meta).to.include({ - [ERROR_TYPE]: error.name, - component: 'koa' + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) + + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + expect(spans[1].meta).to.include({ + [ERROR_TYPE]: error.name, + component: 'koa' + }) + expect(spans[1].error).to.equal(1) }) - expect(spans[1].error).to.equal(1) - }) - .then(done) - .catch(done) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) @@ -609,7 +631,9 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + ws = new WebSocket(`ws://localhost:${port}/message`) ws.on('error', done) ws.on('open', () => { @@ -638,26 +662,28 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -672,26 +698,28 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -714,7 +742,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -742,11 +772,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -770,11 +800,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -801,19 +831,21 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) - expect(spans[0].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) + expect(spans[0].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) diff --git a/packages/datadog-plugin-microgateway-core/test/index.spec.js b/packages/datadog-plugin-microgateway-core/test/index.spec.js index c6beea05ec6..1b76c947122 100644 --- a/packages/datadog-plugin-microgateway-core/test/index.spec.js +++ b/packages/datadog-plugin-microgateway-core/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const os = require('os') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -22,7 +21,11 @@ describe('Plugin', () => { const api = http.createServer((req, res) => res.end('OK')) api.listen(apiPort, function () { + const apiPort = api.address().port + proxy.listen(proxyPort, function () { + const proxyPort = proxy.address().port + gateway = Gateway({ edgemicro: { port: gatewayPort, @@ -34,7 +37,10 @@ describe('Plugin', () => { ] }) - gateway.start(cb) + gateway.start((err, server) => { + gatewayPort = server.address().port + cb(err) + }) }) }) } @@ -47,12 +53,6 @@ describe('Plugin', () => { describe('microgateway-core', () => { withVersions('microgateway-core', 'microgateway-core', (version) => { - beforeEach(async () => { - gatewayPort = await getPort() - proxyPort = await getPort() - apiPort = await getPort() - }) - afterEach(() => { stopGateway() }) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index dbe311b9bf1..133a63e5697 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -175,13 +175,15 @@ class MochaPlugin extends CiPlugin { this.tracer._exporter.flush() }) - this.addSub('ci:mocha:test:finish', (status) => { + this.addSub('ci:mocha:test:finish', ({ status, hasBeenRetried }) => { const store = storage.getStore() const span = store?.span if (span) { span.setTag(TEST_STATUS, status) - + if (hasBeenRetried) { + span.setTag(TEST_IS_RETRY, 'true') + } span.finish() this.telemetry.ciVisEvent( TELEMETRY_EVENT_FINISHED, @@ -204,8 +206,8 @@ class MochaPlugin extends CiPlugin { this.addSub('ci:mocha:test:error', (err) => { const store = storage.getStore() - if (err && store && store.span) { - const span = store.span + const span = store?.span + if (err && span) { if (err.constructor.name === 'Pending' && !this.forbidPending) { span.setTag(TEST_STATUS, 'skip') } else { @@ -215,6 +217,25 @@ class MochaPlugin extends CiPlugin { } }) + this.addSub('ci:mocha:test:retry', (isFirstAttempt) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, 'fail') + if (!isFirstAttempt) { + span.setTag(TEST_IS_RETRY, 'true') + } + + span.finish() + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } + ) + finishAllTraceSpans(span) + } + }) + this.addSub('ci:mocha:test:parameterize', ({ title, params }) => { this._testTitleToParams[title] = params }) diff --git a/packages/datadog-plugin-mocha/test/index.spec.js b/packages/datadog-plugin-mocha/test/index.spec.js index 52110dedae8..15499be8638 100644 --- a/packages/datadog-plugin-mocha/test/index.spec.js +++ b/packages/datadog-plugin-mocha/test/index.spec.js @@ -4,6 +4,7 @@ const path = require('path') const fs = require('fs') const nock = require('nock') +const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ORIGIN_KEY, COMPONENT, ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') @@ -449,13 +450,27 @@ describe('Plugin', () => { }) it('works with retries', (done) => { + let testNames = [] + // retry listener did not happen until 6.0.0 + if (semver.satisfies(version, '>=6.0.0')) { + testNames = [ + ['mocha-test-retries will be retried and pass', 'fail'], + ['mocha-test-retries will be retried and pass', 'fail'], + ['mocha-test-retries will be retried and pass', 'pass'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'], + ['mocha-test-retries will be retried and fail', 'fail'] + ] + } else { + testNames = [ + ['mocha-test-retries will be retried and pass', 'pass'], + ['mocha-test-retries will be retried and fail', 'fail'] + ] + } const testFilePath = path.join(__dirname, 'mocha-test-retries.js') - const testNames = [ - ['mocha-test-retries will be retried and pass', 'pass'], - ['mocha-test-retries will be retried and fail', 'fail'] - ] - const assertionPromises = testNames.map(([testName, status]) => { return agent.use(trace => { const testSpan = trace[0][0] diff --git a/packages/datadog-plugin-net/test/index.spec.js b/packages/datadog-plugin-net/test/index.spec.js index ee91149d9ba..adcf175e405 100644 --- a/packages/datadog-plugin-net/test/index.spec.js +++ b/packages/datadog-plugin-net/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const dns = require('dns') const agent = require('../../dd-trace/test/plugins/agent') const { expectSomeSpan } = require('../../dd-trace/test/plugins/helpers') @@ -36,11 +35,7 @@ describe('Plugin', () => { tracer = require('../../dd-trace') parent = tracer.startSpan('parent') parent.finish() - - return getPort() }).then(_port => { - port = _port - return new Promise(resolve => setImmediate(resolve)) }) }) @@ -49,7 +44,10 @@ describe('Plugin', () => { tcp = new net.Server(socket => { socket.write('') }) - tcp.listen(port, () => done()) + tcp.listen(0, () => { + port = tcp.address().port + done() + }) }) beforeEach(done => { diff --git a/packages/datadog-plugin-next/test/index.spec.js b/packages/datadog-plugin-next/test/index.spec.js index 25143c14608..8c3356ddf02 100644 --- a/packages/datadog-plugin-next/test/index.spec.js +++ b/packages/datadog-plugin-next/test/index.spec.js @@ -37,7 +37,7 @@ describe('Plugin', function () { }) before(function (done) { - this.timeout(40000) + this.timeout(120000) const cwd = standalone ? path.join(__dirname, '.next/standalone') : __dirname @@ -60,15 +60,16 @@ describe('Plugin', function () { }) server.once('error', done) - server.stdout.once('data', () => { - // first log outputted isn't always the server started log - // https://github.com/vercel/next.js/blob/v10.2.0/packages/next/next-server/server/config-utils.ts#L39 - // these are webpack related logs that run during execution time and not build - - // additionally, next.js sets timeouts in 10.x when displaying extra logs - // https://github.com/vercel/next.js/blob/v10.2.0/packages/next/server/next.ts#L132-L133 - setTimeout(done, 700) // relatively high timeout chosen to be safe - }) + + function waitUntilServerStarted (chunk) { + const chunkString = chunk.toString() + if (chunkString?.includes(port) || chunkString?.includes('Ready ')) { + server.stdout.off('data', waitUntilServerStarted) + done() + } + } + server.stdout.on('data', waitUntilServerStarted) + server.stderr.on('data', chunk => process.stderr.write(chunk)) server.stdout.on('data', chunk => process.stdout.write(chunk)) }) @@ -84,7 +85,7 @@ describe('Plugin', function () { } before(async function () { - this.timeout(120 * 1000) // Webpack is very slow and builds on every test run + this.timeout(240 * 1000) // Webpack is very slow and builds on every test run const cwd = __dirname const pkg = require(`../../../versions/next@${version}/package.json`) diff --git a/packages/datadog-plugin-openai/src/index.js b/packages/datadog-plugin-openai/src/index.js index aa00beb0a44..739a80e8e7d 100644 --- a/packages/datadog-plugin-openai/src/index.js +++ b/packages/datadog-plugin-openai/src/index.js @@ -7,6 +7,7 @@ const { storage } = require('../../datadog-core') const services = require('./services') const Sampler = require('../../dd-trace/src/sampler') const { MEASURED } = require('../../../ext/tags') +const { estimateTokens } = require('./token-estimator') // String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3) const RE_NEWLINE = /\n/g @@ -15,14 +16,17 @@ const RE_TAB = /\t/g // TODO: In the future we should refactor config.js to make it requirable let MAX_TEXT_LEN = 128 -let encodingForModel -try { - // eslint-disable-next-line import/no-extraneous-dependencies - encodingForModel = require('tiktoken').encoding_for_model -} catch { - // we will use token count estimations in this case +function safeRequire (path) { + try { + // eslint-disable-next-line import/no-extraneous-dependencies + return require(path) + } catch { + return null + } } +const encodingForModel = safeRequire('tiktoken')?.encoding_for_model + class OpenApiPlugin extends TracingPlugin { static get id () { return 'openai' } static get operation () { return 'request' } @@ -305,6 +309,7 @@ class OpenApiPlugin extends TracingPlugin { } sendLog (methodName, span, tags, store, error) { + if (!store) return if (!Object.keys(store).length) return if (!this.sampler.isSampled()) return @@ -325,9 +330,22 @@ function countPromptTokens (methodName, payload, model) { const messages = payload.messages for (const message of messages) { const content = message.content - const { tokens, estimated } = countTokens(content, model) - promptTokens += tokens - promptEstimated = estimated + if (typeof content === 'string') { + const { tokens, estimated } = countTokens(content, model) + promptTokens += tokens + promptEstimated = estimated + } else if (Array.isArray(content)) { + for (const c of content) { + if (c.type === 'text') { + const { tokens, estimated } = countTokens(c.text, model) + promptTokens += tokens + promptEstimated = estimated + } + // unsupported token computation for image_url + // as even though URL is a string, its true token count + // is based on the image itself, something onerous to do client-side + } + } } } else if (methodName === 'completions.create') { let prompt = payload.prompt @@ -382,25 +400,6 @@ function countTokens (content, model) { } } -// If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens -// Approximate using the following assumptions: -// * English text -// * 1 token ~= 4 chars -// * 1 token ~= ¾ words -function estimateTokens (content) { - let estimatedTokens = 0 - if (typeof content === 'string') { - const estimation1 = content.length / 4 - - const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g) - const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string - estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2) - } else if (Array.isArray(content) && typeof content[0] === 'number') { - estimatedTokens = content.length - } - return estimatedTokens -} - function createEditRequestExtraction (tags, payload, store) { const instruction = payload.instruction tags['openai.request.instruction'] = instruction @@ -418,7 +417,7 @@ function createChatCompletionRequestExtraction (tags, payload, store) { store.messages = payload.messages for (let i = 0; i < payload.messages.length; i++) { const message = payload.messages[i] - tags[`openai.request.messages.${i}.content`] = truncateText(message.content) + tagChatCompletionRequestContent(message.content, i, tags) tags[`openai.request.messages.${i}.role`] = message.role tags[`openai.request.messages.${i}.name`] = message.name tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason @@ -707,7 +706,7 @@ function commonCreateResponseExtraction (tags, body, store, methodName) { for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { const choice = body.choices[choiceIdx] - // logprobs can be nullm and we still want to tag it as 'returned' even when set to 'null' + // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason @@ -781,6 +780,7 @@ function truncateApiKey (apiKey) { */ function truncateText (text) { if (!text) return + if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return text = text .replace(RE_NEWLINE, '\\n') @@ -793,6 +793,28 @@ function truncateText (text) { return text } +function tagChatCompletionRequestContent (contents, messageIdx, tags) { + if (typeof contents === 'string') { + tags[`openai.request.messages.${messageIdx}.content`] = contents + } else if (Array.isArray(contents)) { + // content can also be an array of objects + // which represent text input or image url + for (const contentIdx in contents) { + const content = contents[contentIdx] + const type = content.type + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type + if (type === 'text') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) + } else if (type === 'image_url') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = + truncateText(content.image_url.url) + } + // unsupported type otherwise, won't be tagged + } + } + // unsupported type otherwise, won't be tagged +} + // The server almost always responds with JSON function coerceResponseBody (body, methodName) { switch (methodName) { diff --git a/packages/datadog-plugin-openai/src/token-estimator.js b/packages/datadog-plugin-openai/src/token-estimator.js new file mode 100644 index 00000000000..46595f0c2a5 --- /dev/null +++ b/packages/datadog-plugin-openai/src/token-estimator.js @@ -0,0 +1,20 @@ +'use strict' + +// If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens +// Approximate using the following assumptions: +// * English text +// * 1 token ~= 4 chars +// * 1 token ~= ¾ words +module.exports.estimateTokens = function (content) { + let estimatedTokens = 0 + if (typeof content === 'string') { + const estimation1 = content.length / 4 + + const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g) + const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string + estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2) + } else if (Array.isArray(content) && typeof content[0] === 'number') { + estimatedTokens = content.length + } + return estimatedTokens +} diff --git a/packages/datadog-plugin-openai/test/index.spec.js b/packages/datadog-plugin-openai/test/index.spec.js index cdbcb72b969..b9db0e27c0a 100644 --- a/packages/datadog-plugin-openai/test/index.spec.js +++ b/packages/datadog-plugin-openai/test/index.spec.js @@ -715,7 +715,7 @@ describe('Plugin', () => { }) if (semver.satisfies(realVersion, '<4.0.0')) { - // `edits.create` was deprecated and removed after 4.0.0 + // `edits.create` was deprecated and removed after 4.0.0 it('makes a successful call', async () => { const checkTraces = agent .use(traces => { @@ -1124,11 +1124,11 @@ describe('Plugin', () => { const result = await openai.downloadFile('file-t3k1gVSQDHrfZnPckzftlZ4A') /** - * TODO: Seems like an OpenAI library bug? - * downloading single line JSONL file results in the JSON being converted into an object. - * downloading multi-line JSONL file then provides a basic string. - * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` - */ + * TODO: Seems like an OpenAI library bug? + * downloading single line JSONL file results in the JSON being converted into an object. + * downloading multi-line JSONL file then provides a basic string. + * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` + */ expect(result.data[0]).to.eql('{') // raw JSONL file } @@ -2655,9 +2655,9 @@ describe('Plugin', () => { expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', message: - semver.satisfies(realVersion, '>=4.0.0') - ? 'sampled chat.completions.create' - : 'sampled createChatCompletion', + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', messages: [ { role: 'user', @@ -2703,6 +2703,62 @@ describe('Plugin', () => { await checkTraces }) + + it('should tag image_url', async () => { + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + // image_url is only relevant on request/input, output has the same shape as a normal chat completion + expect(span.meta).to.have.property('openai.request.messages.0.content.0.type', 'text') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.0.text', 'I\'m allergic to peanuts. Should I avoid this food?' + ) + expect(span.meta).to.have.property('openai.request.messages.0.content.1.type', 'image_url') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.1.image_url.url', 'dummy/url/peanut_food.png' + ) + }) + + const params = { + model: 'gpt-4-visual-preview', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'I\'m allergic to peanuts. Should I avoid this food?' + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create(params) + + expect(result.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.choices[0].message.role).to.eql('assistant') + expect(result.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.choices[0].finish_reason).to.eql('length') + } else { + const result = await openai.createChatCompletion(params) + + expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.data.choices[0].message.role).to.eql('assistant') + expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.data.choices[0].finish_reason).to.eql('length') + } + + await checkTraces + }) }) describe('create chat completion with tools', () => { @@ -2809,9 +2865,9 @@ describe('Plugin', () => { expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', message: - semver.satisfies(realVersion, '>=4.0.0') - ? 'sampled chat.completions.create' - : 'sampled createChatCompletion', + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', messages: [ { role: 'user', @@ -3088,13 +3144,11 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.0.message.content', 'Hello! How can I assist you today?') - // token metrics - these should be estimated counts expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.total_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) }) const stream = await openai.chat.completions.create({ @@ -3193,7 +3247,7 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.1.message.role', 'assistant') expect(span.meta).to.have.property('openai.response.choices.1.message.content', 'I\'m just a computer program so I don\'t have feelings, ' + - 'but I\'m here and ready to help you with anything you need. How can I assis...' + 'but I\'m here and ready to help you with anything you need. How can I assis...' ) // message 2 @@ -3202,7 +3256,7 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.2.message.role', 'assistant') expect(span.meta).to.have.property('openai.response.choices.2.message.content', 'I\'m just a computer program, so I don\'t have feelings like humans do. ' + - 'I\'m here and ready to assist you with any questions or tas...' + 'I\'m here and ready to assist you with any questions or tas...' ) }) @@ -3269,6 +3323,54 @@ describe('Plugin', () => { expect(metricStub).to.have.been.calledWith('openai.tokens.total', 16, 'd', expectedTags) }) + it('makes a successful chat completion call without image_url usage computed', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + // we shouldn't be trying to capture the image_url tokens + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 1) + }) + + const stream = await openai.chat.completions.create({ + stream: 1, + model: 'gpt-4o', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'One' // one token, for ease of testing + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + }) + it('makes a successful completion call', async () => { nock('https://api.openai.com:443') .post('/v1/completions') @@ -3289,23 +3391,21 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.organization.name', 'kill-9') expect(span.meta).to.have.property('openai.request.method', 'POST') expect(span.meta).to.have.property('openai.request.endpoint', '/v1/completions') - expect(span.meta).to.have.property('openai.request.model', 'gpt-4o') + expect(span.meta).to.have.property('openai.request.model', 'text-davinci-002') expect(span.meta).to.have.property('openai.request.prompt', 'Hello, OpenAI!') expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'stop') expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') expect(span.meta).to.have.property('openai.response.choices.0.text', ' this is a test.') - // token metrics - these should be estimated counts expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.total_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) }) const stream = await openai.completions.create({ - model: 'gpt-4o', + model: 'text-davinci-002', prompt: 'Hello, OpenAI!', temperature: 0.5, stream: true diff --git a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt index 38b54feeac2..b0f9045ae9e 100644 --- a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt +++ b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt @@ -1,15 +1,15 @@ -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" ","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" ","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"this","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"this","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" test","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" test","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"stop"}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"stop"}],"model":"text-davinci-002"} data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/token-estimator.spec.js b/packages/datadog-plugin-openai/test/token-estimator.spec.js new file mode 100644 index 00000000000..375c655738a --- /dev/null +++ b/packages/datadog-plugin-openai/test/token-estimator.spec.js @@ -0,0 +1,28 @@ +'use strict' + +const { estimateTokens } = require('../src/token-estimator') + +describe('Plugin', () => { + describe('openai token estimation', () => { + function testEstimation (input, expected) { + const tokens = estimateTokens(input) + expect(tokens).to.equal(expected) + } + + it('should compute the number of tokens in a string', () => { + testEstimation('hello world', 2) + }) + + it('should not throw for an empty string', () => { + testEstimation('', 0) + }) + + it('should compute the number of tokens in an array of integer inputs', () => { + testEstimation([1, 2, 3], 3) + }) + + it('should compute no tokens for invalid content', () => { + testEstimation({}, 0) + }) + }) +}) diff --git a/packages/datadog-plugin-paperplane/test/index.spec.js b/packages/datadog-plugin-paperplane/test/index.spec.js index 167fb3841a4..5499e9cc98b 100644 --- a/packages/datadog-plugin-paperplane/test/index.spec.js +++ b/packages/datadog-plugin-paperplane/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -76,7 +75,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -113,7 +112,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -123,11 +124,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -155,7 +154,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -165,11 +166,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -191,7 +190,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -201,11 +202,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -220,7 +219,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -230,13 +231,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -247,7 +246,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -257,11 +258,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -280,7 +279,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -290,10 +291,8 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -318,17 +319,15 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -343,7 +342,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -355,13 +356,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -388,13 +389,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -405,7 +404,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -417,13 +418,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,7 +476,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -487,11 +488,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -506,7 +505,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -516,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -537,7 +536,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -547,13 +548,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) diff --git a/packages/datadog-plugin-restify/test/index.spec.js b/packages/datadog-plugin-restify/test/index.spec.js index bb96b34a132..71dc94d44a4 100644 --- a/packages/datadog-plugin-restify/test/index.spec.js +++ b/packages/datadog-plugin-restify/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -35,7 +34,9 @@ describe('Plugin', () => { it('should do automatic instrumentation', done => { const server = restify.createServer() - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('name', 'restify.request') @@ -51,11 +52,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -67,7 +66,9 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -77,11 +78,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -96,7 +95,9 @@ describe('Plugin', () => { } ) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -106,11 +107,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -135,7 +134,9 @@ describe('Plugin', () => { } ) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(warningSpy).to.not.have.been.called @@ -143,11 +144,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -184,12 +183,12 @@ describe('Plugin', () => { next() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -201,7 +200,9 @@ describe('Plugin', () => { return next() }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -211,11 +212,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -239,12 +238,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -264,7 +263,9 @@ describe('Plugin', () => { throw new Error('uncaught') }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /error') @@ -276,13 +277,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/error`, { - validateStatus: status => status === 599 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/error`, { + validateStatus: status => status === 599 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-router/test/index.spec.js b/packages/datadog-plugin-router/test/index.spec.js index 3c8e8ee68f3..ac208f0e2a1 100644 --- a/packages/datadog-plugin-router/test/index.spec.js +++ b/packages/datadog-plugin-router/test/index.spec.js @@ -5,7 +5,6 @@ const axios = require('axios') const http = require('http') const { once } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const web = require('../../dd-trace/src/plugins/util/web') @@ -87,7 +86,9 @@ describe('Plugin', () => { router.use('/parent', childRouter) - getPort().then(port => { + appListener = server(router).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -97,11 +98,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(router).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child/123`) + .catch(done) }) }) @@ -115,15 +114,15 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) } }, { rejectFirst: true }) - const httpd = server(router).listen(port, 'localhost') + const httpd = server(router).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) @@ -139,7 +138,6 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) @@ -147,8 +145,9 @@ describe('Plugin', () => { }, { rejectFirst: true }) // eslint-disable-next-line n/handle-callback-err - const httpd = server(router, (req, res) => err => res.end()).listen(port, 'localhost') + const httpd = server(router, (req, res) => err => res.end()).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) diff --git a/packages/datadog-plugin-undici/test/index.spec.js b/packages/datadog-plugin-undici/test/index.spec.js index 734e8f6c9a9..5e48be937cb 100644 --- a/packages/datadog-plugin-undici/test/index.spec.js +++ b/packages/datadog-plugin-undici/test/index.spec.js @@ -1,12 +1,14 @@ 'use strict' -const getPort = require('get-port') +const semver = require('semver') + const agent = require('../../dd-trace/test/plugins/agent') const tags = require('../../../ext/tags') const { expect } = require('chai') const { rawExpectedSchema } = require('./naming') const { DD_MAJOR } = require('../../../version') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') +const { NODE_MAJOR } = require('../../../version') const HTTP_REQUEST_HEADERS = tags.HTTP_REQUEST_HEADERS const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS @@ -20,9 +22,12 @@ describe('Plugin', () => { describe('undici-fetch', () => { withVersions('undici', 'undici', version => { - function server (app, port, listener) { + const specificVersion = require(`../../../versions/undici@${version}`).version() + if ((NODE_MAJOR <= 16) && semver.satisfies(specificVersion, '>=6')) return + + function server (app, listener) { const server = require('http').createServer(app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -59,10 +64,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) - }) + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) }) }, rawExpectedSchema.client @@ -73,7 +76,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'test') @@ -89,9 +92,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) - }) + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) }) }) @@ -100,7 +101,7 @@ describe('Plugin', () => { app.post('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -116,9 +117,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) - }) + fetch.fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) }) }) @@ -127,15 +126,13 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch((`http://localhost:${port}/user`)) - .then(res => { - expect(res).to.have.property('status', 200) - done() - }) - .catch(done) - }) + appListener = server(app, port => { + fetch.fetch((`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) }) }) @@ -146,7 +143,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -155,9 +152,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -171,7 +166,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -179,9 +174,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -196,7 +189,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -204,28 +197,24 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'undici') - }) - .then(done) - .catch(done) - - fetch.fetch(`http://localhost:${port}/user`).catch(err => { - error = err + let error + + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'undici') }) + .then(done) + .catch(done) + + fetch.fetch('http://localhost:7357/user').catch(err => { + error = err }) }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -235,7 +224,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -243,9 +232,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`) - }) + fetch.fetch(`http://localhost:${port}/user`) }) }) @@ -256,7 +243,7 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -264,9 +251,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`) - }) + fetch.fetch(`http://localhost:${port}/user`) }) }) @@ -275,7 +260,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -284,15 +269,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch.fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(() => {}) + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) - controller.abort() - }) + controller.abort() }) }) @@ -303,7 +286,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -311,15 +294,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch.fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(() => {}) + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) - controller.abort() - }) + controller.abort() }) }) }) @@ -345,7 +326,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -353,9 +334,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -382,7 +361,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -392,13 +371,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { - headers: { - 'x-baz': 'qux' - } - }).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) }) }) }) @@ -428,7 +405,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('foo', '/foo') @@ -436,9 +413,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -474,10 +449,8 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) @@ -504,7 +477,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -514,9 +487,7 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js new file mode 100644 index 00000000000..c47467528e6 --- /dev/null +++ b/packages/datadog-plugin-vitest/src/index.js @@ -0,0 +1,156 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath, + getTestSuiteCommonTags, + TEST_SOURCE_FILE +} = require('../../dd-trace/src/plugins/util/test') +const { COMPONENT } = require('../../dd-trace/src/constants') + +// Milliseconds that we subtract from the error test duration +// so that they do not overlap with the following test +// This is because there's some loss of resolution. +const MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION = 5 + +class VitestPlugin extends CiPlugin { + static get id () { + return 'vitest' + } + + constructor (...args) { + super(...args) + + this.taskToFinishTime = new WeakMap() + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const store = storage.getStore() + const span = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite + } + ) + + this.enter(span, store) + }) + + this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + const store = storage.getStore() + const span = store?.span + + // we store the finish time to finish at a later hook + // this is because the test might fail at a `afterEach` hook + if (span) { + span.setTag(TEST_STATUS, status) + this.taskToFinishTime.set(task, span._getTime()) + } + }) + + this.addSub('ci:vitest:test:pass', ({ task }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'pass') + span.finish(this.taskToFinishTime.get(task)) + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:error', ({ duration, error }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'fail') + + if (error) { + span.setTag('error', error) + } + span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite, + [TEST_STATUS]: 'skip' + } + ).finish() + }) + + this.addSub('ci:vitest:test-suite:start', (testSuiteAbsolutePath) => { + const testSessionSpanContext = this.tracer.extract('text_map', { + 'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID, + 'x-datadog-parent-id': process.env.DD_CIVISIBILITY_TEST_MODULE_ID + }) + + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( + this.command, + this.frameworkVersion, + testSuite, + 'vitest' + ) + const testSuiteSpan = this.tracer.startSpan('vitest.test_suite', { + childOf: testSessionSpanContext, + tags: { + [COMPONENT]: this.constructor.id, + ...this.testEnvironmentMetadata, + ...testSuiteMetadata + } + }) + const store = storage.getStore() + this.enter(testSuiteSpan, store) + this.testSuiteSpan = testSuiteSpan + }) + + this.addSub('ci:vitest:test-suite:finish', ({ status, onFinish }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, status) + span.finish() + finishAllTraceSpans(span) + } + // TODO: too frequent flush - find for method in worker to decrease frequency + this.tracer._exporter.flush(onFinish) + }) + + this.addSub('ci:vitest:test-suite:error', ({ error }) => { + const store = storage.getStore() + const span = store?.span + if (span && error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } + }) + + this.addSub('ci:vitest:session:finish', ({ status, onFinish, error }) => { + this.testSessionSpan.setTag(TEST_STATUS, status) + this.testModuleSpan.setTag(TEST_STATUS, status) + if (error) { + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + this.testModuleSpan.finish() + this.testSessionSpan.finish() + finishAllTraceSpans(this.testSessionSpan) + this.tracer._exporter.flush(onFinish) + }) + } +} + +module.exports = VitestPlugin diff --git a/packages/dd-trace/src/appsec/iast/path-line.js b/packages/dd-trace/src/appsec/iast/path-line.js index e77459b961d..bf7c3eb2d84 100644 --- a/packages/dd-trace/src/appsec/iast/path-line.js +++ b/packages/dd-trace/src/appsec/iast/path-line.js @@ -3,6 +3,7 @@ const path = require('path') const process = require('process') const { calculateDDBasePath } = require('../../util') +const { getCallSiteList } = require('../stack_trace') const pathLine = { getFirstNonDDPathAndLine, getNodeModulesPaths, @@ -24,24 +25,6 @@ const EXCLUDED_PATH_PREFIXES = [ 'async_hooks' ] -function getCallSiteInfo () { - const previousPrepareStackTrace = Error.prepareStackTrace - const previousStackTraceLimit = Error.stackTraceLimit - let callsiteList - Error.stackTraceLimit = 100 - try { - Error.prepareStackTrace = function (_, callsites) { - callsiteList = callsites - } - const e = new Error() - e.stack - } finally { - Error.prepareStackTrace = previousPrepareStackTrace - Error.stackTraceLimit = previousStackTraceLimit - } - return callsiteList -} - function getFirstNonDDPathAndLineFromCallsites (callsites, externallyExcludedPaths) { if (callsites) { for (let i = 0; i < callsites.length; i++) { @@ -91,7 +74,7 @@ function isExcluded (callsite, externallyExcludedPaths) { } function getFirstNonDDPathAndLine (externallyExcludedPaths) { - return getFirstNonDDPathAndLineFromCallsites(getCallSiteInfo(), externallyExcludedPaths) + return getFirstNonDDPathAndLineFromCallsites(getCallSiteList(), externallyExcludedPaths) } function getNodeModulesPaths (...paths) { diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index cd6bf5f1180..cc25d51b1e9 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -4,6 +4,7 @@ const { MANUAL_KEEP } = require('../../../../../ext/tags') const LRU = require('lru-cache') const vulnerabilitiesFormatter = require('./vulnerabilities-formatter') const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags') +const standalone = require('../standalone') const VULNERABILITIES_KEY = 'vulnerabilities' const VULNERABILITY_HASHES_MAX_SIZE = 1000 @@ -57,6 +58,9 @@ function sendVulnerabilities (vulnerabilities, rootSpan) { tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend) tags[MANUAL_KEEP] = 'true' span.addTags(tags) + + standalone.sample(span) + if (!rootSpan) span.finish() } } diff --git a/packages/dd-trace/src/appsec/index.js b/packages/dd-trace/src/appsec/index.js index af0ffc934f3..a1e2a9d395c 100644 --- a/packages/dd-trace/src/appsec/index.js +++ b/packages/dd-trace/src/appsec/index.js @@ -40,7 +40,7 @@ function enable (_config) { graphql.enable() if (_config.appsec.rasp.enabled) { - rasp.enable() + rasp.enable(_config) } setTemplates(_config) diff --git a/packages/dd-trace/src/appsec/rasp.js b/packages/dd-trace/src/appsec/rasp.js index 1a4873718b9..cede7f3355c 100644 --- a/packages/dd-trace/src/appsec/rasp.js +++ b/packages/dd-trace/src/appsec/rasp.js @@ -1,11 +1,20 @@ 'use strict' const { storage } = require('../../../datadog-core') +const web = require('./../plugins/util/web') const addresses = require('./addresses') const { httpClientRequestStart } = require('./channels') +const { reportStackTrace } = require('./stack_trace') const waf = require('./waf') -function enable () { +const RULE_TYPES = { + SSRF: 'ssrf' +} + +let config + +function enable (_config) { + config = _config httpClientRequestStart.subscribe(analyzeSsrf) } @@ -24,12 +33,30 @@ function analyzeSsrf (ctx) { [addresses.HTTP_OUTGOING_URL]: url } // TODO: Currently this is only monitoring, we should - // block the request if SSRF attempt and - // generate stack traces - waf.run({ persistent }, req) + // block the request if SSRF attempt + const result = waf.run({ persistent }, req, RULE_TYPES.SSRF) + handleResult(result, req) +} + +function getGenerateStackTraceAction (actions) { + return actions?.generate_stack +} + +function handleResult (actions, req) { + const generateStackTraceAction = getGenerateStackTraceAction(actions) + if (generateStackTraceAction && config.appsec.stackTrace.enabled) { + const rootSpan = web.root(req) + reportStackTrace( + rootSpan, + generateStackTraceAction.stack_id, + config.appsec.stackTrace.maxDepth, + config.appsec.stackTrace.maxStackTraces + ) + } } module.exports = { enable, - disable + disable, + handleResult } diff --git a/packages/dd-trace/src/appsec/recommended.json b/packages/dd-trace/src/appsec/recommended.json index 7912743b40a..0b25be934c8 100644 --- a/packages/dd-trace/src/appsec/recommended.json +++ b/packages/dd-trace/src/appsec/recommended.json @@ -1,7 +1,7 @@ { "version": "2.2", "metadata": { - "rules_version": "1.11.0" + "rules_version": "1.12.0" }, "rules": [ { @@ -1921,7 +1921,6 @@ "$ifs", "$oldpwd", "$ostype", - "$path", "$pwd", "dev/fd/", "dev/null", @@ -5849,7 +5848,8 @@ "/website.php", "/stats.php", "/assets/plugins/mp3_id/mp3_id.php", - "/siteminderagent/forms/smpwservices.fcc" + "/siteminderagent/forms/smpwservices.fcc", + "/eval-stdin.php" ] } } @@ -6236,6 +6236,155 @@ ], "transformers": [] }, + { + "id": "rasp-930-100", + "name": "Local file inclusion exploit", + "enabled": false, + "tags": { + "type": "lfi", + "category": "vulnerability_trigger", + "cwe": "22", + "capec": "1000/255/153/126", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.fs.file" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "lfi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "rasp-934-100", + "name": "Server-side request forgery exploit", + "enabled": false, + "tags": { + "type": "ssrf", + "category": "vulnerability_trigger", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.net.url" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "ssrf_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, + { + "id": "rasp-942-100", + "name": "SQL injection exploit", + "enabled": false, + "tags": { + "type": "sql_injection", + "category": "vulnerability_trigger", + "cwe": "89", + "capec": "1000/152/248/66", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.db.statement" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "db_type": [ + { + "address": "server.db.system" + } + ] + }, + "operator": "sqli_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] + }, { "id": "sqr-000-001", "name": "SSRF: Try to access the credential manager of the main cloud services", @@ -8391,6 +8540,34 @@ } ], "scanners": [ + { + "id": "406f8606-52c4-4663-8db9-df70f9e8766c", + "name": "ZIP Code", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\b(?:zip|postal)\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^[0-9]{5}(?:-[0-9]{4})?$", + "options": { + "case_sensitive": true, + "min_length": 5 + } + } + }, + "tags": { + "type": "zipcode", + "category": "address" + } + }, { "id": "JU1sRk3mSzqSUJn6GrVn7g", "name": "American Express Card Scanner (4+4+4+3 digits)", @@ -9157,6 +9334,34 @@ "category": "payment" } }, + { + "id": "18b608bd7a764bff5b2344c0", + "name": "Phone number", + "key": { + "operator": "match_regex", + "parameters": { + "regex": "\\bphone|number|mobile\\b", + "options": { + "case_sensitive": false, + "min_length": 3 + } + } + }, + "value": { + "operator": "match_regex", + "parameters": { + "regex": "^(?:\\(\\+\\d{1,3}\\)|\\+\\d{1,3}|00\\d{1,3})?[-\\s\\.]?(?:\\(\\d{3}\\)[-\\s\\.]?)?(?:\\d[-\\s\\.]?){6,10}$", + "options": { + "case_sensitive": false, + "min_length": 6 + } + } + }, + "tags": { + "type": "phone", + "category": "pii" + } + }, { "id": "de0899e0cbaaa812bb624cf04c912071012f616d-mod", "name": "UK National Insurance Number Scanner", diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index dc85d70256a..e652eaa2099 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -7,11 +7,14 @@ const { ipHeaderList } = require('../plugins/util/ip_extractor') const { incrementWafInitMetric, updateWafRequestsMetricTags, + updateRaspRequestsMetricTags, incrementWafUpdatesMetric, incrementWafRequestsMetric, getRequestMetrics } = require('./telemetry') const zlib = require('zlib') +const { MANUAL_KEEP } = require('../../../../ext/tags') +const standalone = require('./standalone') // default limiter, configurable with setRateLimit() let limiter = new Limiter(100) @@ -26,19 +29,17 @@ const contentHeaderList = [ 'content-language' ] -const REQUEST_HEADERS_MAP = mapHeaderAndTags([ +const EVENT_HEADERS_MAP = mapHeaderAndTags([ ...ipHeaderList, 'forwarded', 'via', ...contentHeaderList, 'host', - 'user-agent', - 'accept', 'accept-encoding', 'accept-language' ], 'http.request.headers.') -const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([ +const identificationHeaders = [ 'x-amzn-trace-id', 'cloudfront-viewer-ja3-fingerprint', 'cf-ray', @@ -47,6 +48,14 @@ const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([ 'x-sigsci-requestid', 'x-sigsci-tags', 'akamai-user-risk' +] + +// these request headers are always collected - it breaks the expected spec orders +const REQUEST_HEADERS_MAP = mapHeaderAndTags([ + 'content-type', + 'user-agent', + 'accept', + ...identificationHeaders ], 'http.request.headers.') const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.') @@ -87,12 +96,12 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) { metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors)) } - metricsQueue.set('manual.keep', 'true') + metricsQueue.set(MANUAL_KEEP, 'true') incrementWafInitMetric(wafVersion, rulesVersion) } -function reportMetrics (metrics) { +function reportMetrics (metrics, raspRuleType) { const store = storage.getStore() const rootSpan = store?.req && web.root(store.req) if (!rootSpan) return @@ -100,8 +109,11 @@ function reportMetrics (metrics) { if (metrics.rulesVersion) { rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion) } - - updateWafRequestsMetricTags(metrics, store.req) + if (raspRuleType) { + updateRaspRequestsMetricTags(metrics, store.req, raspRuleType) + } else { + updateWafRequestsMetricTags(metrics, store.req) + } } function reportAttack (attackData) { @@ -112,12 +124,14 @@ function reportAttack (attackData) { const currentTags = rootSpan.context()._tags - const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) - - newTags['appsec.event'] = 'true' + const newTags = { + 'appsec.event': 'true' + } if (limiter.isAllowed()) { - newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp + newTags[MANUAL_KEEP] = 'true' + + standalone.sample(rootSpan) } // TODO: maybe add this to format.js later (to take decision as late as possible) @@ -134,11 +148,6 @@ function reportAttack (attackData) { newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}' } - const ua = newTags['http.request.headers.user-agent'] - if (ua) { - newTags['http.useragent'] = ua - } - newTags['network.client.ip'] = req.socket.remoteAddress rootSpan.addTags(newTags) @@ -168,6 +177,8 @@ function finishRequest (req, res) { if (metricsQueue.size) { rootSpan.addTags(Object.fromEntries(metricsQueue)) + standalone.sample(rootSpan) + metricsQueue.clear() } @@ -180,22 +191,55 @@ function finishRequest (req, res) { rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt) } + if (metrics?.raspDuration) { + rootSpan.setTag('_dd.appsec.rasp.duration', metrics.raspDuration) + } + + if (metrics?.raspDurationExt) { + rootSpan.setTag('_dd.appsec.rasp.duration_ext', metrics.raspDurationExt) + } + + if (metrics?.raspEvalCount) { + rootSpan.setTag('_dd.appsec.rasp.rule.eval', metrics.raspEvalCount) + } + incrementWafRequestsMetric(req) // collect some headers even when no attack is detected - rootSpan.addTags(filterHeaders(req.headers, IDENTIFICATION_HEADERS_MAP)) + const mandatoryTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) + const ua = mandatoryTags['http.request.headers.user-agent'] + if (ua) { + mandatoryTags['http.useragent'] = ua + } + rootSpan.addTags(mandatoryTags) - if (!rootSpan.context()._tags['appsec.event']) return + const tags = rootSpan.context()._tags + if (!shouldCollectEventHeaders(tags)) return const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP) + Object.assign(newTags, filterHeaders(req.headers, EVENT_HEADERS_MAP)) - if (req.route && typeof req.route.path === 'string') { + if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') { newTags['http.endpoint'] = req.route.path } rootSpan.addTags(newTags) } +function shouldCollectEventHeaders (tags = {}) { + if (tags['appsec.event'] === 'true') { + return true + } + + for (const tagName of Object.keys(tags)) { + if (tagName.startsWith('appsec.events.')) { + return true + } + } + + return false +} + function setRateLimit (rateLimit) { limiter = new Limiter(rateLimit) } diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 8debb932090..61500e2cfbe 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -4,6 +4,7 @@ const log = require('../../log') const { getRootSpan } = require('./utils') const { MANUAL_KEEP } = require('../../../../../ext/tags') const { setUserTags } = require('./set_user') +const standalone = require('../standalone') function trackUserLoginSuccessEvent (tracer, user, metadata) { // TODO: better user check here and in _setUser() ? @@ -73,6 +74,8 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { } rootSpan.addTags(tags) + + standalone.sample(rootSpan) } module.exports = { diff --git a/packages/dd-trace/src/appsec/stack_trace.js b/packages/dd-trace/src/appsec/stack_trace.js new file mode 100644 index 00000000000..ea49ed1e877 --- /dev/null +++ b/packages/dd-trace/src/appsec/stack_trace.js @@ -0,0 +1,90 @@ +'use strict' + +const { calculateDDBasePath } = require('../util') + +const ddBasePath = calculateDDBasePath(__dirname) + +const LIBRARY_FRAMES_BUFFER = 20 + +function getCallSiteList (maxDepth = 100) { + const previousPrepareStackTrace = Error.prepareStackTrace + const previousStackTraceLimit = Error.stackTraceLimit + let callsiteList + Error.stackTraceLimit = maxDepth + + try { + Error.prepareStackTrace = function (_, callsites) { + callsiteList = callsites + } + const e = new Error() + e.stack + } finally { + Error.prepareStackTrace = previousPrepareStackTrace + Error.stackTraceLimit = previousStackTraceLimit + } + + return callsiteList +} + +function filterOutFramesFromLibrary (callSiteList) { + return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath)) +} + +function getFramesForMetaStruct (callSiteList, maxDepth = 32) { + const filteredFrames = filterOutFramesFromLibrary(callSiteList) + + const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity + + const indexedFrames = [] + for (let i = 0; i < Math.min(filteredFrames.length, maxDepth); i++) { + const index = i < half ? i : i + filteredFrames.length - maxDepth + const callSite = filteredFrames[index] + indexedFrames.push({ + id: index, + file: callSite.getFileName(), + line: callSite.getLineNumber(), + column: callSite.getColumnNumber(), + function: callSite.getFunctionName(), + class_name: callSite.getTypeName() + }) + } + + return indexedFrames +} + +function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) { + if (!rootSpan) return + + if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) { + // Since some frames will be discarded because they come from tracer codebase, a buffer is added + // to the limit in order to get as close as `maxDepth` number of frames. + if (maxDepth < 1) maxDepth = Infinity + const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER) + if (!Array.isArray(callSiteList)) return + + if (!rootSpan.meta_struct) { + rootSpan.meta_struct = {} + } + + if (!rootSpan.meta_struct['_dd.stack']) { + rootSpan.meta_struct['_dd.stack'] = {} + } + + if (!rootSpan.meta_struct['_dd.stack'].exploit) { + rootSpan.meta_struct['_dd.stack'].exploit = [] + } + + const frames = getFramesForMetaStruct(callSiteList, maxDepth) + + rootSpan.meta_struct['_dd.stack'].exploit.push({ + id: stackId, + language: 'nodejs', + frames + }) + } +} + +module.exports = { + getCallSiteList, + reportStackTrace +} diff --git a/packages/dd-trace/src/appsec/standalone.js b/packages/dd-trace/src/appsec/standalone.js new file mode 100644 index 00000000000..9d75dd36260 --- /dev/null +++ b/packages/dd-trace/src/appsec/standalone.js @@ -0,0 +1,130 @@ +'use strict' + +const { channel } = require('dc-polyfill') +const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority') +const { MANUAL_KEEP } = require('../../../../ext/tags') +const PrioritySampler = require('../priority_sampler') +const RateLimiter = require('../rate_limiter') +const TraceState = require('../opentracing/propagation/tracestate') +const { hasOwn } = require('../util') +const { APM_TRACING_ENABLED_KEY, APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_DEFAULT } = require('../constants') + +const startCh = channel('dd-trace:span:start') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + +let enabled + +class StandAloneAsmPrioritySampler extends PrioritySampler { + constructor (env) { + super(env, { sampleRate: 0, rateLimit: 0, rules: [] }) + + // let some regular APM traces go through, 1 per minute to keep alive the service + this._limiter = new RateLimiter(1, 'minute') + } + + configure (env, config) { + // rules not supported + this._env = env + } + + _getPriorityFromTags (tags, context) { + if (hasOwn(tags, MANUAL_KEEP) && + tags[MANUAL_KEEP] !== false && + hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY) + ) { + return USER_KEEP + } + } + + _getPriorityFromAuto (span) { + const context = this._getContext(span) + + context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT + + if (hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)) { + return USER_KEEP + } + + return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT + } +} + +function onSpanStart ({ span, fields }) { + const tags = span.context?.()?._tags + if (!tags) return + + const { parent } = fields + if (!parent || parent._isRemote) { + tags[APM_TRACING_ENABLED_KEY] = 0 + } +} + +function onSpanInject ({ spanContext, carrier }) { + if (!spanContext?._trace?.tags || !carrier) return + + // do not inject trace and sampling if there is no appsec event + if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) { + for (const key in carrier) { + const lKey = key.toLowerCase() + if (lKey.startsWith('x-datadog')) { + delete carrier[key] + } else if (lKey === 'tracestate') { + const tracestate = TraceState.fromString(carrier[key]) + tracestate.forVendor('dd', state => state.clear()) + carrier[key] = tracestate.toString() + } + } + } +} + +function onSpanExtract ({ spanContext = {} }) { + if (!spanContext._trace?.tags || !spanContext._sampling) return + + // reset upstream priority if _dd.p.appsec is not found + if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) { + spanContext._sampling.priority = undefined + } else if (spanContext._sampling.priority !== USER_KEEP) { + spanContext._sampling.priority = USER_KEEP + } +} + +function sample (span) { + const spanContext = span.context?.() + if (enabled && spanContext?._trace?.tags) { + spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1' + + // TODO: ask. can we reset here sampling like this? + if (spanContext._sampling?.priority < AUTO_KEEP) { + spanContext._sampling.priority = undefined + } + } +} + +function configure (config) { + const configChanged = enabled !== config.appsec?.standalone?.enabled + if (!configChanged) return + + enabled = config.appsec?.standalone?.enabled + + let prioritySampler + if (enabled) { + startCh.subscribe(onSpanStart) + injectCh.subscribe(onSpanInject) + extractCh.subscribe(onSpanExtract) + + prioritySampler = new StandAloneAsmPrioritySampler(config.env) + } else { + if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart) + if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject) + if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract) + } + + return prioritySampler +} + +module.exports = { + configure, + sample, + StandAloneAsmPrioritySampler +} diff --git a/packages/dd-trace/src/appsec/telemetry.js b/packages/dd-trace/src/appsec/telemetry.js index 1b7dee55ef9..6e7fd6ae47d 100644 --- a/packages/dd-trace/src/appsec/telemetry.js +++ b/packages/dd-trace/src/appsec/telemetry.js @@ -31,7 +31,10 @@ function newStore () { return { [DD_TELEMETRY_REQUEST_METRICS]: { duration: 0, - durationExt: 0 + durationExt: 0, + raspDuration: 0, + raspDurationExt: 0, + raspEvalCount: 0 } } } @@ -76,6 +79,28 @@ function getOrCreateMetricTags (store, versionsTags) { return metricTags } +function updateRaspRequestsMetricTags (metrics, req, raspRuleType) { + if (!req) return + + const store = getStore(req) + + // it does not depend on whether telemetry is enabled or not + addRaspRequestMetrics(store, metrics) + + if (!enabled) return + + const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion } + appsecMetrics.count('appsec.rasp.rule.eval', tags).inc(1) + + if (metrics.wafTimeout) { + appsecMetrics.count('appsec.rasp.timeout', tags).inc(1) + } + + if (metrics.ruleTriggered) { + appsecMetrics.count('appsec.rasp.rule.match', tags).inc(1) + } +} + function updateWafRequestsMetricTags (metrics, req) { if (!req) return @@ -141,6 +166,12 @@ function addRequestMetrics (store, { duration, durationExt }) { store[DD_TELEMETRY_REQUEST_METRICS].durationExt += durationExt || 0 } +function addRaspRequestMetrics (store, { duration, durationExt }) { + store[DD_TELEMETRY_REQUEST_METRICS].raspDuration += duration || 0 + store[DD_TELEMETRY_REQUEST_METRICS].raspDurationExt += durationExt || 0 + store[DD_TELEMETRY_REQUEST_METRICS].raspEvalCount++ +} + function getRequestMetrics (req) { if (req) { const store = getStore(req) @@ -153,6 +184,7 @@ module.exports = { disable, updateWafRequestsMetricTags, + updateRaspRequestsMetricTags, incrementWafInitMetric, incrementWafUpdatesMetric, incrementWafRequestsMetric, diff --git a/packages/dd-trace/src/appsec/waf/index.js b/packages/dd-trace/src/appsec/waf/index.js index 13190e9c7be..8aa30fabbb4 100644 --- a/packages/dd-trace/src/appsec/waf/index.js +++ b/packages/dd-trace/src/appsec/waf/index.js @@ -46,7 +46,7 @@ function update (newRules) { } } -function run (data, req) { +function run (data, req, raspRuleType) { if (!req) { const store = storage.getStore() if (!store || !store.req) { @@ -59,7 +59,7 @@ function run (data, req) { const wafContext = waf.wafManager.getWAFContext(req) - return wafContext.run(data) + return wafContext.run(data, raspRuleType) } function disposeContext (req) { diff --git a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js index 53da0d5d5df..7b437388b67 100644 --- a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +++ b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js @@ -19,7 +19,7 @@ class WAFContextWrapper { this.addressesToSkip = new Set() } - run ({ persistent, ephemeral }) { + run ({ persistent, ephemeral }, raspRuleType) { const payload = {} let payloadHasData = false const inputs = {} @@ -72,7 +72,7 @@ class WAFContextWrapper { blockTriggered, wafVersion: this.wafVersion, wafTimeout: result.timeout - }) + }, raspRuleType) if (ruleTriggered) { Reporter.reportAttack(JSON.stringify(result.events)) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index f282b27a50a..9d3f93cd5fc 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -190,7 +190,8 @@ class CiVisibilityExporter extends AgentInfoExporter { requireGit, isEarlyFlakeDetectionEnabled, earlyFlakeDetectionNumRetries, - earlyFlakeDetectionFaultyThreshold + earlyFlakeDetectionFaultyThreshold, + isFlakyTestRetriesEnabled } = remoteConfiguration return { isCodeCoverageEnabled, @@ -199,7 +200,8 @@ class CiVisibilityExporter extends AgentInfoExporter { requireGit, isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled, earlyFlakeDetectionNumRetries, - earlyFlakeDetectionFaultyThreshold + earlyFlakeDetectionFaultyThreshold, + isFlakyTestRetriesEnabled } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 34828231dfc..6a0adb0aa03 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -93,7 +93,8 @@ function getLibraryConfiguration ({ tests_skipping: isSuitesSkippingEnabled, itr_enabled: isItrEnabled, require_git: requireGit, - early_flake_detection: earlyFlakeDetectionConfig + early_flake_detection: earlyFlakeDetectionConfig, + flaky_test_retries_enabled: isFlakyTestRetriesEnabled } } } = JSON.parse(res) @@ -107,7 +108,8 @@ function getLibraryConfiguration ({ earlyFlakeDetectionNumRetries: earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES, earlyFlakeDetectionFaultyThreshold: - earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD + earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, + isFlakyTestRetriesEnabled } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 2cc7cd3d8eb..537e6c156e0 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -26,50 +26,104 @@ const telemetryCounters = { 'otel.env.invalid': {} } -function getCounter (event, ddVar, otelVar, otelTracesSamplerArg) { +function getCounter (event, ddVar, otelVar) { const counters = telemetryCounters[event] const tags = [] + const ddVarPrefix = 'config.datadog:' + const otelVarPrefix = 'config.opentelemetry:' + if (ddVar) { + ddVar = ddVarPrefix + ddVar + tags.push(ddVar) + } + if (otelVar) { + otelVar = otelVarPrefix + otelVar + tags.push(otelVar) + } - if (ddVar) tags.push(ddVar) - if (otelVar) tags.push(otelVar) - if (otelTracesSamplerArg) tags.push(otelTracesSamplerArg) - - if (!(ddVar in counters)) counters[ddVar] = {} + if (!(otelVar in counters)) counters[otelVar] = {} const counter = tracerMetrics.count(event, tags) - counters[ddVar][otelVar] = counter + counters[otelVar][ddVar] = counter return counter } const otelDdEnvMapping = { - DD_TRACE_LOG_LEVEL: 'OTEL_LOG_LEVEL', - DD_TRACE_PROPAGATION_STYLE: 'OTEL_PROPAGATORS', - DD_SERVICE: 'OTEL_SERVICE_NAME', - DD_TRACE_SAMPLE_RATE: 'OTEL_TRACES_SAMPLER', - DD_TRACE_ENABLED: 'OTEL_TRACES_EXPORTER', - DD_RUNTIME_METRICS_ENABLED: 'OTEL_METRICS_EXPORTER', - DD_TAGS: 'OTEL_RESOURCE_ATTRIBUTES', - DD_TRACE_OTEL_ENABLED: 'OTEL_SDK_DISABLED' + OTEL_LOG_LEVEL: 'DD_TRACE_LOG_LEVEL', + OTEL_PROPAGATORS: 'DD_TRACE_PROPAGATION_STYLE', + OTEL_SERVICE_NAME: 'DD_SERVICE', + OTEL_TRACES_SAMPLER: 'DD_TRACE_SAMPLE_RATE', + OTEL_TRACES_SAMPLER_ARG: 'DD_TRACE_SAMPLE_RATE', + OTEL_TRACES_EXPORTER: 'DD_TRACE_ENABLED', + OTEL_METRICS_EXPORTER: 'DD_RUNTIME_METRICS_ENABLED', + OTEL_RESOURCE_ATTRIBUTES: 'DD_TAGS', + OTEL_SDK_DISABLED: 'DD_TRACE_OTEL_ENABLED', + OTEL_LOGS_EXPORTER: undefined } -const otelInvalidEnv = ['OTEL_LOGS_EXPORTER'] +const VALID_PROPAGATION_STYLES = new Set(['datadog', 'tracecontext', 'b3', 'b3 single header', 'none']) -function checkIfBothOtelAndDdEnvVarSet () { - for (const [ddVar, otelVar] of Object.entries(otelDdEnvMapping)) { - if (process.env[ddVar] && process.env[otelVar]) { - log.warn(`both ${ddVar} and ${otelVar} environment variables are set`) - getCounter('otel.env.hiding', ddVar, otelVar, - otelVar === 'OTEL_TRACES_SAMPLER' && - process.env.OTEL_TRACES_SAMPLER_ARG - ? 'OTEL_TRACES_SAMPLER_ARG' - : undefined).inc() +const VALID_LOG_LEVELS = new Set(['debug', 'info', 'warn', 'error']) + +function getFromOtelSamplerMap (otelTracesSampler, otelTracesSamplerArg) { + const OTEL_TRACES_SAMPLER_MAPPING = { + always_on: '1.0', + always_off: '0.0', + traceidratio: otelTracesSamplerArg, + parentbased_always_on: '1.0', + parentbased_always_off: '0.0', + parentbased_traceidratio: otelTracesSamplerArg + } + return OTEL_TRACES_SAMPLER_MAPPING[otelTracesSampler] +} + +function validateOtelPropagators (propagators) { + if (!process.env.PROPAGATION_STYLE_EXTRACT && + !process.env.PROPAGATION_STYLE_INJECT && + !process.env.DD_TRACE_PROPAGATION_STYLE && + process.env.OTEL_PROPAGATORS) { + for (const style in propagators) { + if (!VALID_PROPAGATION_STYLES.has(style)) { + log.warn('unexpected value for OTEL_PROPAGATORS environment variable') + getCounter('otel.env.invalid', 'DD_TRACE_PROPAGATION_STYLE', 'OTEL_PROPAGATORS').inc() + } } } +} - for (const otelVar of otelInvalidEnv) { - if (process.env[otelVar]) { - log.warn(`${otelVar} is not supported by the Datadog SDK`) - getCounter('otel.env.invalid', otelVar).inc() +function validateEnvVarType (envVar) { + const value = process.env[envVar] + switch (envVar) { + case 'OTEL_LOG_LEVEL': + return VALID_LOG_LEVELS.has(value) + case 'OTEL_PROPAGATORS': + case 'OTEL_RESOURCE_ATTRIBUTES': + case 'OTEL_SERVICE_NAME': + return typeof value === 'string' + case 'OTEL_TRACES_SAMPLER': + return getFromOtelSamplerMap(value, process.env.OTEL_TRACES_SAMPLER_ARG) !== undefined + case 'OTEL_TRACES_SAMPLER_ARG': + return !isNaN(parseFloat(value)) + case 'OTEL_SDK_DISABLED': + return value.toLowerCase() === 'true' || value.toLowerCase() === 'false' + case 'OTEL_TRACES_EXPORTER': + case 'OTEL_METRICS_EXPORTER': + case 'OTEL_LOGS_EXPORTER': + return value.toLowerCase() === 'none' + default: + return false + } +} + +function checkIfBothOtelAndDdEnvVarSet () { + for (const [otelEnvVar, ddEnvVar] of Object.entries(otelDdEnvMapping)) { + if (ddEnvVar && process.env[ddEnvVar] && process.env[otelEnvVar]) { + log.warn(`both ${ddEnvVar} and ${otelEnvVar} environment variables are set`) + getCounter('otel.env.hiding', ddEnvVar, otelEnvVar).inc() + } + + if (process.env[otelEnvVar] && !validateEnvVarType(otelEnvVar)) { + log.warn(`unexpected value for ${otelEnvVar} environment variable`) + getCounter('otel.env.invalid', ddEnvVar, otelEnvVar).inc() } } } @@ -80,9 +134,9 @@ const fromEntries = Object.fromEntries || (entries => // eslint-disable-next-line max-len const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}' // eslint-disable-next-line max-len -const defaultWafObfuscatorKeyRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization' +const defaultWafObfuscatorKeyRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt' // eslint-disable-next-line max-len -const defaultWafObfuscatorValueRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}' +const defaultWafObfuscatorValueRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}' const runtimeId = uuid() function maybeFile (filepath) { @@ -235,6 +289,9 @@ class Config { options.tracePropagationStyle, defaultPropagationStyle ) + + validateOtelPropagators(PROPAGATION_STYLE_INJECT) + const DD_TRACE_PROPAGATION_EXTRACT_FIRST = coalesce( process.env.DD_TRACE_PROPAGATION_EXTRACT_FIRST, false @@ -440,6 +497,10 @@ class Config { this._setValue(defaults, 'appsec.rateLimit', 100) this._setValue(defaults, 'appsec.rules', undefined) this._setValue(defaults, 'appsec.sca.enabled', null) + this._setValue(defaults, 'appsec.standalone.enabled', undefined) + this._setValue(defaults, 'appsec.stackTrace.enabled', true) + this._setValue(defaults, 'appsec.stackTrace.maxDepth', 32) + this._setValue(defaults, 'appsec.stackTrace.maxStackTraces', 2) this._setValue(defaults, 'appsec.wafTimeout', 5e3) // µs this._setValue(defaults, 'clientIpEnabled', false) this._setValue(defaults, 'clientIpHeader', null) @@ -524,10 +585,13 @@ class Config { DD_APPSEC_ENABLED, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, + DD_APPSEC_MAX_STACK_TRACES, + DD_APPSEC_MAX_STACK_TRACE_DEPTH, DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP, DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP, DD_APPSEC_RULES, DD_APPSEC_SCA_ENABLED, + DD_APPSEC_STACK_TRACE_ENABLED, DD_APPSEC_RASP_ENABLED, DD_APPSEC_TRACE_RATE_LIMIT, DD_APPSEC_WAF_TIMEOUT, @@ -536,6 +600,7 @@ class Config { DD_DOGSTATSD_HOSTNAME, DD_DOGSTATSD_PORT, DD_ENV, + DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED, DD_EXPERIMENTAL_PROFILING_ENABLED, JEST_WORKER_ID, DD_IAST_DEDUPLICATION_ENABLED, @@ -627,6 +692,12 @@ class Config { this._setString(env, 'appsec.rules', DD_APPSEC_RULES) // DD_APPSEC_SCA_ENABLED is never used locally, but only sent to the backend this._setBoolean(env, 'appsec.sca.enabled', DD_APPSEC_SCA_ENABLED) + this._setBoolean(env, 'appsec.standalone.enabled', DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED) + this._setBoolean(env, 'appsec.stackTrace.enabled', DD_APPSEC_STACK_TRACE_ENABLED) + this._setValue(env, 'appsec.stackTrace.maxDepth', maybeInt(DD_APPSEC_MAX_STACK_TRACE_DEPTH)) + this._envUnprocessed['appsec.stackTrace.maxDepth'] = DD_APPSEC_MAX_STACK_TRACE_DEPTH + this._setValue(env, 'appsec.stackTrace.maxStackTraces', maybeInt(DD_APPSEC_MAX_STACK_TRACES)) + this._envUnprocessed['appsec.stackTrace.maxStackTraces'] = DD_APPSEC_MAX_STACK_TRACES this._setValue(env, 'appsec.wafTimeout', maybeInt(DD_APPSEC_WAF_TIMEOUT)) this._envUnprocessed['appsec.wafTimeout'] = DD_APPSEC_WAF_TIMEOUT this._setBoolean(env, 'clientIpEnabled', DD_TRACE_CLIENT_IP_ENABLED) @@ -701,15 +772,8 @@ class Config { : undefined this._setBoolean(env, 'runtimeMetrics', DD_RUNTIME_METRICS_ENABLED || otelSetRuntimeMetrics) - const OTEL_TRACES_SAMPLER_MAPPING = { - always_on: '1.0', - always_off: '0.0', - traceidratio: OTEL_TRACES_SAMPLER_ARG, - parentbased_always_on: '1.0', - parentbased_always_off: '0.0', - parentbased_traceidratio: OTEL_TRACES_SAMPLER_ARG - } - this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE || OTEL_TRACES_SAMPLER_MAPPING[OTEL_TRACES_SAMPLER]) + this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE || + getFromOtelSamplerMap(OTEL_TRACES_SAMPLER, OTEL_TRACES_SAMPLER_ARG)) this._setValue(env, 'sampler.rateLimit', DD_TRACE_RATE_LIMIT) this._setSamplingRule(env, 'sampler.rules', safeJsonParse(DD_TRACE_SAMPLING_RULES)) this._envUnprocessed['sampler.rules'] = DD_TRACE_SAMPLING_RULES @@ -767,6 +831,12 @@ class Config { this._setValue(opts, 'appsec.rateLimit', maybeInt(options.appsec.rateLimit)) this._optsUnprocessed['appsec.rateLimit'] = options.appsec.rateLimit this._setString(opts, 'appsec.rules', options.appsec.rules) + this._setBoolean(opts, 'appsec.standalone.enabled', options.experimental?.appsec?.standalone?.enabled) + this._setBoolean(opts, 'appsec.stackTrace.enabled', options.appsec.stackTrace?.enabled) + this._setValue(opts, 'appsec.stackTrace.maxDepth', maybeInt(options.appsec.stackTrace?.maxDepth)) + this._optsUnprocessed['appsec.stackTrace.maxDepth'] = options.appsec.stackTrace?.maxDepth + this._setValue(opts, 'appsec.stackTrace.maxStackTraces', maybeInt(options.appsec.stackTrace?.maxStackTraces)) + this._optsUnprocessed['appsec.stackTrace.maxStackTraces'] = options.appsec.stackTrace?.maxStackTraces this._setValue(opts, 'appsec.wafTimeout', maybeInt(options.appsec.wafTimeout)) this._optsUnprocessed['appsec.wafTimeout'] = options.appsec.wafTimeout this._setBoolean(opts, 'clientIpEnabled', options.clientIpEnabled) diff --git a/packages/dd-trace/src/constants.js b/packages/dd-trace/src/constants.js index 47fd67e438d..0d9bcc495dd 100644 --- a/packages/dd-trace/src/constants.js +++ b/packages/dd-trace/src/constants.js @@ -32,5 +32,7 @@ module.exports = { PEER_SERVICE_SOURCE_KEY: '_dd.peer.service.source', PEER_SERVICE_REMAP_KEY: '_dd.peer.service.remapped_from', SCI_REPOSITORY_URL: '_dd.git.repository_url', - SCI_COMMIT_SHA: '_dd.git.commit.sha' + SCI_COMMIT_SHA: '_dd.git.commit.sha', + APM_TRACING_ENABLED_KEY: '_dd.apm.enabled', + APPSEC_PROPAGATION_KEY: '_dd.p.appsec' } diff --git a/packages/dd-trace/src/datastreams/processor.js b/packages/dd-trace/src/datastreams/processor.js index 9aaaafa9de2..9a40ca153a5 100644 --- a/packages/dd-trace/src/datastreams/processor.js +++ b/packages/dd-trace/src/datastreams/processor.js @@ -211,7 +211,8 @@ class DataStreamsProcessor { Stats, TracerVersion: pkg.version, Version: this.version, - Lang: 'javascript' + Lang: 'javascript', + Tags: Object.entries(this.tags).map(([key, value]) => `${key}:${value}`) } this.writer.flush(payload) } diff --git a/packages/dd-trace/src/exporters/agent/index.js b/packages/dd-trace/src/exporters/agent/index.js index c617d27e89b..b2f25eeda99 100644 --- a/packages/dd-trace/src/exporters/agent/index.js +++ b/packages/dd-trace/src/exporters/agent/index.js @@ -7,7 +7,7 @@ const Writer = require('./writer') class AgentExporter { constructor (config, prioritySampler) { this._config = config - const { url, hostname, port, lookup, protocolVersion, stats = {} } = config + const { url, hostname, port, lookup, protocolVersion, stats = {}, appsec } = config this._url = url || new URL(format({ protocol: 'http:', hostname: hostname || 'localhost', @@ -15,7 +15,7 @@ class AgentExporter { })) const headers = {} - if (stats.enabled) { + if (stats.enabled || appsec?.standalone?.enabled) { headers['Datadog-Client-Computed-Stats'] = 'yes' } diff --git a/packages/dd-trace/src/format.js b/packages/dd-trace/src/format.js index fcb2a07d01d..1b7b86d17f0 100644 --- a/packages/dd-trace/src/format.js +++ b/packages/dd-trace/src/format.js @@ -53,6 +53,7 @@ function formatSpan (span) { resource: String(spanContext._name), error: 0, meta: {}, + meta_struct: span.meta_struct, metrics: {}, start: Math.round(span._startTime * 1e6), duration: Math.round(span._duration * 1e6), diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index 0b74674f21e..a183e977d7f 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -6,9 +6,13 @@ const DatadogSpanContext = require('../span_context') const log = require('../../log') const TraceState = require('./tracestate') const tags = require('../../../../../ext/tags') +const { channel } = require('dc-polyfill') const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + const traceKey = 'x-datadog-trace-id' const spanKey = 'x-datadog-parent-id' const originKey = 'x-datadog-origin' @@ -54,6 +58,10 @@ class TextMapPropagator { this._injectB3SingleHeader(spanContext, carrier) this._injectTraceparent(spanContext, carrier) + if (injectCh.hasSubscribers) { + injectCh.publish({ spanContext, carrier }) + } + log.debug(() => `Inject into carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) } @@ -62,6 +70,10 @@ class TextMapPropagator { if (!spanContext) return spanContext + if (extractCh.hasSubscribers) { + extractCh.publish({ spanContext, carrier }) + } + log.debug(() => `Extract from carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) return spanContext diff --git a/packages/dd-trace/src/opentracing/span.js b/packages/dd-trace/src/opentracing/span.js index f71cf329c02..723597ff043 100644 --- a/packages/dd-trace/src/opentracing/span.js +++ b/packages/dd-trace/src/opentracing/span.js @@ -99,7 +99,10 @@ class DatadogSpan { unfinishedRegistry.register(this, operationName, this) } spanleak.addSpan(this, operationName) - startCh.publish(this) + + if (startCh.hasSubscribers) { + startCh.publish({ span: this, fields }) + } } toString () { diff --git a/packages/dd-trace/src/opentracing/tracer.js b/packages/dd-trace/src/opentracing/tracer.js index 13e6b9c1500..56a6f956907 100644 --- a/packages/dd-trace/src/opentracing/tracer.js +++ b/packages/dd-trace/src/opentracing/tracer.js @@ -19,7 +19,7 @@ const REFERENCE_CHILD_OF = 'child_of' const REFERENCE_FOLLOWS_FROM = 'follows_from' class DatadogTracer { - constructor (config) { + constructor (config, prioritySampler) { const Exporter = getExporter(config.experimental.exporter) this._config = config @@ -28,7 +28,7 @@ class DatadogTracer { this._env = config.env this._logInjection = config.logInjection this._debug = config.debug - this._prioritySampler = new PrioritySampler(config.env, config.sampler) + this._prioritySampler = prioritySampler ?? new PrioritySampler(config.env, config.sampler) this._exporter = new Exporter(config, this._prioritySampler) this._processor = new SpanProcessor(this._exporter, this._prioritySampler, config) this._url = this._exporter._url diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 310cb2dc940..4daeb02a4bf 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -91,6 +91,13 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + // only for vitest + // These are added for the worker threads to use + if (this.constructor.id === 'vitest') { + process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId() + process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId() + } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 0b98cd9c076..fd9288afcc4 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -18,6 +18,7 @@ module.exports = { get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') }, get '@redis/client' () { return require('../../../datadog-plugin-redis/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, + get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, get amqp10 () { return require('../../../datadog-plugin-amqp10/src') }, get amqplib () { return require('../../../datadog-plugin-amqplib/src') }, @@ -54,6 +55,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get vitest () { return require('../../../datadog-plugin-vitest/src') }, get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') }, diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index d1d1861ea5d..6be8f831ce2 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -95,6 +95,9 @@ const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80 const EFD_STRING = "Retried by Datadog's Early Flake Detection" const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') +// Flaky test retries +const NUM_FAILED_TEST_RETRIES = 5 + module.exports = { TEST_CODE_OWNERS, TEST_FRAMEWORK, @@ -167,7 +170,8 @@ module.exports = { TEST_BROWSER_DRIVER, TEST_BROWSER_DRIVER_VERSION, TEST_BROWSER_NAME, - TEST_BROWSER_VERSION + TEST_BROWSER_VERSION, + NUM_FAILED_TEST_RETRIES } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19 diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index b3b7737fc58..aae366c2622 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -4,6 +4,7 @@ const RateLimiter = require('./rate_limiter') const Sampler = require('./sampler') const { setSamplingRules } = require('./startup-log') const SamplingRule = require('./sampling_rule') +const { hasOwn } = require('./util') const { SAMPLING_MECHANISM_DEFAULT, @@ -66,7 +67,7 @@ class PrioritySampler { if (context._sampling.priority !== undefined) return if (!root) return // noop span - const tag = this._getPriorityFromTags(context._tags) + const tag = this._getPriorityFromTags(context._tags, context) if (this.validate(tag)) { context._sampling.priority = tag @@ -202,8 +203,4 @@ class PrioritySampler { } } -function hasOwn (object, prop) { - return Object.prototype.hasOwnProperty.call(object, prop) -} - module.exports = PrioritySampler diff --git a/packages/dd-trace/src/profiling/profiler.js b/packages/dd-trace/src/profiling/profiler.js index 17422c5f993..b5b7712b691 100644 --- a/packages/dd-trace/src/profiling/profiler.js +++ b/packages/dd-trace/src/profiling/profiler.js @@ -69,7 +69,7 @@ class Profiler extends EventEmitter { setLogger(config.logger) mapper = await maybeSourceMap(config.sourceMap, SourceMapper, config.debugSourceMaps) - if (config.SourceMap && config.debugSourceMaps) { + if (config.sourceMap && config.debugSourceMaps) { this._logger.debug(() => { return mapper.infoMap.size === 0 ? 'Found no source maps' diff --git a/packages/dd-trace/src/proxy.js b/packages/dd-trace/src/proxy.js index f7b1c9e26eb..7f3a0e81780 100644 --- a/packages/dd-trace/src/proxy.js +++ b/packages/dd-trace/src/proxy.js @@ -15,6 +15,7 @@ const NoopDogStatsDClient = require('./noop/dogstatsd') const spanleak = require('./spanleak') const { SSIHeuristics } = require('./profiling/ssi-heuristics') const telemetryLog = require('dc-polyfill').channel('datadog:telemetry:log') +const appsecStandalone = require('./appsec/standalone') class LazyModule { constructor (provider) { @@ -178,7 +179,8 @@ class Tracer extends NoopProxy { this._modules.appsec.enable(config) } if (!this._tracingInitialized) { - this._tracer = new DatadogTracer(config) + const prioritySampler = appsecStandalone.configure(config) + this._tracer = new DatadogTracer(config, prioritySampler) this.appsec = new AppsecSdk(this._tracer, config) this._tracingInitialized = true } diff --git a/packages/dd-trace/src/rate_limiter.js b/packages/dd-trace/src/rate_limiter.js index 99b7ceb57ec..8417d777896 100644 --- a/packages/dd-trace/src/rate_limiter.js +++ b/packages/dd-trace/src/rate_limiter.js @@ -3,9 +3,9 @@ const limiter = require('limiter') class RateLimiter { - constructor (rateLimit) { + constructor (rateLimit, interval = 'second') { this._rateLimit = parseInt(rateLimit) - this._limiter = new limiter.RateLimiter(this._rateLimit, 'second') + this._limiter = new limiter.RateLimiter(this._rateLimit, interval) this._tokensRequested = 0 this._prevIntervalTokens = 0 this._prevTokensRequested = 0 diff --git a/packages/dd-trace/src/span_stats.js b/packages/dd-trace/src/span_stats.js index 67b8089ec16..3f7b5e34ea7 100644 --- a/packages/dd-trace/src/span_stats.js +++ b/packages/dd-trace/src/span_stats.js @@ -126,7 +126,8 @@ class SpanStatsProcessor { port, url, env, - tags + tags, + appsec } = {}) { this.exporter = new SpanStatsExporter({ hostname, @@ -138,12 +139,12 @@ class SpanStatsProcessor { this.bucketSizeNs = interval * 1e9 this.buckets = new TimeBuckets() this.hostname = os.hostname() - this.enabled = enabled + this.enabled = enabled && !appsec?.standalone?.enabled this.env = env this.tags = tags || {} this.sequence = 0 - if (enabled) { + if (this.enabled) { this.timer = setInterval(this.onInterval.bind(this), interval * 1e3) this.timer.unref() } diff --git a/packages/dd-trace/src/telemetry/init-telemetry.js b/packages/dd-trace/src/telemetry/init-telemetry.js new file mode 100644 index 00000000000..a126ecc6238 --- /dev/null +++ b/packages/dd-trace/src/telemetry/init-telemetry.js @@ -0,0 +1,75 @@ +'use strict' + +const fs = require('fs') +const { spawn } = require('child_process') +const tracerVersion = require('../../../../package.json').version +const log = require('../log') + +module.exports = sendTelemetry + +if (!process.env.DD_INJECTION_ENABLED) { + module.exports = () => {} +} + +if (!process.env.DD_TELEMETRY_FORWARDER_PATH) { + module.exports = () => {} +} + +if (!fs.existsSync(process.env.DD_TELEMETRY_FORWARDER_PATH)) { + module.exports = () => {} +} + +const metadata = { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: tracerVersion, + pid: process.pid +} + +const seen = [] +function hasSeen (point) { + if (point.name === 'abort') { + // This one can only be sent once, regardless of tags + return seen.includes('abort') + } + if (point.name === 'abort.integration') { + // For now, this is the only other one we want to dedupe + const compiledPoint = point.name + point.tags.join('') + return seen.includes(compiledPoint) + } + return false +} + +function sendTelemetry (name, tags = []) { + let points = name + if (typeof name === 'string') { + points = [{ name, tags }] + } + if (['1', 'true', 'True'].includes(process.env.DD_INJECT_FORCE)) { + points = points.filter(p => ['error', 'complete'].includes(p.name)) + } + points = points.filter(p => !hasSeen(p)) + points.forEach(p => { + p.name = `library_entrypoint.${p.name}` + }) + if (points.length === 0) { + return + } + const proc = spawn(process.env.DD_TELEMETRY_FORWARDER_PATH, ['library_entrypoint'], { + stdio: 'pipe' + }) + proc.on('error', () => { + log.error('Failed to spawn telemetry forwarder') + }) + proc.on('exit', (code) => { + if (code !== 0) { + log.error(`Telemetry forwarder exited with code ${code}`) + } + }) + proc.stdin.on('error', () => { + log.error('Failed to write telemetry data to telemetry forwarder') + }) + proc.stdin.end(JSON.stringify({ metadata, points })) +} diff --git a/packages/dd-trace/src/tracer.js b/packages/dd-trace/src/tracer.js index 5c36d0ee90e..1ed08e20373 100644 --- a/packages/dd-trace/src/tracer.js +++ b/packages/dd-trace/src/tracer.js @@ -20,8 +20,8 @@ const SERVICE_NAME = tags.SERVICE_NAME const MEASURED = tags.MEASURED class DatadogTracer extends Tracer { - constructor (config) { - super(config) + constructor (config, prioritySampler) { + super(config, prioritySampler) this._dataStreamsProcessor = new DataStreamsProcessor(config) this._scope = new Scope() setStartupLogConfig(config) diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index 77ab79aa537..04048c9b187 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -69,10 +69,15 @@ function calculateDDBasePath (dirname) { return dirSteps.slice(0, packagesIndex + 1).join(path.sep) + path.sep } +function hasOwn (object, prop) { + return Object.prototype.hasOwnProperty.call(object, prop) +} + module.exports = { isTrue, isFalse, isError, globMatch, - calculateDDBasePath + calculateDDBasePath, + hasOwn } diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js index a1cf874af04..9ea9638eddf 100644 --- a/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../plugins/agent') const { schema, @@ -41,10 +40,9 @@ withVersions('apollo-server-core', 'express', '>=4', expressVersion => { server.applyMiddleware({ app }) - config.port = await getPort() - return new Promise(resolve => { expressServer = app.listen({ port: config.port }, () => { + config.port = expressServer.address().port resolve() }) }) diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js index f6e423fb0e0..f8f4721fe3d 100644 --- a/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js +++ b/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../plugins/agent') const { schema, @@ -41,10 +40,9 @@ withVersions('apollo-server-core', 'fastify', '3', fastifyVersion => { app.register(server.createHandler()) - config.port = await getPort() - return new Promise(resolve => { app.listen({ port: config.port }, (data) => { + config.port = app.server.address().port resolve() }) }) diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js index f5b3863f963..b0f527db141 100644 --- a/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js +++ b/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const { @@ -31,9 +30,9 @@ withVersions('apollo-server', '@apollo/server', apolloServerVersion => { resolvers }) - config.port = await getPort() + const { url } = await startStandaloneServer(server, { listen: { port: 0 } }) - await startStandaloneServer(server, { listen: { port: config.port } }) + config.port = new URL(url).port }) after(async () => { diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js index 1047e3da5ae..91b6e2849f6 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../../../plugins/agent') const { schema, @@ -41,10 +40,9 @@ withVersions('graphql', 'express', '>=4', expressVersion => { server.applyMiddleware({ app }) - config.port = await getPort() - return new Promise(resolve => { expressServer = app.listen({ port: config.port }, () => { + config.port = expressServer.address().port resolve() }) }) diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js index b33820de369..bc6f0b7f079 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const path = require('path') const agent = require('../../../../plugins/agent') const { @@ -31,9 +30,9 @@ withVersions('apollo-server', '@apollo/server', apolloServerVersion => { resolvers }) - config.port = await getPort() + const { url } = await startStandaloneServer(server, { listen: { port: config.port } }) - await startStandaloneServer(server, { listen: { port: config.port } }) + config.port = new URL(url).port }) after(async () => { diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js index 7e1626a2b6f..7465f6b2408 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/sources/taint-tracking.express.plugin.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../../../plugins/agent') const Config = require('../../../../../src/config') @@ -59,13 +58,13 @@ describe('URI sourcing with express', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/path/vulnerable`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/path/vulnerable`) + .then(() => done()) + .catch(done) }) }) }) @@ -137,13 +136,13 @@ describe('Path params sourcing with express', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -172,13 +171,13 @@ describe('Path params sourcing with express', () => { app.use('/:parameterParent', nestedRouter) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -192,13 +191,13 @@ describe('Path params sourcing with express', () => { app.param('parameter1', checkParamIsTaintedAndNext) app.param('parameter2', checkParamIsTaintedAndNext) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) @@ -216,13 +215,13 @@ describe('Path params sourcing with express', () => { app.param('parameter1') app.param('parameter2') - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/tainted1/tainted2`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/tainted1/tainted2`) + .then(() => done()) + .catch(done) }) }) }) diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index ad40fd8e892..d308696e988 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -4,7 +4,6 @@ const fs = require('fs') const os = require('os') const path = require('path') -const getPort = require('get-port') const agent = require('../../plugins/agent') const axios = require('axios') const iast = require('../../../src/appsec/iast') @@ -17,12 +16,6 @@ function testInRequest (app, tests) { let appListener const config = {} - beforeEach(() => { - return getPort().then(newPort => { - config.port = newPort - }) - }) - beforeEach(() => { listener = (req, res) => { const appResult = app && app(req, res) @@ -48,7 +41,10 @@ function testInRequest (app, tests) { beforeEach(done => { const server = new http.Server(listener) appListener = server - .listen(config.port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + config.port = appListener.address().port + done() + }) }) afterEach(() => { @@ -219,12 +215,6 @@ function prepareTestServerForIast (description, tests, iastConfig) { let appListener let app - before(() => { - return getPort().then(newPort => { - config.port = newPort - }) - }) - before(() => { listener = (req, res) => { endResponse(res, app && app(req, res)) @@ -241,7 +231,10 @@ function prepareTestServerForIast (description, tests, iastConfig) { before(done => { const server = new http.Server(listener) appListener = server - .listen(config.port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + config.port = appListener.address().port + done() + }) }) beforeEachIastTest(iastConfig) @@ -311,11 +304,10 @@ function prepareTestServerForIastInExpress (description, expressVersion, loadMid } expressApp.all('/', listener) - getPort().then(newPort => { - config.port = newPort - server = expressApp.listen(newPort, () => { - done() - }) + + server = expressApp.listen(0, () => { + config.port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js index bab1f9aca53..c96171eddbd 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js @@ -1,6 +1,9 @@ const { addVulnerability, sendVulnerabilities, clearCache, start, stop } = require('../../../src/appsec/iast/vulnerability-reporter') const VulnerabilityAnalyzer = require('../../../../dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer') +const appsecStandalone = require('../../../src/appsec/standalone') +const { APPSEC_PROPAGATION_KEY } = require('../../../src/constants') + describe('vulnerability-reporter', () => { let vulnerabilityAnalyzer @@ -9,6 +12,8 @@ describe('vulnerability-reporter', () => { vulnerabilityAnalyzer = new VulnerabilityAnalyzer('ANALYZER_TYPE') }) + afterEach(sinon.restore) + describe('addVulnerability', () => { it('should not break with invalid input', () => { addVulnerability() @@ -130,10 +135,13 @@ describe('vulnerability-reporter', () => { describe('sendVulnerabilities', () => { let span + let context beforeEach(() => { + context = { _trace: { tags: {} } } span = { - addTags: sinon.stub() + addTags: sinon.stub(), + context: sinon.stub().returns(context) } start({ iast: { @@ -378,6 +386,40 @@ describe('vulnerability-reporter', () => { '{"spanId":888,"path":"filename.js","line":88}}]}' }) }) + + it('should add _dd.p.appsec trace tag with standalone enabled', () => { + appsecStandalone.configure({ appsec: { standalone: { enabled: true } } }) + const iastContext = { rootSpan: span } + addVulnerability(iastContext, + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + + sendVulnerabilities(iastContext.vulnerabilities, span) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'manual.keep': 'true', + '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' + }) + + expect(span.context()._trace.tags).to.have.property(APPSEC_PROPAGATION_KEY) + }) + + it('should not add _dd.p.appsec trace tag with standalone disabled', () => { + appsecStandalone.configure({ appsec: {} }) + const iastContext = { rootSpan: span } + addVulnerability(iastContext, + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + + sendVulnerabilities(iastContext.vulnerabilities, span) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'manual.keep': 'true', + '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + + '"evidence":{"value":"sha1"},"location":{"spanId":999}}]}' + }) + + expect(span.context()._trace.tags).to.not.have.property(APPSEC_PROPAGATION_KEY) + }) }) describe('clearCache', () => { diff --git a/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js b/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js index f48d330279c..458a69ee97d 100644 --- a/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.body-parser.plugin.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') @@ -27,11 +26,9 @@ withVersions('body-parser', 'body-parser', version => { res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(port, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js b/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js index fbc49565e2a..fed6bbcbf45 100644 --- a/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.cookie-parser.plugin.spec.js @@ -2,7 +2,6 @@ const { assert } = require('chai') const axios = require('axios') -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') @@ -28,11 +27,9 @@ withVersions('cookie-parser', 'cookie-parser', version => { res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(port, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.express.plugin.spec.js b/packages/dd-trace/test/appsec/index.express.plugin.spec.js index 5bfd37bf75a..e8b0d4a50e4 100644 --- a/packages/dd-trace/test/appsec/index.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.express.plugin.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const path = require('path') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') @@ -45,11 +44,9 @@ withVersions('express', 'express', version => { res.jsonp({ jsonResKey: 'jsonResValue' }) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(port, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js index 656532e883c..07013a570d2 100644 --- a/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js @@ -2,7 +2,6 @@ const path = require('path') const axios = require('axios') -const getPort = require('get-port') const agent = require('../plugins/agent') const appsec = require('../../src/appsec') const Config = require('../../src/config') @@ -69,11 +68,9 @@ describe('sequelize', () => { res.json(users) }) - getPort().then(newPort => { - port = newPort - server = app.listen(newPort, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) diff --git a/packages/dd-trace/test/appsec/index.spec.js b/packages/dd-trace/test/appsec/index.spec.js index 652aae028ec..747ad2d4fe4 100644 --- a/packages/dd-trace/test/appsec/index.spec.js +++ b/packages/dd-trace/test/appsec/index.spec.js @@ -19,7 +19,6 @@ const Reporter = require('../../src/appsec/reporter') const agent = require('../plugins/agent') const Config = require('../../src/config') const axios = require('axios') -const getPort = require('get-port') const blockedTemplate = require('../../src/appsec/blocked_templates') const { storage } = require('../../../datadog-core') const telemetryMetrics = require('../../src/telemetry/metrics') @@ -33,7 +32,9 @@ const resultActions = { } } -describe('AppSec Index', () => { +describe('AppSec Index', function () { + this.timeout(5000) + let config let AppSec let web @@ -200,7 +201,7 @@ describe('AppSec Index', () => { it('should call rasp enable', () => { AppSec.enable(config) - expect(rasp.enable).to.be.calledOnceWithExactly() + expect(rasp.enable).to.be.calledOnceWithExactly(config) }) it('should not call rasp enable when rasp is disabled', () => { @@ -978,7 +979,9 @@ describe('AppSec Index', () => { }) }) -describe('IP blocking', () => { +describe('IP blocking', function () { + this.timeout(5000) + const invalidIp = '1.2.3.4' const validIp = '4.3.2.1' const ruleData = { @@ -1000,11 +1003,6 @@ describe('IP blocking', () => { const jsonDefaultContent = JSON.parse(blockedTemplate.json) let http, appListener, port - before(() => { - return getPort().then(newPort => { - port = newPort - }) - }) before(() => { return agent.load('http') .then(() => { @@ -1017,7 +1015,10 @@ describe('IP blocking', () => { res.end(JSON.stringify({ message: 'OK' })) }) appListener = server - .listen(port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) beforeEach(() => { diff --git a/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js index 249af2ae727..0229984b6fa 100644 --- a/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js @@ -2,15 +2,16 @@ const Axios = require('axios') const agent = require('../plugins/agent') -const getPort = require('get-port') const appsec = require('../../src/appsec') const Config = require('../../src/config') const path = require('path') const { assert } = require('chai') +function noop () {} + withVersions('express', 'express', expressVersion => { describe('RASP', () => { - let app, server, port, axios + let app, server, axios before(() => { return agent.load(['http'], { client: false }) @@ -32,14 +33,12 @@ withVersions('express', 'express', expressVersion => { } })) - getPort().then(newPort => { - port = newPort + server = expressApp.listen(0, () => { + const port = server.address().port axios = Axios.create({ baseURL: `http://localhost:${port}` }) - server = expressApp.listen(port, () => { - done() - }) + done() }) }) @@ -65,7 +64,8 @@ withVersions('express', 'express', expressVersion => { describe(`Test using ${protocol}`, () => { it('Should not detect threat', async () => { app = (req, res) => { - require(protocol).get(`${protocol}://${req.query.host}`) + const clientRequest = require(protocol).get(`${protocol}://${req.query.host}`) + clientRequest.on('error', noop) res.end('end') } @@ -74,21 +74,27 @@ withVersions('express', 'express', expressVersion => { await agent.use((traces) => { const span = getWebSpan(traces) assert.notProperty(span.meta, '_dd.appsec.json') + assert.notProperty(span.meta_struct || {}, '_dd.stack') }) }) it('Should detect threat doing a GET request', async () => { app = (req, res) => { - require(protocol).get(`${protocol}://${req.query.host}`) + const clientRequest = require(protocol).get(`${protocol}://${req.query.host}`) + clientRequest.on('error', noop) res.end('end') } - axios.get('/?host=ifconfig.pro') + axios.get('/?host=localhost/ifconfig.pro') await agent.use((traces) => { const span = getWebSpan(traces) assert.property(span.meta, '_dd.appsec.json') assert(span.meta['_dd.appsec.json'].includes('rasp-ssrf-rule-id-1')) + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + assert(span.metrics['_dd.appsec.rasp.duration'] > 0) + assert(span.metrics['_dd.appsec.rasp.duration_ext'] > 0) + assert.property(span.meta_struct, '_dd.stack') }) }) @@ -96,17 +102,22 @@ withVersions('express', 'express', expressVersion => { app = (req, res) => { const clientRequest = require(protocol) .request(`${protocol}://${req.query.host}`, { method: 'POST' }) + clientRequest.on('error', noop) clientRequest.write('dummy_post_data') clientRequest.end() res.end('end') } - axios.get('/?host=ifconfig.pro') + axios.get('/?host=localhost/ifconfig.pro') await agent.use((traces) => { const span = getWebSpan(traces) assert.property(span.meta, '_dd.appsec.json') assert(span.meta['_dd.appsec.json'].includes('rasp-ssrf-rule-id-1')) + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + assert(span.metrics['_dd.appsec.rasp.duration'] > 0) + assert(span.metrics['_dd.appsec.rasp.duration_ext'] > 0) + assert.property(span.meta_struct, '_dd.stack') }) }) }) diff --git a/packages/dd-trace/test/appsec/rasp.spec.js b/packages/dd-trace/test/appsec/rasp.spec.js index 7f7d6dc4c50..3d0ba5da0fd 100644 --- a/packages/dd-trace/test/appsec/rasp.spec.js +++ b/packages/dd-trace/test/appsec/rasp.spec.js @@ -5,7 +5,7 @@ const { httpClientRequestStart } = require('../../src/appsec/channels') const addresses = require('../../src/appsec/addresses') describe('RASP', () => { - let waf, rasp, datadogCore + let waf, rasp, datadogCore, stackTrace, web beforeEach(() => { datadogCore = { storage: { @@ -16,18 +16,88 @@ describe('RASP', () => { run: sinon.stub() } + stackTrace = { + reportStackTrace: sinon.stub() + } + + web = { + root: sinon.stub() + } + rasp = proxyquire('../../src/appsec/rasp', { '../../../datadog-core': datadogCore, - './waf': waf + './waf': waf, + './stack_trace': stackTrace, + './../plugins/util/web': web }) - rasp.enable() + const config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + rasp.enable(config) }) afterEach(() => { + sinon.restore() rasp.disable() }) + describe('handleResult', () => { + it('should report stack trace when generate_stack action is present in waf result', () => { + const req = {} + const rootSpan = {} + const stackId = 'test_stack_id' + const result = { + generate_stack: { + stack_id: stackId + } + } + + web.root.returns(rootSpan) + + rasp.handleResult(result, req) + sinon.assert.calledOnceWithExactly(stackTrace.reportStackTrace, rootSpan, stackId, 42, 2) + }) + + it('should not report stack trace when no action is present in waf result', () => { + const req = {} + const result = {} + + rasp.handleResult(result, req) + sinon.assert.notCalled(stackTrace.reportStackTrace) + }) + + it('should not report stack trace when stack trace reporting is disabled', () => { + const req = {} + const result = { + generate_stack: { + stack_id: 'stackId' + } + } + const config = { + appsec: { + stackTrace: { + enabled: false, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + + rasp.enable(config) + + rasp.handleResult(result, req) + sinon.assert.notCalled(stackTrace.reportStackTrace) + }) + }) + describe('analyzeSsrf', () => { it('should analyze ssrf', () => { const ctx = { @@ -41,7 +111,7 @@ describe('RASP', () => { httpClientRequestStart.publish(ctx) const persistent = { [addresses.HTTP_OUTGOING_URL]: 'http://example.com' } - sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req) + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'ssrf') }) it('should not analyze ssrf if rasp is disabled', () => { diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 8f189e8a6b5..b8ce6d94fb6 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -9,6 +9,7 @@ describe('reporter', () => { let span let web let telemetry + let sample beforeEach(() => { span = { @@ -26,14 +27,20 @@ describe('reporter', () => { telemetry = { incrementWafInitMetric: sinon.stub(), updateWafRequestsMetricTags: sinon.stub(), + updateRaspRequestsMetricTags: sinon.stub(), incrementWafUpdatesMetric: sinon.stub(), incrementWafRequestsMetric: sinon.stub(), getRequestMetrics: sinon.stub() } + sample = sinon.stub() + Reporter = proxyquire('../../src/appsec/reporter', { '../plugins/util/web': web, - './telemetry': telemetry + './telemetry': telemetry, + './standalone': { + sample + } }) }) @@ -147,6 +154,7 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWithExactly(req) expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should set ext duration metrics if set', () => { @@ -155,6 +163,7 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWithExactly(req) expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should set rulesVersion if set', () => { @@ -162,6 +171,7 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWithExactly(req) expect(span.setTag).to.have.been.calledOnceWithExactly('_dd.appsec.event_rules.version', '1.2.3') + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called }) it('should call updateWafRequestsMetricTags', () => { @@ -171,6 +181,17 @@ describe('reporter', () => { Reporter.reportMetrics(metrics) expect(telemetry.updateWafRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req) + expect(telemetry.updateRaspRequestsMetricTags).to.not.have.been.called + }) + + it('should call updateRaspRequestsMetricTags when ruleType if provided', () => { + const metrics = { rulesVersion: '1.2.3' } + const store = storage.getStore() + + Reporter.reportMetrics(metrics, 'rule_type') + + expect(telemetry.updateRaspRequestsMetricTags).to.have.been.calledOnceWithExactly(metrics, store.req, 'rule_type') + expect(telemetry.updateWafRequestsMetricTags).to.not.have.been.called }) }) @@ -204,9 +225,6 @@ describe('reporter', () => { 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}', - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -246,12 +264,9 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.appsec.json': '{"triggers":[]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -264,16 +279,31 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) + + it('should call standalone sample', () => { + span.context()._tags = { '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}' } + + const result = Reporter.reportAttack('[{"rule":{}},{"rule":{},"rule_matches":[{}]}]') + expect(result).to.not.be.false + expect(web.root).to.have.been.calledOnceWith(req) + + expect(span.addTags).to.have.been.calledOnceWithExactly({ + 'appsec.event': 'true', + 'manual.keep': 'true', + '_dd.origin': 'appsec', + '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', + 'network.client.ip': '8.8.8.8' + }) + + expect(sample).to.have.been.calledOnceWithExactly(span) + }) }) describe('reportWafUpdate', () => { @@ -323,6 +353,33 @@ describe('reporter', () => { describe('finishRequest', () => { let wafContext + const requestHeadersToTrackOnEvent = [ + 'x-forwarded-for', + 'x-real-ip', + 'true-client-ip', + 'x-client-ip', + 'x-forwarded', + 'forwarded-for', + 'x-cluster-client-ip', + 'fastly-client-ip', + 'cf-connecting-ip', + 'cf-connecting-ipv6', + 'forwarded', + 'via', + 'content-length', + 'content-encoding', + 'content-language', + 'host', + 'accept-encoding', + 'accept-language' + ] + const requestHeadersAndValuesToTrackOnEvent = {} + const expectedRequestTagsToTrackOnEvent = {} + requestHeadersToTrackOnEvent.forEach((header, index) => { + requestHeadersAndValuesToTrackOnEvent[header] = `val-${index}` + expectedRequestTagsToTrackOnEvent[`http.request.headers.${header}`] = `val-${index}` + }) + beforeEach(() => { wafContext = { dispose: sinon.stub() @@ -356,7 +413,7 @@ describe('reporter', () => { expect(Reporter.metricsQueue).to.be.empty }) - it('should only add identification headers when no attack was previously found', () => { + it('should only add mandatory headers when no attack or event was previously found', () => { const req = { headers: { 'not-included': 'hello', @@ -367,7 +424,10 @@ describe('reporter', () => { 'x-appgw-trace-id': 'e', 'x-sigsci-requestid': 'f', 'x-sigsci-tags': 'g', - 'akamai-user-risk': 'h' + 'akamai-user-risk': 'h', + 'content-type': 'i', + accept: 'j', + 'user-agent': 'k' } } @@ -381,7 +441,11 @@ describe('reporter', () => { 'http.request.headers.x-appgw-trace-id': 'e', 'http.request.headers.x-sigsci-requestid': 'f', 'http.request.headers.x-sigsci-tags': 'g', - 'http.request.headers.akamai-user-risk': 'h' + 'http.request.headers.akamai-user-risk': 'h', + 'http.request.headers.content-type': 'i', + 'http.request.headers.accept': 'j', + 'http.request.headers.user-agent': 'k', + 'http.useragent': 'k' }) }) @@ -442,6 +506,108 @@ describe('reporter', () => { }) }) + it('should add http request data inside request span when appsec.event is true', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + span.context()._tags['appsec.event'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login success is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.success.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login failure is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.failure.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user custom event is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.custon.event.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + it('should call incrementWafRequestsMetric', () => { const req = {} const res = {} @@ -457,6 +623,21 @@ describe('reporter', () => { expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.waf.duration', 1337) expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.waf.duration_ext', 42) + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.duration') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.duration_ext') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.rasp.rule.eval') + }) + + it('should set rasp.duration tags if there are metrics stored', () => { + telemetry.getRequestMetrics.returns({ raspDuration: 123, raspDurationExt: 321, raspEvalCount: 3 }) + + Reporter.finishRequest({}, {}) + + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.waf.duration') + expect(span.setTag).to.not.have.been.calledWith('_dd.appsec.waf.duration_ext') + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.duration', 123) + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.duration_ext', 321) + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.rule.eval', 3) }) }) }) diff --git a/packages/dd-trace/test/appsec/response_blocking.spec.js b/packages/dd-trace/test/appsec/response_blocking.spec.js index 672933784e7..2868a42b05b 100644 --- a/packages/dd-trace/test/appsec/response_blocking.spec.js +++ b/packages/dd-trace/test/appsec/response_blocking.spec.js @@ -1,7 +1,6 @@ 'use strict' const { assert } = require('chai') -const getPort = require('get-port') const agent = require('../plugins/agent') const Axios = require('axios') const appsec = require('../../src/appsec') @@ -17,8 +16,6 @@ describe('HTTP Response Blocking', () => { let axios before(async () => { - const port = await getPort() - await agent.load('http') const http = require('http') @@ -38,16 +35,20 @@ describe('HTTP Response Blocking', () => { }) await new Promise((resolve, reject) => { - server.listen(port, 'localhost') - .once('listening', resolve) + server.listen(0, 'localhost') + .once('listening', (...args) => { + const port = server.address().port + + axios = Axios.create(({ + baseURL: `http://localhost:${port}`, + validateStatus: null + })) + + resolve(...args) + }) .once('error', reject) }) - axios = Axios.create(({ - baseURL: `http://localhost:${port}`, - validateStatus: null - })) - appsec.enable(new Config({ appsec: { enabled: true, diff --git a/packages/dd-trace/test/appsec/sdk/set_user.spec.js b/packages/dd-trace/test/appsec/sdk/set_user.spec.js index a582837e419..9327a88afcd 100644 --- a/packages/dd-trace/test/appsec/sdk/set_user.spec.js +++ b/packages/dd-trace/test/appsec/sdk/set_user.spec.js @@ -3,7 +3,6 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') const tracer = require('../../../../../index') -const getPort = require('get-port') const axios = require('axios') describe('set_user', () => { @@ -83,7 +82,6 @@ describe('set_user', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -91,7 +89,10 @@ describe('set_user', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) }) after(() => { diff --git a/packages/dd-trace/test/appsec/sdk/track_event.spec.js b/packages/dd-trace/test/appsec/sdk/track_event.spec.js index 93d8959783e..acc5db1e905 100644 --- a/packages/dd-trace/test/appsec/sdk/track_event.spec.js +++ b/packages/dd-trace/test/appsec/sdk/track_event.spec.js @@ -2,7 +2,6 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') -const getPort = require('get-port') const axios = require('axios') const tracer = require('../../../../../index') @@ -14,6 +13,7 @@ describe('track_event', () => { let getRootSpan let setUserTags let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent, trackEvent + let sample beforeEach(() => { log = { @@ -28,6 +28,8 @@ describe('track_event', () => { setUserTags = sinon.stub() + sample = sinon.stub() + const trackEvents = proxyquire('../../../src/appsec/sdk/track_event', { '../../log': log, './utils': { @@ -35,6 +37,9 @@ describe('track_event', () => { }, './set_user': { setUserTags + }, + '../standalone': { + sample } }) @@ -249,6 +254,16 @@ describe('track_event', () => { 'appsec.events.event.metakey2': 'metaValue2' }) }) + + it('should call standalone sample', () => { + trackEvent('event', undefined, 'trackEvent', rootSpan, undefined) + + expect(rootSpan.addTags).to.have.been.calledOnceWithExactly({ + 'appsec.events.event.track': 'true', + 'manual.keep': 'true' + }) + expect(sample).to.have.been.calledOnceWithExactly(rootSpan) + }) }) }) @@ -265,7 +280,6 @@ describe('track_event', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -273,7 +287,10 @@ describe('track_event', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) }) after(() => { diff --git a/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js b/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js index 04d3da4647d..30937f28de9 100644 --- a/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js +++ b/packages/dd-trace/test/appsec/sdk/user_blocking.spec.js @@ -5,7 +5,6 @@ const agent = require('../../plugins/agent') const tracer = require('../../../../../index') const appsec = require('../../../src/appsec') const Config = require('../../../src/config') -const getPort = require('get-port') const axios = require('axios') const path = require('path') const waf = require('../../../src/appsec/waf') @@ -166,7 +165,6 @@ describe('user_blocking', () => { } before(async () => { - port = await getPort() await agent.load('http') http = require('http') }) @@ -174,7 +172,10 @@ describe('user_blocking', () => { before(done => { const server = new http.Server(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) appsec.enable(config) }) diff --git a/packages/dd-trace/test/appsec/stack_trace.spec.js b/packages/dd-trace/test/appsec/stack_trace.spec.js new file mode 100644 index 00000000000..1ac2ca4db5e --- /dev/null +++ b/packages/dd-trace/test/appsec/stack_trace.spec.js @@ -0,0 +1,357 @@ +'use strict' + +const { assert } = require('chai') +const path = require('path') + +const { reportStackTrace } = require('../../src/appsec/stack_trace') + +describe('Stack trace reporter', () => { + describe('frame filtering', () => { + it('should filer out frames from library', () => { + const callSiteList = + Array(10).fill().map((_, i) => ( + { + getFileName: () => path.join(__dirname, `file${i}`), + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `libraryFunction${i}`, + getTypeName: () => `LibraryClass${i}` + } + )).concat( + Array(10).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `Class${i}` + } + )) + ).concat([ + { + getFileName: () => null, + getLineNumber: () => null, + getColumnNumber: () => null, + getFunctionName: () => null, + getTypeName: () => null + } + ]) + + const expectedFrames = Array(10).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `Class${i}` + } + )) + .concat([ + { + id: 10, + file: null, + line: null, + column: null, + function: null, + class_name: null + } + ]) + + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 32 + const maxStackTraces = 2 + reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + }) + + describe('report stack traces', () => { + const callSiteList = Array(20).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )) + + it('should not fail if no root span is passed', () => { + const rootSpan = undefined + const stackId = 'test_stack_id' + const maxDepth = 32 + try { + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + } catch (e) { + assert.fail() + } + }) + + it('should add stack trace to rootSpan when meta_struct is not present', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('should add stack trace to rootSpan when meta_struct is already present', () => { + const rootSpan = { + meta_struct: { + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace to rootSpan when meta_struct is already present and contains another stack', () => { + const rootSpan = { + meta_struct: { + another_tag: [], + '_dd.stack': { + exploit: [callSiteList] + } + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const expectedFrames = Array(20).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].id, stackId) + assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].language, 'nodejs') + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].frames, expectedFrames) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should not report stack trace when the maximum has been reached', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 2) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace when the max stack trace is 0', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, 0, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should add stack trace when the max stack trace is negative', () => { + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: [callSiteList, callSiteList] + }, + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + + reportStackTrace(rootSpan, stackId, maxDepth, -1, () => callSiteList) + + assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) + assert.property(rootSpan.meta_struct, 'another_tag') + }) + + it('should not report stackTraces if callSiteList is undefined', () => { + const rootSpan = { + meta_struct: { + another_tag: [] + } + } + const stackId = 'test_stack_id' + const maxDepth = 32 + const maxStackTraces = 2 + reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => undefined) + assert.property(rootSpan.meta_struct, 'another_tag') + assert.notProperty(rootSpan.meta_struct, '_dd.stack') + }) + }) + + describe('limit stack traces frames', () => { + const callSiteList = Array(120).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )) + + it('limit frames to max depth', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 5 + const expectedFrames = [0, 1, 2, 118, 119].map(i => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('limit frames to max depth with filtered frames', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 5 + const callSiteListWithLibraryFrames = [ + { + getFileName: () => path.join(__dirname, 'firstFrame'), + getLineNumber: () => 314, + getColumnNumber: () => 271, + getFunctionName: () => 'libraryFunction', + getTypeName: () => 'libraryType' + } + ].concat(Array(120).fill().map((_, i) => ( + { + getFileName: () => `file${i}`, + getLineNumber: () => i, + getColumnNumber: () => i, + getFunctionName: () => `function${i}`, + getTypeName: () => `type${i}` + } + )).concat([ + { + getFileName: () => path.join(__dirname, 'lastFrame'), + getLineNumber: () => 271, + getColumnNumber: () => 314, + getFunctionName: () => 'libraryFunction', + getTypeName: () => 'libraryType' + } + ])) + const expectedFrames = [0, 1, 2, 118, 119].map(i => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteListWithLibraryFrames) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('no limit if maxDepth is 0', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = 0 + const expectedFrames = Array(120).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + + it('no limit if maxDepth is negative', () => { + const rootSpan = {} + const stackId = 'test_stack_id' + const maxDepth = -1 + const expectedFrames = Array(120).fill().map((_, i) => ( + { + id: i, + file: `file${i}`, + line: i, + column: i, + function: `function${i}`, + class_name: `type${i}` + } + )) + + reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + + assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/standalone.spec.js b/packages/dd-trace/test/appsec/standalone.spec.js new file mode 100644 index 00000000000..027e23c3b5e --- /dev/null +++ b/packages/dd-trace/test/appsec/standalone.spec.js @@ -0,0 +1,455 @@ +'use strict' + +const { channel } = require('dc-polyfill') +const { assert } = require('chai') +const standalone = require('../../src/appsec/standalone') +const DatadogSpan = require('../../src/opentracing/span') +const { + APM_TRACING_ENABLED_KEY, + APPSEC_PROPAGATION_KEY, + SAMPLING_MECHANISM_APPSEC, + DECISION_MAKER_KEY +} = require('../../src/constants') +const { USER_KEEP, AUTO_KEEP, AUTO_REJECT, USER_REJECT } = require('../../../../ext/priority') +const TextMapPropagator = require('../../src/opentracing/propagation/text_map') +const TraceState = require('../../src/opentracing/propagation/tracestate') + +const startCh = channel('dd-trace:span:start') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + +describe('Appsec Standalone', () => { + let config + let tracer, processor, prioritySampler + + beforeEach(() => { + config = { + appsec: { standalone: { enabled: true } }, + + tracePropagationStyle: { + inject: ['datadog', 'tracecontext'], + extract: ['datadog'] + } + } + + tracer = {} + processor = {} + prioritySampler = {} + }) + + afterEach(() => { sinon.restore() }) + + describe('configure', () => { + let startChSubscribe + let startChUnsubscribe + let injectChSubscribe + let injectChUnsubscribe + let extractChSubscribe + let extractChUnsubscribe + + beforeEach(() => { + startChSubscribe = sinon.stub(startCh, 'subscribe') + startChUnsubscribe = sinon.stub(startCh, 'unsubscribe') + injectChSubscribe = sinon.stub(injectCh, 'subscribe') + injectChUnsubscribe = sinon.stub(injectCh, 'unsubscribe') + extractChSubscribe = sinon.stub(extractCh, 'subscribe') + extractChUnsubscribe = sinon.stub(extractCh, 'unsubscribe') + }) + + it('should subscribe to start span if standalone enabled', () => { + standalone.configure(config) + + sinon.assert.calledOnce(startChSubscribe) + sinon.assert.calledOnce(injectChSubscribe) + sinon.assert.calledOnce(extractChSubscribe) + }) + + it('should not subscribe to start span if standalone disabled', () => { + delete config.appsec.standalone + + standalone.configure(config) + + sinon.assert.notCalled(startChSubscribe) + sinon.assert.notCalled(injectChSubscribe) + sinon.assert.notCalled(extractChSubscribe) + sinon.assert.notCalled(startChUnsubscribe) + sinon.assert.notCalled(injectChUnsubscribe) + sinon.assert.notCalled(extractChUnsubscribe) + }) + + it('should subscribe only once', () => { + standalone.configure(config) + standalone.configure(config) + standalone.configure(config) + + sinon.assert.calledOnce(startChSubscribe) + }) + + it('should not return a prioritySampler when standalone ASM is disabled', () => { + const prioritySampler = standalone.configure({ appsec: { standalone: { enabled: false } } }) + + assert.isUndefined(prioritySampler) + }) + + it('should return a StandAloneAsmPrioritySampler when standalone ASM is enabled', () => { + const prioritySampler = standalone.configure(config) + + assert.instanceOf(prioritySampler, standalone.StandAloneAsmPrioritySampler) + }) + }) + + describe('sample', () => { + it('should add _dd.p.appsec tag if enabled', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + standalone.sample(span) + + assert.propertyVal(span.context()._trace.tags, APPSEC_PROPAGATION_KEY, '1') + }) + + it('should reset priority', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span.context()._sampling.priority = USER_REJECT + + standalone.sample(span) + + assert.strictEqual(span.context()._sampling.priority, undefined) + }) + + it('should not add _dd.p.appsec tag if disabled', () => { + delete config.appsec.standalone + + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + standalone.sample(span) + + assert.notProperty(span.context()._trace.tags, APPSEC_PROPAGATION_KEY) + }) + }) + + describe('onStartSpan', () => { + it('should not add _dd.apm.enabled tag when standalone is disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.notProperty(span.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should add _dd.apm.enabled tag when standalone is enabled', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.property(span.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should not add _dd.apm.enabled tag in child spans with local parent', () => { + standalone.configure(config) + + const parent = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + assert.propertyVal(parent.context()._tags, APM_TRACING_ENABLED_KEY, 0) + + const child = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation', + parent + }) + + assert.notProperty(child.context()._tags, APM_TRACING_ENABLED_KEY) + }) + + it('should add _dd.apm.enabled tag in child spans with remote parent', () => { + standalone.configure(config) + + const parent = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + parent._isRemote = true + + const child = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation', + parent + }) + + assert.propertyVal(child.context()._tags, APM_TRACING_ENABLED_KEY, 0) + }) + }) + + describe('onSpanExtract', () => { + it('should reset priority if _dd.p.appsec not present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2 + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.isUndefined(spanContext._sampling.priority) + }) + + it('should not reset dm if _dd.p.appsec not present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2, + 'x-datadog-tags': '_dd.p.dm=-4' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.propertyVal(spanContext._trace.tags, DECISION_MAKER_KEY, '-4') + }) + + it('should keep priority if _dd.p.appsec is present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2, + 'x-datadog-tags': '_dd.p.appsec=1,_dd.p.dm=-5' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + assert.propertyVal(spanContext._trace.tags, DECISION_MAKER_KEY, '-5') + }) + + it('should set USER_KEEP priority if _dd.p.appsec=1 is present', () => { + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 1, + 'x-datadog-tags': '_dd.p.appsec=1' + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + }) + + it('should keep priority if standalone is disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const carrier = { + 'x-datadog-trace-id': 123123, + 'x-datadog-parent-id': 345345, + 'x-datadog-sampling-priority': 2 + } + + const propagator = new TextMapPropagator(config) + const spanContext = propagator.extract(carrier) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + }) + }) + + describe('onSpanInject', () => { + it('should reset priority if standalone enabled and there is no appsec event', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.notProperty(carrier, 'x-datadog-trace-id') + assert.notProperty(carrier, 'x-datadog-parent-id') + assert.notProperty(carrier, 'x-datadog-sampling-priority') + }) + + it('should keep priority if standalone enabled and there is an appsec event', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + span._spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1' + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.property(carrier, 'x-datadog-trace-id') + assert.property(carrier, 'x-datadog-parent-id') + assert.property(carrier, 'x-datadog-sampling-priority') + assert.propertyVal(carrier, 'x-datadog-tags', '_dd.p.appsec=1') + }) + + it('should not reset priority if standalone disabled', () => { + delete config.appsec.standalone + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.property(carrier, 'x-datadog-trace-id') + assert.property(carrier, 'x-datadog-parent-id') + assert.property(carrier, 'x-datadog-sampling-priority') + }) + + it('should clear tracestate datadog info', () => { + standalone.configure(config) + + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + span._spanContext._sampling = { + priority: USER_KEEP, + mechanism: SAMPLING_MECHANISM_APPSEC + } + + const tracestate = new TraceState() + tracestate.set('dd', 't.tid:666b118100000000;t.dm:-1;s:1;p:73a164d716fcddff') + tracestate.set('other', 'id:0xC0FFEE') + span._spanContext._tracestate = tracestate + + const carrier = {} + const propagator = new TextMapPropagator(config) + propagator.inject(span._spanContext, carrier) + + assert.propertyVal(carrier, 'tracestate', 'other=id:0xC0FFEE') + }) + }) + + describe('StandaloneASMPriorityManager', () => { + let prioritySampler + let tags + let context + let root + + beforeEach(() => { + tags = { 'manual.keep': 'true' } + prioritySampler = new standalone.StandAloneAsmPrioritySampler('test') + + root = {} + context = { + _sampling: {}, + _trace: { + tags: {}, + started: [root] + } + } + sinon.stub(prioritySampler, '_getContext').returns(context) + }) + + describe('sample', () => { + it('should provide the context when invoking _getPriorityFromTags', () => { + const span = new DatadogSpan(tracer, processor, prioritySampler, { + operationName: 'operation' + }) + + const _getPriorityFromTags = sinon.stub(prioritySampler, '_getPriorityFromTags') + + prioritySampler.sample(span, false) + + sinon.assert.calledWithExactly(_getPriorityFromTags, context._tags, context) + }) + }) + + describe('_getPriorityFromTags', () => { + it('should keep the trace if manual.keep and _dd.p.appsec are present', () => { + context._trace.tags[APPSEC_PROPAGATION_KEY] = 1 + assert.strictEqual(prioritySampler._getPriorityFromTags(tags, context), USER_KEEP) + }) + + it('should return undefined if manual.keep or _dd.p.appsec are not present', () => { + assert.isUndefined(prioritySampler._getPriorityFromTags(tags, context)) + }) + }) + + describe('_getPriorityFromAuto', () => { + it('should keep one trace per 1 min', () => { + const span = { + _trace: {} + } + + const clock = sinon.useFakeTimers() + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_KEEP) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_REJECT) + + clock.tick(30000) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_REJECT) + + clock.tick(60000) + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_KEEP) + + clock.restore() + }) + + it('should keep trace if it contains _dd.p.appsec tag', () => { + const span = { + _trace: {} + } + + context._trace.tags[APPSEC_PROPAGATION_KEY] = 1 + + assert.strictEqual(prioritySampler._getPriorityFromAuto(span), USER_KEEP) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/telemetry.spec.js b/packages/dd-trace/test/appsec/telemetry.spec.js index c181630a3e8..0e234ad962d 100644 --- a/packages/dd-trace/test/appsec/telemetry.spec.js +++ b/packages/dd-trace/test/appsec/telemetry.spec.js @@ -161,6 +161,72 @@ describe('Appsec Telemetry metrics', () => { }) }) + describe('updateRaspRequestsMetricTags', () => { + it('should increment appsec.rasp.rule.eval metric', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('appsec.rasp.rule.eval') + expect(count).to.not.have.been.calledWith('appsec.rasp.timeout') + expect(count).to.not.have.been.calledWith('appsec.rasp.rule.match') + expect(inc).to.have.been.calledOnceWith(1) + }) + + it('should increment appsec.rasp.timeout metric if timeout', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52, + wafTimeout: true + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('appsec.rasp.rule.eval') + expect(count).to.have.been.calledWith('appsec.rasp.timeout') + expect(count).to.not.have.been.calledWith('appsec.rasp.rule.match') + expect(inc).to.have.been.calledTwice + }) + + it('should increment appsec.rasp.rule.match metric if ruleTriggered', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52, + ruleTriggered: true + }, req, 'rule-type') + + expect(count).to.have.been.calledWith('appsec.rasp.rule.match') + expect(count).to.have.been.calledWith('appsec.rasp.rule.eval') + expect(count).to.not.have.been.calledWith('appsec.rasp.timeout') + expect(inc).to.have.been.calledTwice + }) + + it('should sum rasp.duration and eval metrics', () => { + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule-type') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule-type') + + const { + duration, + durationExt, + raspDuration, + raspDurationExt, + raspEvalCount + } = appsecTelemetry.getRequestMetrics(req) + + expect(duration).to.be.eq(0) + expect(durationExt).to.be.eq(0) + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) + }) + describe('incWafInitMetric', () => { it('should increment waf.init metric', () => { appsecTelemetry.incrementWafInitMetric(wafVersion, rulesVersion) @@ -238,6 +304,35 @@ describe('Appsec Telemetry metrics', () => { }) }) + it('should not modify waf.requests metric tags when rasp rule type is provided', () => { + appsecTelemetry.updateWafRequestsMetricTags({ + blockTriggered: false, + ruleTriggered: false, + wafTimeout: false, + wafVersion, + rulesVersion + }, req) + + appsecTelemetry.updateRaspRequestsMetricTags({ + blockTriggered: true, + ruleTriggered: true, + wafTimeout: true, + wafVersion, + rulesVersion + }, req, 'rule_type') + + expect(count).to.have.not.been.calledWith('waf.requests') + appsecTelemetry.incrementWafRequestsMetric(req) + + expect(count).to.have.been.calledWithExactly('waf.requests', { + request_blocked: false, + rule_triggered: false, + waf_timeout: false, + waf_version: wafVersion, + event_rules_version: rulesVersion + }) + }) + it('should not fail if req has no previous tag', () => { appsecTelemetry.incrementWafRequestsMetric(req) @@ -316,5 +411,68 @@ describe('Appsec Telemetry metrics', () => { expect(durationExt).to.be.eq(77) }) }) + + describe('updateRaspRequestsMetricTags', () => { + it('should sum rasp.duration and rasp.durationExt request metrics', () => { + appsecTelemetry.enable({ + enabled: false, + metrics: true + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rasp_rule') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rasp_rule') + + const { raspDuration, raspDurationExt, raspEvalCount } = appsecTelemetry.getRequestMetrics(req) + + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) + + it('should sum rasp.duration and rasp.durationExt with telemetry enabled and metrics disabled', () => { + appsecTelemetry.enable({ + enabled: true, + metrics: false + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 42, + durationExt: 52 + }, req, 'rule_type') + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule_type') + + const { raspDuration, raspDurationExt, raspEvalCount } = appsecTelemetry.getRequestMetrics(req) + + expect(raspDuration).to.be.eq(66) + expect(raspDurationExt).to.be.eq(77) + expect(raspEvalCount).to.be.eq(2) + }) + + it('should not increment any metric if telemetry metrics are disabled', () => { + appsecTelemetry.enable({ + enabled: true, + metrics: false + }) + + appsecTelemetry.updateRaspRequestsMetricTags({ + duration: 24, + durationExt: 25 + }, req, 'rule_type') + + expect(count).to.not.have.been.called + expect(inc).to.not.have.been.called + }) + }) }) }) diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index 56676f1ccdd..1b313b0eaa6 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -94,6 +94,32 @@ describe('WAF Manager', () => { }) }) + describe('run', () => { + it('should call wafManager.run with raspRuleType', () => { + const run = sinon.stub() + WAFManager.prototype.getWAFContext = sinon.stub().returns({ run }) + waf.init(rules, config.appsec) + + const payload = { persistent: { 'server.io.net.url': 'http://example.com' } } + const req = {} + waf.run(payload, req, 'ssrf') + + expect(run).to.be.calledOnceWithExactly(payload, 'ssrf') + }) + + it('should call wafManager.run without raspRuleType', () => { + const run = sinon.stub() + WAFManager.prototype.getWAFContext = sinon.stub().returns({ run }) + waf.init(rules, config.appsec) + + const payload = { persistent: { 'server.io.net.url': 'http://example.com' } } + const req = {} + waf.run(payload, req) + + expect(run).to.be.calledOnceWithExactly(payload, undefined) + }) + }) + describe('wafManager.createDDWAFContext', () => { beforeEach(() => { DDWAF.prototype.constructor.version.returns('4.5.6') @@ -271,6 +297,44 @@ describe('WAF Manager', () => { expect(reportMetricsArg.ruleTriggered).to.be.true }) + it('should report raspRuleType', () => { + const result = { + totalRuntime: 1, + durationExt: 1 + } + + ddwafContext.run.returns(result) + const params = { + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } + } + + wafContextWrapper.run(params, 'rule_type') + + expect(Reporter.reportMetrics).to.be.calledOnce + expect(Reporter.reportMetrics.firstCall.args[1]).to.be.equal('rule_type') + }) + + it('should not report raspRuleType when it is not provided', () => { + const result = { + totalRuntime: 1, + durationExt: 1 + } + + ddwafContext.run.returns(result) + const params = { + persistent: { + 'server.request.headers.no_cookies': { header: 'value' } + } + } + + wafContextWrapper.run(params) + + expect(Reporter.reportMetrics).to.be.calledOnce + expect(Reporter.reportMetrics.firstCall.args[1]).to.be.equal(undefined) + }) + it('should not report attack when ddwafContext does not return events', () => { ddwafContext.run.returns({ totalRuntime: 1, durationExt: 1 }) const params = { diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index e9793664f47..fd03f17c523 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -213,9 +213,12 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.rules', undefined) expect(config).to.have.nested.property('appsec.rasp.enabled', false) expect(config).to.have.nested.property('appsec.rateLimit', 100) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', true) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 32) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 2) expect(config).to.have.nested.property('appsec.wafTimeout', 5e3) - expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex').with.length(155) - expect(config).to.have.nested.property('appsec.obfuscatorValueRegex').with.length(443) + expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex').with.length(271) + expect(config).to.have.nested.property('appsec.obfuscatorValueRegex').with.length(550) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateJson', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', undefined) @@ -224,6 +227,7 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.requestSampling', 0.1) expect(config).to.have.nested.property('appsec.sca.enabled', null) + expect(config).to.have.nested.property('appsec.standalone.enabled', undefined) expect(config).to.have.nested.property('remoteConfig.enabled', true) expect(config).to.have.nested.property('remoteConfig.pollInterval', 5) expect(config).to.have.nested.property('iast.enabled', false) @@ -244,19 +248,23 @@ describe('Config', () => { { name: 'appsec.obfuscatorKeyRegex', // eslint-disable-next-line max-len - value: '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization', + value: '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt', origin: 'default' }, { name: 'appsec.obfuscatorValueRegex', // eslint-disable-next-line max-len - value: '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}', + value: '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}', origin: 'default' }, { name: 'appsec.rasp.enabled', value: false, origin: 'default' }, { name: 'appsec.rateLimit', value: 100, origin: 'default' }, { name: 'appsec.rules', value: undefined, origin: 'default' }, { name: 'appsec.sca.enabled', value: null, origin: 'default' }, + { name: 'appsec.standalone.enabled', value: undefined, origin: 'default' }, + { name: 'appsec.stackTrace.enabled', value: true, origin: 'default' }, + { name: 'appsec.stackTrace.maxDepth', value: 32, origin: 'default' }, + { name: 'appsec.stackTrace.maxStackTraces', value: 2, origin: 'default' }, { name: 'appsec.wafTimeout', value: 5e3, origin: 'default' }, { name: 'clientIpEnabled', value: false, origin: 'default' }, { name: 'clientIpHeader', value: null, origin: 'default' }, @@ -419,8 +427,11 @@ describe('Config', () => { process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = 'true' process.env.DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = true process.env.DD_APPSEC_ENABLED = 'true' + process.env.DD_APPSEC_MAX_STACK_TRACES = '5' + process.env.DD_APPSEC_MAX_STACK_TRACE_DEPTH = '42' process.env.DD_APPSEC_RASP_ENABLED = 'true' process.env.DD_APPSEC_RULES = RULES_JSON_PATH + process.env.DD_APPSEC_STACK_TRACE_ENABLED = 'false' process.env.DD_APPSEC_TRACE_RATE_LIMIT = '42' process.env.DD_APPSEC_WAF_TIMEOUT = '42' process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = '.*' @@ -430,6 +441,7 @@ describe('Config', () => { process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = BLOCKED_TEMPLATE_GRAPHQL_PATH process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'extended' process.env.DD_APPSEC_SCA_ENABLED = true + process.env.DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED = 'true' process.env.DD_REMOTE_CONFIGURATION_ENABLED = 'false' process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = '42' process.env.DD_IAST_ENABLED = 'true' @@ -510,6 +522,9 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.rasp.enabled', true) expect(config).to.have.nested.property('appsec.rules', RULES_JSON_PATH) expect(config).to.have.nested.property('appsec.rateLimit', 42) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', false) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 42) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 5) expect(config).to.have.nested.property('appsec.wafTimeout', 42) expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex', '.*') expect(config).to.have.nested.property('appsec.obfuscatorValueRegex', '.*') @@ -521,6 +536,7 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.requestSampling', 1) expect(config).to.have.nested.property('appsec.sca.enabled', true) + expect(config).to.have.nested.property('appsec.standalone.enabled', true) expect(config).to.have.nested.property('remoteConfig.enabled', false) expect(config).to.have.nested.property('remoteConfig.pollInterval', 42) expect(config).to.have.nested.property('iast.enabled', true) @@ -549,7 +565,11 @@ describe('Config', () => { { name: 'appsec.rateLimit', value: '42', origin: 'env_var' }, { name: 'appsec.rasp.enabled', value: true, origin: 'env_var' }, { name: 'appsec.rules', value: RULES_JSON_PATH, origin: 'env_var' }, + { name: 'appsec.stackTrace.enabled', value: false, origin: 'env_var' }, + { name: 'appsec.stackTrace.maxDepth', value: '42', origin: 'env_var' }, + { name: 'appsec.stackTrace.maxStackTraces', value: '5', origin: 'env_var' }, { name: 'appsec.sca.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.standalone.enabled', value: true, origin: 'env_var' }, { name: 'appsec.wafTimeout', value: '42', origin: 'env_var' }, { name: 'clientIpEnabled', value: true, origin: 'env_var' }, { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'env_var' }, @@ -732,6 +752,11 @@ describe('Config', () => { redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN', telemetryVerbosity: 'DEBUG' + }, + appsec: { + standalone: { + enabled: true + } } }, appsec: false, @@ -780,6 +805,7 @@ describe('Config', () => { expect(config).to.have.nested.property('experimental.exporter', 'log') expect(config).to.have.nested.property('experimental.enableGetRumData', true) expect(config).to.have.nested.property('appsec.enabled', false) + expect(config).to.have.nested.property('appsec.standalone.enabled', true) expect(config).to.have.nested.property('remoteConfig.pollInterval', 42) expect(config).to.have.nested.property('iast.enabled', true) expect(config).to.have.nested.property('iast.requestSampling', 50) @@ -815,6 +841,7 @@ describe('Config', () => { expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ { name: 'appsec.enabled', value: false, origin: 'code' }, + { name: 'appsec.standalone.enabled', value: true, origin: 'code' }, { name: 'clientIpEnabled', value: true, origin: 'code' }, { name: 'clientIpHeader', value: 'x-true-client-ip', origin: 'code' }, { name: 'dogstatsd.hostname', value: 'agent-dsd', origin: 'code' }, @@ -1006,8 +1033,11 @@ describe('Config', () => { process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED = 'true' process.env.DD_TRACE_EXPERIMENTAL_INTERNAL_ERRORS_ENABLED = 'true' process.env.DD_APPSEC_ENABLED = 'false' + process.env.DD_APPSEC_MAX_STACK_TRACES = '11' + process.env.DD_APPSEC_MAX_STACK_TRACE_DEPTH = '11' process.env.DD_APPSEC_RASP_ENABLED = 'true' process.env.DD_APPSEC_RULES = RECOMMENDED_JSON_PATH + process.env.DD_APPSEC_STACK_TRACE_ENABLED = 'true' process.env.DD_APPSEC_TRACE_RATE_LIMIT = 11 process.env.DD_APPSEC_WAF_TIMEOUT = 11 process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = '^$' @@ -1089,6 +1119,11 @@ describe('Config', () => { }, rasp: { enabled: false + }, + stackTrace: { + enabled: false, + maxDepth: 42, + maxStackTraces: 5 } }, remoteConfig: { @@ -1131,6 +1166,9 @@ describe('Config', () => { expect(config).to.have.nested.property('appsec.rasp.enabled', false) expect(config).to.have.nested.property('appsec.rules', RULES_JSON_PATH) expect(config).to.have.nested.property('appsec.rateLimit', 42) + expect(config).to.have.nested.property('appsec.stackTrace.enabled', false) + expect(config).to.have.nested.property('appsec.stackTrace.maxDepth', 42) + expect(config).to.have.nested.property('appsec.stackTrace.maxStackTraces', 5) expect(config).to.have.nested.property('appsec.wafTimeout', 42) expect(config).to.have.nested.property('appsec.obfuscatorKeyRegex', '.*') expect(config).to.have.nested.property('appsec.obfuscatorValueRegex', '.*') @@ -1217,6 +1255,14 @@ describe('Config', () => { }, rasp: { enabled: false + }, + standalone: { + enabled: undefined + }, + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 32 } }) }) diff --git a/packages/dd-trace/test/datastreams/processor.spec.js b/packages/dd-trace/test/datastreams/processor.spec.js index fe2f31e0c36..fdf80b8bdb8 100644 --- a/packages/dd-trace/test/datastreams/processor.spec.js +++ b/packages/dd-trace/test/datastreams/processor.spec.js @@ -203,7 +203,7 @@ describe('DataStreamsProcessor', () => { env: 'test', version: 'v1', service: 'service1', - tags: { tag: 'some tag' } + tags: { foo: 'foovalue', bar: 'barvalue' } } beforeEach(() => { @@ -306,7 +306,8 @@ describe('DataStreamsProcessor', () => { Backlogs: [] }], TracerVersion: pkg.version, - Lang: 'javascript' + Lang: 'javascript', + Tags: ['foo:foovalue', 'bar:barvalue'] }) }) }) diff --git a/packages/dd-trace/test/exporters/agent/exporter.spec.js b/packages/dd-trace/test/exporters/agent/exporter.spec.js index 9bfeb2ec04f..a92a4d975de 100644 --- a/packages/dd-trace/test/exporters/agent/exporter.spec.js +++ b/packages/dd-trace/test/exporters/agent/exporter.spec.js @@ -43,6 +43,18 @@ describe('Exporter', () => { }) }) + it('should pass computed stats header through to writer if standalone appsec is enabled', () => { + const stats = { enabled: false } + const appsec = { standalone: { enabled: true } } + exporter = new Exporter({ url, flushInterval, stats, appsec }, prioritySampler) + + expect(Writer).to.have.been.calledWithMatch({ + headers: { + 'Datadog-Client-Computed-Stats': 'yes' + } + }) + }) + it('should support IPv6', () => { const stats = { enabled: true } exporter = new Exporter({ hostname: '::1', flushInterval, stats }, prioritySampler) diff --git a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js index 9bc86bc16ff..e6a206a8bbd 100644 --- a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js @@ -6,11 +6,15 @@ const Config = require('../../../src/config') const id = require('../../../src/id') const SpanContext = require('../../../src/opentracing/span_context') const TraceState = require('../../../src/opentracing/propagation/tracestate') +const { channel } = require('dc-polyfill') const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority') const { SAMPLING_MECHANISM_MANUAL } = require('../../../src/constants') const { expect } = require('chai') +const injectCh = channel('dd-trace:span:inject') +const extractCh = channel('dd-trace:span:extract') + describe('TextMapPropagator', () => { let TextMapPropagator let propagator @@ -319,6 +323,26 @@ describe('TextMapPropagator', () => { expect(carrier).to.not.have.property('x-datadog-origin') expect(carrier).to.not.have.property('x-datadog-tags') }) + + it('should publish spanContext and carrier', () => { + const carrier = {} + const spanContext = createContext({ + traceId: id('0000000000000123'), + spanId: id('0000000000000456') + }) + + const onSpanInject = sinon.stub() + injectCh.subscribe(onSpanInject) + + propagator.inject(spanContext, carrier) + + try { + expect(onSpanInject).to.be.calledOnce + expect(onSpanInject.firstCall.args[0]).to.be.deep.equal({ spanContext, carrier }) + } finally { + injectCh.unsubscribe(onSpanInject) + } + }) }) describe('extract', () => { @@ -551,6 +575,21 @@ describe('TextMapPropagator', () => { expect(spanContext._trace.tags).to.have.property('_dd.parent_id', '0000000000000001') }) + it('should publish spanContext and carrier', () => { + const onSpanExtract = sinon.stub() + extractCh.subscribe(onSpanExtract) + + const carrier = textMap + const spanContext = propagator.extract(carrier) + + try { + expect(onSpanExtract).to.be.calledOnce + expect(onSpanExtract.firstCall.args[0]).to.be.deep.equal({ spanContext, carrier }) + } finally { + extractCh.unsubscribe(onSpanExtract) + } + }) + describe('with B3 propagation as multiple headers', () => { beforeEach(() => { config.tracePropagationStyle.extract = ['b3multi'] diff --git a/packages/dd-trace/test/opentracing/span.spec.js b/packages/dd-trace/test/opentracing/span.spec.js index 326796cae26..dbb248eb920 100644 --- a/packages/dd-trace/test/opentracing/span.spec.js +++ b/packages/dd-trace/test/opentracing/span.spec.js @@ -5,6 +5,9 @@ require('../setup/tap') const Config = require('../../src/config') const TextMapPropagator = require('../../src/opentracing/propagation/text_map') +const { channel } = require('dc-polyfill') +const startCh = channel('dd-trace:span:start') + describe('Span', () => { let Span let span @@ -163,6 +166,24 @@ describe('Span', () => { expect(span.context()._trace.tags['_dd.p.tid']).to.match(/^[a-f0-9]{8}0{8}$/) }) + it('should be published via dd-trace:span:start channel', () => { + const onSpan = sinon.stub() + startCh.subscribe(onSpan) + + const fields = { + operationName: 'operation' + } + + try { + span = new Span(tracer, processor, prioritySampler, fields) + + expect(onSpan).to.have.been.calledOnce + expect(onSpan.firstCall.args[0]).to.deep.equal({ span, fields }) + } finally { + startCh.unsubscribe(onSpan) + } + }) + describe('tracer', () => { it('should return its parent tracer', () => { span = new Span(tracer, processor, prioritySampler, { operationName: 'operation' }) diff --git a/packages/dd-trace/test/opentracing/tracer.spec.js b/packages/dd-trace/test/opentracing/tracer.spec.js index 2b6927e99d4..1a6ae261f0b 100644 --- a/packages/dd-trace/test/opentracing/tracer.spec.js +++ b/packages/dd-trace/test/opentracing/tracer.spec.js @@ -103,6 +103,14 @@ describe('Tracer', () => { expect(SpanProcessor).to.have.been.calledWith(agentExporter, prioritySampler, config) }) + it('should allow to configure an alternative prioritySampler', () => { + const sampler = {} + tracer = new Tracer(config, sampler) + + expect(AgentExporter).to.have.been.calledWith(config, sampler) + expect(SpanProcessor).to.have.been.calledWith(agentExporter, sampler, config) + }) + describe('startSpan', () => { it('should start a span', () => { fields.tags = { foo: 'bar' } diff --git a/packages/dd-trace/test/plugins/agent.js b/packages/dd-trace/test/plugins/agent.js index 8b64a7b74da..06644401bb4 100644 --- a/packages/dd-trace/test/plugins/agent.js +++ b/packages/dd-trace/test/plugins/agent.js @@ -4,7 +4,6 @@ const http = require('http') const bodyParser = require('body-parser') const msgpack = require('msgpack-lite') const codec = msgpack.createCodec({ int64: true }) -const getPort = require('get-port') const express = require('express') const path = require('path') const ritm = require('../../src/ritm') @@ -270,8 +269,6 @@ module.exports = { res.status(200).send() }) - const port = await getPort() - const server = this.server = http.createServer(agent) const emit = server.emit @@ -283,7 +280,25 @@ module.exports = { server.on('connection', socket => sockets.push(socket)) const promise = new Promise((resolve, reject) => { - listener = server.listen(port, () => resolve()) + listener = server.listen(0, () => { + const port = listener.address().port + + tracer.init(Object.assign({}, { + service: 'test', + env: 'tester', + port, + flushInterval: 0, + plugins: false + }, tracerConfig)) + + tracer.setUrl(`http://127.0.0.1:${port}`) + + for (let i = 0, l = pluginName.length; i < l; i++) { + tracer.use(pluginName[i], config[i]) + } + + resolve() + }) }) pluginName = [].concat(pluginName) @@ -295,20 +310,6 @@ module.exports = { dsmStats = [] }) - tracer.init(Object.assign({}, { - service: 'test', - env: 'tester', - port, - flushInterval: 0, - plugins: false - }, tracerConfig)) - - tracer.setUrl(`http://127.0.0.1:${port}`) - - for (let i = 0, l = pluginName.length; i < l; i++) { - tracer.use(pluginName[i], config[i]) - } - return promise }, diff --git a/packages/dd-trace/test/proxy.spec.js b/packages/dd-trace/test/proxy.spec.js index 07fcd41eca6..6b694ea805f 100644 --- a/packages/dd-trace/test/proxy.spec.js +++ b/packages/dd-trace/test/proxy.spec.js @@ -523,6 +523,30 @@ describe('TracerProxy', () => { expect(telemetry.start).to.have.been.called }) + + it('should configure appsec standalone', () => { + const standalone = { + configure: sinon.stub() + } + + const options = {} + const DatadogProxy = proxyquire('../src/proxy', { + './tracer': DatadogTracer, + './config': Config, + './appsec': appsec, + './appsec/iast': iast, + './appsec/remote_config': remoteConfig, + './appsec/sdk': AppsecSdk, + './appsec/standalone': standalone, + './telemetry': telemetry + }) + + const proxy = new DatadogProxy() + proxy.init(options) + + const config = AppsecSdk.firstCall.args[1] + expect(standalone.configure).to.have.been.calledOnceWithExactly(config) + }) }) describe('trace', () => { diff --git a/packages/dd-trace/test/setup/mocha.js b/packages/dd-trace/test/setup/mocha.js index 15131c2946d..e3a6496bb7c 100644 --- a/packages/dd-trace/test/setup/mocha.js +++ b/packages/dd-trace/test/setup/mocha.js @@ -85,7 +85,8 @@ function withNamingSchema ( const { opName, serviceName } = expected[versionName] - it('should conform to the naming schema', () => { + it('should conform to the naming schema', function () { + this.timeout(10000) return new Promise((resolve, reject) => { agent .use(traces => { @@ -203,10 +204,14 @@ function withVersions (plugin, modules, range, cb) { versions .filter(version => !process.env.RANGE || semver.subset(version, process.env.RANGE)) .forEach(version => { - const min = semver.coerce(version).version + if (version !== '*') { + const min = semver.coerce(version).version + + testVersions.set(min, { range: version, test: min }) + } + const max = require(`../../../../versions/${moduleName}@${version}`).version() - testVersions.set(min, { range: version, test: min }) testVersions.set(max, { range: version, test: version }) }) }) diff --git a/packages/dd-trace/test/span_stats.spec.js b/packages/dd-trace/test/span_stats.spec.js index 77f59f68641..3ffdd4899ab 100644 --- a/packages/dd-trace/test/span_stats.spec.js +++ b/packages/dd-trace/test/span_stats.spec.js @@ -255,6 +255,14 @@ describe('SpanStatsProcessor', () => { expect(processor.tags).to.deep.equal(config.tags) }) + it('should construct a disabled instance if appsec standalone is enabled', () => { + const standaloneConfig = { appsec: { standalone: { enabled: true } }, ...config } + const processor = new SpanStatsProcessor(standaloneConfig) + + expect(processor.enabled).to.be.false + expect(processor.timer).to.be.undefined + }) + it('should track span stats', () => { expect(processor.buckets.size).to.equal(0) for (let i = 0; i < n; i++) { diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index f61774f5619..5d7fed89bdc 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -12,12 +12,12 @@ const externals = require('../packages/dd-trace/test/plugins/externals') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') +// Can remove aerospike after removing support for aerospike < 5.2.0 (for Node.js 22, v5.12.1 is required) // Can remove couchbase after removing support for couchbase <= 3.2.0 -const excludeList = os.arch() === 'arm64' ? ['couchbase', 'grpc', 'oracledb'] : [] +const excludeList = os.arch() === 'arm64' ? ['aerospike', 'couchbase', 'grpc', 'oracledb'] : [] const workspaces = new Set() const versionLists = {} const deps = {} -const names = [] const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') Object.keys(externals).forEach(external => externals[external].forEach(thing => { @@ -29,15 +29,10 @@ Object.keys(externals).forEach(external => externals[external].forEach(thing => } })) -fs.readdirSync(path.join(__dirname, '../packages/datadog-instrumentations/src')) - .filter(file => file.endsWith('js')) - .forEach(file => { - file = file.replace('.js', '') - - if (!filter || filter.includes(file)) { - names.push(file) - } - }) +const names = fs.readdirSync(path.join(__dirname, '..', 'packages', 'datadog-instrumentations', 'src')) + .filter(file => file.endsWith('.js')) + .map(file => file.slice(0, -3)) + .filter(file => !filter || filter.includes(file)) run() @@ -80,13 +75,16 @@ async function assertVersions () { } async function assertInstrumentation (instrumentation, external) { - const versions = process.env.PACKAGE_VERSION_RANGE + const versions = process.env.PACKAGE_VERSION_RANGE && !external ? [process.env.PACKAGE_VERSION_RANGE] : [].concat(instrumentation.versions || []) for (const version of versions) { if (version) { - await assertModules(instrumentation.name, semver.coerce(version).version, external) + if (version !== '*') { + await assertModules(instrumentation.name, semver.coerce(version).version, external) + } + await assertModules(instrumentation.name, version, external) } } diff --git a/static-analysis.datadog.yml b/static-analysis.datadog.yml new file mode 100644 index 00000000000..a46fba39176 --- /dev/null +++ b/static-analysis.datadog.yml @@ -0,0 +1,4 @@ +rulesets: + - sit-ci-best-practices: + only: + - ".github/workflows" diff --git a/yarn.lock b/yarn.lock index fcbb9cd6c3f..e60e7587969 100644 --- a/yarn.lock +++ b/yarn.lock @@ -427,10 +427,10 @@ lru-cache "^7.14.0" node-gyp-build "^4.5.0" -"@datadog/native-iast-taint-tracking@2.1.0": - version "2.1.0" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-2.1.0.tgz#65e0350f04064a991e3a980daf1b68147069c9f2" - integrity sha512-DjZ6itJcjLrTdKk2vP96hak2xS0ABd0NIB8poZG3OBQU5efkzu8JOQoxbIKMklG/0P2zh7EquvGP88PdVXT9aA== +"@datadog/native-iast-taint-tracking@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.0.0.tgz#a0be16f49f49c2a5917d08e5986c0fc6c877ed13" + integrity sha512-V+25+edlNCQSNRUvL45IajN+CFEjii9NbjfSMG6HRHbH/zeLL9FCNE+GU88dwB1bqXKNpBdrIxsfgTN65Yq9tA== dependencies: node-gyp-build "^3.9.0" @@ -2824,10 +2824,10 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" -import-in-the-middle@^1.7.4: - version "1.7.4" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.7.4.tgz#508da6e91cfa84f210dcdb6c0a91ab0c9e8b3ebc" - integrity sha512-Lk+qzWmiQuRPPulGQeK5qq0v32k2bHnWrRPFgqyvhw7Kkov5L6MOLOIU3pcWeujc9W4q54Cp3Q2WV16eQkc7Bg== +import-in-the-middle@^1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.8.1.tgz#8b51c2cc631b64e53e958d7048d2d9463ce628f8" + integrity sha512-yhRwoHtiLGvmSozNOALgjRPFI6uYsds60EoMqqnXyyv+JOIW/BrrLejuTGBt+bq0T5tLzOHrN0T7xYTm4Qt/ng== dependencies: acorn "^8.8.2" acorn-import-attributes "^1.9.5" @@ -4874,6 +4874,11 @@ through@^2.3.8, through@~2.3.4: resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +tiktoken@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/tiktoken/-/tiktoken-1.0.15.tgz#a1e11681fa51b50c81bb7eaaee53b7a66e844a23" + integrity sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw== + tildify@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz"