Exploit Prevention LFI (#4676)

* rasp lfi and iast using rasp fs-plugin

* Add rasp lfi capability in RC

* Handle aborted operations in fs instrumentation

* enable test without express

* cleanup and console log to debug test error

* Do not throw

* another test

* Try increasing timeout

* Enable debug again

* Enable debug again

* increase timeout a lot

* increase timeout more

* New lfi test

* Increase test timeout

* print all errors

* remote debug info

* Handle the different invocation cases

* Handle non string properties

* specify types to be analyzed

* a bunch of tests

* clean up

* rasp lfi subs delayed (#4715)

* Delay Appsec fs plugin subscription to fs:operations until the first req is received

* disable rasp in tests

* fix tests recursive call

* Avoid multiple subscriptions to incomingHttpRequestStart

* another try

* replace spy with stub

* execute unsubscribe asynchronously

* sinon.assert async

* clarify comment

* Use a constant

* Do not enable rasp in some tests

* Remove not needed config property

* Rename properties

* Test iast and rasp fs-plugin subscription order

* Avoid multiple analyzeLfi subscriptions

* Block synchronous operations

* Include synchronous blocking integration test

* Test refactor

* rename test file

* Cleanup

packages/datadog-instrumentations/src/express.js

@@ -3,6 +3,7 @@
 const { createWrapRouterMethod } = require('./router')
 const shimmer = require('../../datadog-shimmer')
 const { addHook, channel } = require('./helpers/instrument')
+const tracingChannel = require('dc-polyfill').tracingChannel
 
 const handleChannel = channel('apm:express:request:handle')
 
@@ -35,13 +36,35 @@ function wrapResponseJson (json) {
   }
 }
 
+const responseRenderChannel = tracingChannel('datadog:express:response:render')
+
+function wrapResponseRender (render) {
+  return function wrappedRender (view, options, callback) {
+    if (!responseRenderChannel.start.hasSubscribers) {
+      return render.apply(this, arguments)
+    }
+
+    return responseRenderChannel.traceSync(
+      render,
+      {
+        req: this.req,
+        view,
+        options
+      },
+      this,
+      ...arguments
+    )
+  }
+}
+
 addHook({ name: 'express', versions: ['>=4'] }, express => {
   shimmer.wrap(express.application, 'handle', wrapHandle)
   shimmer.wrap(express.Router, 'use', wrapRouterMethod)
   shimmer.wrap(express.Router, 'route', wrapRouterMethod)
 
   shimmer.wrap(express.response, 'json', wrapResponseJson)
   shimmer.wrap(express.response, 'jsonp', wrapResponseJson)
+  shimmer.wrap(express.response, 'render', wrapResponseRender)
 
   return express
 })

packages/datadog-instrumentations/src/fs.js

@@ -266,24 +266,44 @@ function createWrapFunction (prefix = '', override = '') {
       const lastIndex = arguments.length - 1
       const cb = typeof arguments[lastIndex] === 'function' && arguments[lastIndex]
       const innerResource = new AsyncResource('bound-anonymous-fn')
-      const message = getMessage(method, getMethodParamsRelationByPrefix(prefix)[operation], arguments, this)
+      const params = getMethodParamsRelationByPrefix(prefix)[operation]
+      const abortController = new AbortController()
+      const message = { ...getMessage(method, params, arguments, this), abortController }
+
+      const finish = innerResource.bind(function (error) {
+        if (error !== null && typeof error === 'object') { // fs.exists receives a boolean
+          errorChannel.publish(error)
+        }
+        finishChannel.publish()
+      })
 
       if (cb) {
         const outerResource = new AsyncResource('bound-anonymous-fn')
 
         arguments[lastIndex] = shimmer.wrapFunction(cb, cb => innerResource.bind(function (e) {
-          if (e !== null && typeof e === 'object') { // fs.exists receives a boolean
-            errorChannel.publish(e)
-          }
-
-          finishChannel.publish()
-
+          finish(e)
           return outerResource.runInAsyncScope(() => cb.apply(this, arguments))
         }))
       }
 
       return innerResource.runInAsyncScope(() => {
         startChannel.publish(message)
+
+        if (abortController.signal.aborted) {
+          const error = abortController.signal.reason || new Error('Aborted')
+
+          if (prefix === 'promises.') {
+            finish(error)
+            return Promise.reject(error)
+          } else if (name.includes('Sync') || !cb) {
+            finish(error)
+            throw error
+          } else if (cb) {
+            arguments[lastIndex](error)
+            return
+          }
+        }
+
         try {
           const result = original.apply(this, arguments)
           if (cb) return result

packages/dd-trace/src/appsec/addresses.js

@@ -23,6 +23,8 @@ module.exports = {
   WAF_CONTEXT_PROCESSOR: 'waf.context.processor',
 
   HTTP_OUTGOING_URL: 'server.io.net.url',
+  FS_OPERATION_PATH: 'server.io.fs.file',
+
   DB_STATEMENT: 'server.db.statement',
   DB_SYSTEM: 'server.db.system'
 }

packages/dd-trace/src/appsec/channels.js

@@ -25,5 +25,7 @@ module.exports = {
   pgQueryStart: dc.channel('apm:pg:query:start'),
   pgPoolQueryStart: dc.channel('datadog:pg:pool:query:start'),
   mysql2OuterQueryStart: dc.channel('datadog:mysql2:outerquery:start'),
-  wafRunFinished: dc.channel('datadog:waf:run:finish')
+  wafRunFinished: dc.channel('datadog:waf:run:finish'),
+  fsOperationStart: dc.channel('apm:fs:operation:start'),
+  expressMiddlewareError: dc.channel('apm:express:middleware:error')
 }

packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js

@@ -29,7 +29,14 @@ class PathTraversalAnalyzer extends InjectionAnalyzer {
 
   onConfigure () {
     this.addSub('apm:fs:operation:start', (obj) => {
-      if (ignoredOperations.includes(obj.operation)) return
+      const store = storage.getStore()
+      const outOfReqOrChild = !store?.fs?.root
+
+      // we could filter out all the nested fs.operations based on store.fs.root
+      // but if we spect a store in the context to be present we are going to exclude
+      // all out_of_the_request fs.operations
+      // AppsecFsPlugin must be enabled
+      if (ignoredOperations.includes(obj.operation) || outOfReqOrChild) return
 
       const pathArguments = []
       if (obj.dest) {

packages/dd-trace/src/appsec/iast/index.js

@@ -14,6 +14,7 @@ const {
 } = require('./taint-tracking')
 const { IAST_ENABLED_TAG_KEY } = require('./tags')
 const iastTelemetry = require('./telemetry')
+const { enable: enableFsPlugin, disable: disableFsPlugin, IAST_MODULE } = require('../rasp/fs-plugin')
 
 // TODO Change to `apm:http:server:request:[start|close]` when the subscription
 //  order of the callbacks can be enforce
@@ -27,6 +28,7 @@ function enable (config, _tracer) {
   if (isEnabled) return
 
   iastTelemetry.configure(config, config.iast?.telemetryVerbosity)
+  enableFsPlugin(IAST_MODULE)
   enableAllAnalyzers(config)
   enableTaintTracking(config.iast, iastTelemetry.verbosity)
   requestStart.subscribe(onIncomingHttpRequestStart)
@@ -44,6 +46,7 @@ function disable () {
   isEnabled = false
 
   iastTelemetry.stop()
+  disableFsPlugin(IAST_MODULE)
   disableAllAnalyzers()
   disableTaintTracking()
   overheadController.finishGlobalContext()

packages/dd-trace/src/appsec/rasp/fs-plugin.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const Plugin = require('../../plugins/plugin')
+const { storage } = require('../../../../datadog-core')
+const log = require('../../log')
+
+const RASP_MODULE = 'rasp'
+const IAST_MODULE = 'iast'
+
+const enabledFor = {
+  [RASP_MODULE]: false,
+  [IAST_MODULE]: false
+}
+
+let fsPlugin
+
+function enterWith (fsProps, store = storage.getStore()) {
+  if (store && !store.fs?.opExcluded) {
+    storage.enterWith({
+      ...store,
+      fs: {
+        ...store.fs,
+        ...fsProps,
+        parentStore: store
+      }
+    })
+  }
+}
+
+class AppsecFsPlugin extends Plugin {
+  enable () {
+    this.addSub('apm:fs:operation:start', this._onFsOperationStart)
+    this.addSub('apm:fs:operation:finish', this._onFsOperationFinishOrRenderEnd)
+    this.addSub('tracing:datadog:express:response:render:start', this._onResponseRenderStart)
+    this.addSub('tracing:datadog:express:response:render:end', this._onFsOperationFinishOrRenderEnd)
+
+    super.configure(true)
+  }
+
+  disable () {
+    super.configure(false)
+  }
+
+  _onFsOperationStart () {
+    const store = storage.getStore()
+    if (store) {
+      enterWith({ root: store.fs?.root === undefined }, store)
+    }
+  }
+
+  _onResponseRenderStart () {
+    enterWith({ opExcluded: true })
+  }
+
+  _onFsOperationFinishOrRenderEnd () {
+    const store = storage.getStore()
+    if (store?.fs?.parentStore) {
+      storage.enterWith(store.fs.parentStore)
+    }
+  }
+}
+
+function enable (mod) {
+  if (enabledFor[mod] !== false) return
+
+  enabledFor[mod] = true
+
+  if (!fsPlugin) {
+    fsPlugin = new AppsecFsPlugin()
+    fsPlugin.enable()
+  }
+
+  log.info(`Enabled AppsecFsPlugin for ${mod}`)
+}
+
+function disable (mod) {
+  if (!mod || !enabledFor[mod]) return
+
+  enabledFor[mod] = false
+
+  const allDisabled = Object.values(enabledFor).every(val => val === false)
+  if (allDisabled) {
+    fsPlugin?.disable()
+
+    fsPlugin = undefined
+  }
+
+  log.info(`Disabled AppsecFsPlugin for ${mod}`)
+}
+
+module.exports = {
+  enable,
+  disable,
+
+  AppsecFsPlugin,
+
+  RASP_MODULE,
+  IAST_MODULE
+}

packages/dd-trace/src/appsec/rasp/index.js

@@ -1,10 +1,11 @@
 'use strict'
 
 const web = require('../../plugins/util/web')
-const { setUncaughtExceptionCaptureCallbackStart } = require('../channels')
-const { block } = require('../blocking')
+const { setUncaughtExceptionCaptureCallbackStart, expressMiddlewareError } = require('../channels')
+const { block, isBlocked } = require('../blocking')
 const ssrf = require('./ssrf')
 const sqli = require('./sql_injection')
+const lfi = require('./lfi')
 
 const { DatadogRaspAbortError } = require('./utils')
 
@@ -30,17 +31,13 @@ function findDatadogRaspAbortError (err, deep = 10) {
     return err
   }
 
-  if (err.cause && deep > 0) {
+  if (err?.cause && deep > 0) {
     return findDatadogRaspAbortError(err.cause, deep - 1)
   }
 }
 
-function handleUncaughtExceptionMonitor (err) {
-  const abortError = findDatadogRaspAbortError(err)
-  if (!abortError) return
-
-  const { req, res, blockingAction } = abortError
-  block(req, res, web.root(req), null, blockingAction)
+function handleUncaughtExceptionMonitor (error) {
+  if (!blockOnDatadogRaspAbortError({ error })) return
 
   if (!process.hasUncaughtExceptionCaptureCallback()) {
     const cleanUp = removeAllListeners(process, 'uncaughtException')
@@ -82,22 +79,39 @@ function handleUncaughtExceptionMonitor (err) {
   }
 }
 
+function blockOnDatadogRaspAbortError ({ error }) {
+  const abortError = findDatadogRaspAbortError(error)
+  if (!abortError) return false
+
+  const { req, res, blockingAction } = abortError
+  if (!isBlocked(res)) {
+    block(req, res, web.root(req), null, blockingAction)
+  }
+
+  return true
+}
+
 function enable (config) {
   ssrf.enable(config)
   sqli.enable(config)
+  lfi.enable(config)
 
   process.on('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor)
+  expressMiddlewareError.subscribe(blockOnDatadogRaspAbortError)
 }
 
 function disable () {
   ssrf.disable()
   sqli.disable()
+  lfi.disable()
 
   process.off('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor)
+  if (expressMiddlewareError.hasSubscribers) expressMiddlewareError.unsubscribe(blockOnDatadogRaspAbortError)
 }
 
 module.exports = {
   enable,
   disable,
-  handleUncaughtExceptionMonitor // exported only for testing purpose
+  handleUncaughtExceptionMonitor, // exported only for testing purpose
+  blockOnDatadogRaspAbortError // exported only for testing purpose
 }