From 24c68673832951f39cdbf7bd1a6bf0ff080e4f81 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:17:25 -0800 Subject: [PATCH] Include elastic_integration plugin. (#1579) * Include elastic_integration plugin manually for now. * Update docs/plugins/filters/elastic_integration.asciidoc Define plugin as a non-default plugin. --- .../filters/elastic_integration.asciidoc | 528 ++++++++++++++++++ 1 file changed, 528 insertions(+) create mode 100644 docs/plugins/filters/elastic_integration.asciidoc diff --git a/docs/plugins/filters/elastic_integration.asciidoc b/docs/plugins/filters/elastic_integration.asciidoc new file mode 100644 index 00000000..547d3a9b --- /dev/null +++ b/docs/plugins/filters/elastic_integration.asciidoc @@ -0,0 +1,528 @@ +:plugin: elastic_integration +:type: filter +:default_plugin: 0 + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v0.1.3 +:release_date: 2023-12-20 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v0.1.3/CHANGELOG.md +:include_path: ../../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin + +include::{include_path}/plugin_header.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Using this filter you can process Elastic integrations powered by {es} Ingest Node in Logstash. + +Once configured to point to an {es} cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <> or auto-detecting the event's data-stream and its default pipeline. + +It will then load that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +IMPORTANT: This plugin requires minimum Java 17 and Logstash 8.7.0 versions. + +.Technology Preview +**** +This {elastic-integration-name} filter plugin is part of a _Technology Preview_, which means that both configuration options and implementation details are subject to change in minor releases without being preceded by deprecation warnings. + +Before upgrading this plugin or Logstash itself, please pay special attention to the changelogs to avoid being caught by surprise. +**** + +[id="plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum Configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <>. + +[id="plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <>) +* A list of one or more host URLs (see <>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <>) + +You can also configure which aspects of the proof-of-identity are verified (see <>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <>) +* A JKS- or PKCS12-formatted Keystore (see <>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <>) +* Cloud Auth: (see <>) +* HTTP Basic Auth: (see <> and <>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uriparts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <> + +|======================================================================= + + +[id="plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, the following Elasticsearch IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +:mcc-prefix: [@metadata][_ingest_document] + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `{mcc-prefix}[id]` +| `_index` | `{mcc-prefix}[index]` +| `_routing` | `{mcc-prefix}[routing]` +| `_version` | `{mcc-prefix}[version]` +| `_version_type` | `{mcc-prefix}[version_type]` +| `_ingest.timestamp` | `{mcc-prefix}[timestamp]` +|======================================================================= + + +[id="plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> |<>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> |<>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>|No +| <> | <>, one of `["full", "certificate", "none"]`|No +| <> | <>|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<>=>none`. + +[id="plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is <> +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is <> +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is <> +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is <> +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +Databases are registered by file name, and most integrations rely on databases being present named: + +* `GeoLite2-ASN.mmdb` +* `GeoLite2-City.mmdb` +* `GeoLite2-Country.mmdb` + +This plugin will discover any regular file with the `.mmdb` suffix in the provided directory, and expects the files it finds to be in the MaxMind DB format. + +[id="plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of <>s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with < false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with < true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<>=>false` + +============ + +[id="plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is <> +* There is no default value for this setting. +* Required when request auth is configured with <> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is <> +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is <> +* There is no default value for this setting. +* When present, <> and <> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of <>s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is <> +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is <> +* There is no default value for this setting. +* Required when connection identity is configured with <> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is <> +* There is no default value for this setting. +* Required when connection identity is configured with <> +* {prohibit-ssl-disabled-effective} + +Password for the <>. + +[id="plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is <> +* There is no default value for this setting. +* When present, <> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is <> +* There is no default value for this setting. +* Required when connection identity is configured with <> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <>. + +[id="plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is <> +* There is no default value for this setting. +* When present, <> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is <> +* There is no default value for this setting. +* Required when connection trust is configured with <> +* {prohibit-ssl-disabled-effective} + +Password for the <>. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is <> +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is <> +* There is no default value for this setting. +* When present, <> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[]