From 37cb4c08919548c650f93ce7dc073264bfa4cd5c Mon Sep 17 00:00:00 2001 From: Daniel Adam Date: Wed, 24 Jul 2024 17:49:09 +0200 Subject: [PATCH] Lead Resource Type documentation --- .markdownlint.yaml | 12 ++ .mdl_style.rb | 1 + .../configuration/cloud2cloud-connector.md | 1 + .../docs/configuration/cloud2cloud-gateway.md | 1 + content/en/docs/configuration/coap-gateway.md | 1 + content/en/docs/configuration/grpc-gateway.md | 1 + .../docs/configuration/resource-aggregate.md | 6 +- .../docs/configuration/resource-directory.md | 1 + .../en/docs/configuration/snippet-service.md | 1 + .../docs/features/control-plane/jetstream.md | 56 ------ .../en/docs/features/control-plane/nats.md | 187 ++++++++++++++++++ 11 files changed, 211 insertions(+), 57 deletions(-) create mode 100644 .markdownlint.yaml create mode 100644 content/en/docs/features/control-plane/nats.md diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..fcde7f00 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,12 @@ +# configuration for markdownlint vscode extension +# something is broken about yaml front-matter detection +MD002: false +MD041: false +# enable long lines +MD013: false # line-length +# enable ordered lists indexed by letters (a, b, c) +MD029: false +MD033: false +# in mdl the default is 3 spaces +MD007: # ul-indent + indent: 3 diff --git a/.mdl_style.rb b/.mdl_style.rb index 3bb8f285..0d8d23c1 100644 --- a/.mdl_style.rb +++ b/.mdl_style.rb @@ -1,3 +1,4 @@ +# configuration for mdl linter all # something is broken about yaml front-matter detection exclude_rule 'MD002' diff --git a/content/en/docs/configuration/cloud2cloud-connector.md b/content/en/docs/configuration/cloud2cloud-connector.md index 3f739f0a..d5c673fb 100644 --- a/content/en/docs/configuration/cloud2cloud-connector.md +++ b/content/en/docs/configuration/cloud2cloud-connector.md @@ -96,6 +96,7 @@ plgd hub uses NATS messaging system as an event bus. | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### GRPC Gateway Client diff --git a/content/en/docs/configuration/cloud2cloud-gateway.md b/content/en/docs/configuration/cloud2cloud-gateway.md index 7cf460b5..0a478840 100644 --- a/content/en/docs/configuration/cloud2cloud-gateway.md +++ b/content/en/docs/configuration/cloud2cloud-gateway.md @@ -67,6 +67,7 @@ plgd hub uses NATS messaging system as an event bus. | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### GRPC Gateway Client diff --git a/content/en/docs/configuration/coap-gateway.md b/content/en/docs/configuration/coap-gateway.md index 591e7076..91cf4e46 100644 --- a/content/en/docs/configuration/coap-gateway.md +++ b/content/en/docs/configuration/coap-gateway.md @@ -100,6 +100,7 @@ plgd hub uses NATS messaging system as an event bus. | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### Identity Store Client diff --git a/content/en/docs/configuration/grpc-gateway.md b/content/en/docs/configuration/grpc-gateway.md index 226524de..09bc7e47 100644 --- a/content/en/docs/configuration/grpc-gateway.md +++ b/content/en/docs/configuration/grpc-gateway.md @@ -93,6 +93,7 @@ plgd hub uses NATS messaging system as a event bus. | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### Resource Aggregate Client diff --git a/content/en/docs/configuration/resource-aggregate.md b/content/en/docs/configuration/resource-aggregate.md index ae8b5f2b..3e581abd 100644 --- a/content/en/docs/configuration/resource-aggregate.md +++ b/content/en/docs/configuration/resource-aggregate.md @@ -70,11 +70,15 @@ plgd hub uses NATS messaging system as a event bus. | ---------- | -------- | -------------- | ------- | | `clients.eventBus.nats.url` | string | `URL to nats messaging system.` | `"nats://localhost:4222"` | | `clients.eventBus.nats.flusherTimeout` | string | `Define flush timeout for publishing message.` | `"30s"` | -| `clients.eventBus.nats.jetstream`| bool | `If true, events will be published to jetstream.` | `false` | | `clients.eventBus.nats.tls.caPool` | []string | `File paths to the root certificates in PEM format. The file may contain multiple certificates.` | `[]` | | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.jetstream`| bool | `If true, events will be published to jetstream.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, publish events to NATS with lead resource type subject token included.` | `false` | +| `clients.eventBus.nats.leadResourceType.regexFilter` | []string | `List of regex expressions used to find resouce type that will be used as the lead resource type subject for resource-level events. The expressions are iterated in order and the first matched resource type of the resource associated with the event is used in the NATS subject.` | `[]` | +`clients.eventBus.nats.leadResourceType.filter` | string | `Can contain values "first", "last" or be kept empty (""). If "first" is used, the first resource type of the resource associated with the event will be used as the lead resource type in the NATS subject. If "last" is used, the last resource type is used in the NATS subject. If both "regexFilter" and "filter" properties are set, "regexFilter" takes precedence and "filter" is only examined if no resource type is matched by "regexFilter". If neither are set, the NATS subject will not have a lead resource type, but it will just have the "leadrt" token if the feature is enabled.` | `""` | +| `clients.eventBus.nats.leadResourceType.useUUID` | bool | `If true then do not use the selected resource type as a token directly, but encode it into a UUID string.` | `false` | ### Event Store diff --git a/content/en/docs/configuration/resource-directory.md b/content/en/docs/configuration/resource-directory.md index 2f49f778..2ef51d2b 100644 --- a/content/en/docs/configuration/resource-directory.md +++ b/content/en/docs/configuration/resource-directory.md @@ -77,6 +77,7 @@ plgd hub uses NATS messaging system as a event bus. | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### Event Store diff --git a/content/en/docs/configuration/snippet-service.md b/content/en/docs/configuration/snippet-service.md index 91811b1b..ea7bd1a5 100644 --- a/content/en/docs/configuration/snippet-service.md +++ b/content/en/docs/configuration/snippet-service.md @@ -88,6 +88,7 @@ plgd hub uses NATS messaging system as an event bus to be notified about device | `clients.eventBus.nats.tls.keyFile` | string | `File name of private key in PEM format.` | `""` | | `clients.eventBus.nats.tls.certFile` | string | `File name of certificate in PEM format.` | `""` | | `clients.eventBus.nats.tls.useSystemCAPool` | bool | `If true, use system certification pool.` | `false` | +| `clients.eventBus.nats.leadResourceType.enabled` | bool | `If true, subscribe to NATS with lead resource type subject token included.` | `false` | ### Storage diff --git a/content/en/docs/features/control-plane/jetstream.md b/content/en/docs/features/control-plane/jetstream.md index 36224a20..bc2cc3e8 100644 --- a/content/en/docs/features/control-plane/jetstream.md +++ b/content/en/docs/features/control-plane/jetstream.md @@ -17,62 +17,6 @@ There are still some edge-cases when the plgd event couldn't be published to the More information about the JetStream can be found [here](https://docs.nats.io/jetstream). {{< /note >}} -## NATS subjects overview - -### Definitions - -- `ownerID` is the owner of the device. It is calculated as `uuid.NewV5(uuid.NamespaceURL, value of JWT ownerClaim)`. -- `deviceID` is the UUID of the device. -- `hrefID` is the identifier of resource. It is calculated as `uuid.NewV5(uuid.NamespaceURL, href)`, where the `href` is a resource path. (eg "/oic/d"). - -### Device events - -Each event is compressed by [snappy](https://github.com/google/snappy) and encoded in protobuf [devices event envelope](https://github.com/plgd-dev/hub/blob/main/resource-aggregate/cqrs/eventbus/pb/eventbus.proto). The event envelope consist of `Event.data` containing the event and `Event.event_type` describing the type of the event. - -#### Resource links - -- `plgd.owners.{ownerID}.devices.{deviceID}.resource-links.{eventType}` publishes resource-links events of types `resourcelinkspublished`, `resourcelinksunpublished`, `resourcelinkssnapshottaken` for device `deviceID` and `ownerID` - -#### Metadata events - -- `plgd.owners.{ownerID}.devices.{deviceID}.metadata.{eventType}` publishes metadata events of types `devicemetadataupdatepending`,`devicemetadataupdated`, `devicemetadatasnapshottaken` for device `deviceID` and `ownerID` - -#### Resources events - -- `plgd.owners.{ownerID}.devices.{deviceID}.resources.{hrefID}.{eventType}` publishes resources events of types `resourcechanged`, `resourcecreated`, `resourcecreatepending`, `resourcedeleted` `resourcedeletepending`, `resourceretrieved`, `resourceretrievepending`, `resourcestatesnapshottaken`, `resourceupdated`, `resourceupdatepending` for resource `hrefID`, `deviceID` and `ownerID`. - -### Owner events - -Each event is encoded in protobuf [event envelope](https://github.com/plgd-dev/hub/blob/main/authorization/pb/events.proto) and then compressed by [snappy](https://github.com/google/snappy). - -#### Registration events - -- `plgd.owners.{ownerID}.registrations.{eventType}` publishes owner events of types `devicesregistered`,`devicesunregistered` for `ownerID`. - -### Consumer subscriptions options - -For the consumers of events you can subscribe to: - -- `plgd.owners.>` retrieves all events of the hub. -- `plgd.owners.{ownerId}.>` retrieves all events of the owner with the `ownerId`. -- `plgd.owners.*.devices.{deviceId}.>` retrieves all events of the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.resource-links.>` retrieves all resource link events of the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.resource-links.resourcelinkspublished` retrieves the `resourcelinkspublished` event of the device with the `deviceId`. -- `plgd.owners.*.devices.*.resource-links.>` retrieves all resource link events of all devices. -- `plgd.owners.{ownerId}.devices.*.resource-links.>` retrieves all resource link events of all devices belonging to the owner with the `ownerId`. -- `plgd.owners.*.devices.{deviceId}.metadata.>` retrieves all metadata events of the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.metadata.devicemetadataupdated` retrieves the `devicemetadataupdated` event of the device with the `deviceId`. -- `plgd.owners.*.devices.*.metadata.>` retrieves all metadata events of all devices. -- `plgd.owners.{ownerId}.devices.*.metadata.>` retrieves all metadata events of all devices belonging to the owner with the `ownerId`. -- `plgd.owners.*.devices.{deviceId}.resources.>` retrieves all resource events of the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.resources.{hrefId}.>` retrieves all events of the resource with the `hrefId` for the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.resources.{hrefId}.resourcechanged` retrieves the `resourcechanged` events of the resource with the `hrefId` for the device with the `deviceId`. -- `plgd.owners.*.devices.{deviceId}.resources.*.resourcechanged` retrieves the `resourcechanged` events of all resources for the device with the `deviceId`. -- `plgd.owners.*.devices.*.resources.*.resourcechanged` retrieves the `resourcechanged` events of all resources for all devices. -- `plgd.owners.*.devices.*.resources.{hrefId}.resourcechanged` retrieves the `resourcechanged` events of the resource with the `hrefId` for all devices. -- `plgd.owners.{ownerId}.devices.*.resources.*.resourcechanged` retrieves the `resourcechanged` events of all resources for all devices belonging to the owner with the `ownerId`. -- `plgd.owners.{ownerId}.devices.*.resources.{hrefId}.resourcechanged` retrieves the `resourcechanged` events of the resource with the `hrefId` for all devices belonging to the owner with the `ownerId`. - ## Enable JetStream {{< note >}} diff --git a/content/en/docs/features/control-plane/nats.md b/content/en/docs/features/control-plane/nats.md new file mode 100644 index 00000000..dddae201 --- /dev/null +++ b/content/en/docs/features/control-plane/nats.md @@ -0,0 +1,187 @@ +--- +title: 'NATS' +description: 'How does plgd publish and subscribe to events?' +date: '2024-07-18' +categories: [features] +keywords: ['events', 'nats'] +weight: 75 +--- + +Plgd hub services use NATS as an EventBus and MongoDB as an EventStore. + +## NATS Subjects Overview + +### Definitions + +- `ownerID`: The owner of the device, calculated as `uuid.NewV5(uuid.NamespaceURL, value of JWT ownerClaim)`. +- `deviceID`: The UUID of the device. +- `hrefID`: The identifier of a resource, calculated as `uuid.NewV5(uuid.NamespaceURL, href)`, where `href` is a resource path (e.g. "/oic/d"). +- `leadResourceType`: The selected resource type for resource events. +- `leadResourceTypeID`: The identifier of the selected resource type, calculated as `uuid.NewV5(uuid.NamespaceURL, resourceType)`, where `resourceType` is a resource type (e.g. "oic.wk.d"). + +### Device Events + +Each event is compressed by [snappy](https://github.com/google/snappy) and encoded in protobuf [devices event envelope](https://github.com/plgd-dev/hub/blob/main/resource-aggregate/cqrs/eventbus/pb/eventbus.proto). The event envelope consists of `Event.data` containing the event and `Event.event_type` describing the type of the event. + +#### Resource Links + +- `plgd.owners.{ownerID}.devices.{deviceID}.resource-links.{eventType}` publishes resource-links events of types `resourcelinkspublished`, `resourcelinksunpublished`, and `resourcelinkssnapshottaken` for device `deviceID` and `ownerID`. + +#### Metadata Events + +- `plgd.owners.{ownerID}.devices.{deviceID}.metadata.{eventType}` publishes metadata events of types `devicemetadataupdatepending`, `devicemetadataupdated`, and `devicemetadatasnapshottaken` for device `deviceID` and `ownerID`. + +#### Resource Events + +- `plgd.owners.{ownerID}.devices.{deviceID}.resources.{hrefID}.{eventType}` publishes resource events of types `resourcechanged`, `resourcecreated`, `resourcecreatepending`, `resourcedeleted`, `resourcedeletepending`, `resourceretrieved`, `resourceretrievepending`, `resourcestatesnapshottaken`, `resourceupdated`, and `resourceupdatepending` for resource `hrefID`, `deviceID`, and `ownerID`. + +##### Lead Resource Type + +The Lead Resource Type feature in the plgd hub allows for the specification and handling of a primary resource type for resource events. It is disabled by default but can be enabled through configuration settings for both NATS publishers and subscribers. + +###### Configuration + +To enable the Lead Resource Type feature, modify the YAML configuration files of your service: + +1. **Publisher Configuration:** + + ```yaml + leadResourceType: + enabled: {bool value} + regexFilter: {array of REGEX expressions} + filter: {"first", "last", or ""} + useUUID: {bool value} + ``` + +2. **Subscriber Configuration:** + + ```yaml + leadResourceType: + enabled: {bool value} + ``` + +###### Subjects with Lead Resource Type + +When enabled, the subject is modified based on the selected resource type. If the resource type is not empty, the subject will include the `leadrt` prefix and the resource type as a string. If `useUUID` is set to true, the resource type is encoded to a UUID string. The subject format is modified as follows: + +1. **With `useUUID=false`:** + + ```text + plgd.owners.{ownerID}.devices.{deviceID}.resources.{hrefID}.{eventType}.leadrt.{leadResourceType} + ``` + +2. **With `useUUID=true`:** + + ```text + plgd.owners.{ownerID}.devices.{deviceID}.resources.{hrefID}.{eventType}.leadrt.{leadResourceTypeID} + ``` + +3. **When the resource type is empty:** + + ```text + plgd.owners.{ownerID}.devices.{deviceID}.resources.{hrefID}.{eventType}.leadrt + ``` + +Even when no resource type is selected, the `leadrt` token is added to the subject, allowing the use of the `.>` wildcard to subscribe to resource events with no or any lead resource type. + +###### Selecting the Lead Resource Type + +The lead resource type for a given resource is selected based on the following criteria: + +- **Regex Filter:** + + - `regexFilter` is an array of regex expressions used to match resource types. + - The first matching resource type in the array is used. + +- **Filter:** + + - `filter` is an enumeration of string values ("first", "last"). + - If `regexFilter` does not match any resource type, the `filter` property determines which resource type to use: + - If `filter` is "first", the first resource type is used. + - If `filter` is "last", the last resource type is used. + - Otherwise, no resource type is used. + +The algorithm for selecting the lead resource type is as follows: + +```pseudocode +# Iterate regexFilter first +iterate resource types of given resource + iterate regexFilter expressions + if the expression matches the resource type + use the resource type + +# regexFilter was empty or no resource type was matched +if filter equals "first" + use the first resource type +else if filter equals "last" + use the last resource type +else + don't use resource type +``` + +###### Configure Lead Resource Type at plgd #bundle + +The following environmental variables can be used to configure the Lead Resource Type feature for plgd #bundle: + +- set `LEAD_RESOURCE_TYPE_ENABLED`, `LEAD_RESOURCE_TYPE_USE_UUID` to boolean values +- set `LEAD_RESOURCE_TYPE_FILTER` to "first", "last" or "" +- set `LEAD_RESOURCE_TYPE_REGEX_FILTER` to comma separated list of REGEX expressions + +For example: + +```bash +LEAD_RESOURCE_TYPE_ENABLED=true +LEAD_RESOURCE_TYPE_FILTER="last" +LEAD_RESOURCE_TYPE_REGEX_FILTER="^oic.*,core.*" +LEAD_RESOURCE_TYPE_USE_UUID=true +``` + +```bash +docker run -it --rm -e LEAD_RESOURCE_TYPE_ENABLED=true -e LEAD_RESOURCE_TYPE_FILTER="last" -e LEAD_RESOURCE_TYPE_REGEX_FILTER="^oic.*,core.*" -e LEAD_RESOURCE_TYPE_USE_UUID=true --network=host -v `pwd`/.tmp/data:/data ghcr.io/plgd-dev/hub/bundle:latest +``` + +### Owner events + +Each event is encoded in protobuf [event envelope](https://github.com/plgd-dev/hub/blob/main/authorization/pb/events.proto) and then compressed by [snappy](https://github.com/google/snappy). + +#### Registration events + +- `plgd.owners.{ownerID}.registrations.{eventType}` publishes owner events of types `devicesregistered`,`devicesunregistered` for `ownerID`. + +### Consumer subscriptions options + +For the consumers of events, you can subscribe to: + +- `plgd.owners.>` retrieves all events of the hub. +- `plgd.owners.{ownerId}.>` retrieves all events of the owner with the `ownerId`. +- `plgd.owners.*.devices.{deviceId}.>` retrieves all events of the device with the `deviceId`. +- `plgd.owners.*.devices.{deviceId}.resource-links.>` retrieves all resource link events of the device with the `deviceId`. +- `plgd.owners.*.devices.{deviceId}.resource-links.resourcelinkspublished` retrieves the `resourcelinkspublished` event of the device with the `deviceId`. +- `plgd.owners.*.devices.*.resource-links.>` retrieves all resource link events of all devices. +- `plgd.owners.{ownerId}.devices.*.resource-links.>` retrieves all resource link events of all devices belonging to the owner with the `ownerId`. +- `plgd.owners.*.devices.{deviceId}.metadata.>` retrieves all metadata events of the device with the `deviceId`. +- `plgd.owners.*.devices.{deviceId}.metadata.devicemetadataupdated` retrieves the `devicemetadataupdated` event of the device with the `deviceId`. +- `plgd.owners.*.devices.*.metadata.>` retrieves all metadata events of all devices. +- `plgd.owners.{ownerId}.devices.*.metadata.>` retrieves all metadata events of all devices belonging to the owner with the `ownerId`. +- `plgd.owners.*.devices.{deviceId}.resources.>` retrieves all resource events of the device with the `deviceId`. +- `plgd.owners.*.devices.{deviceId}.resources.{hrefId}.>` retrieves all events of the resource with the `hrefId` for the device with the `deviceId`. + +If the Lead Resource Type feature is disabled: + +- `plgd.owners.*.devices.{deviceId}.resources.{hrefId}.{eventType}` retrieves the `eventType` events of the resource with the `hrefId` for the device with the `deviceId`. +- `plgd.owners.*.devices.{deviceId}.resources.*.{eventType}` retrieves the `eventType` events of all resources for the device with the `deviceId`. +- `plgd.owners.*.devices.*.resources.*.{eventType}` retrieves the `eventType` events of all resources for all devices. +- `plgd.owners.*.devices.*.resources.{hrefId}.{eventType}` retrieves the `eventType` events of the resource with the `hrefId` for all devices. +- `plgd.owners.{ownerId}.devices.*.resources.*.{eventType}` retrieves the `eventType` events of all resources for all devices belonging to the owner with the `ownerId`. +- `plgd.owners.{ownerId}.devices.*.resources.{hrefId}.{eventType}` retrieves the `eventType` events of the resource with the `hrefId` for all devices belonging to the owner with the `ownerId`. + +With the Lead Resource Type feature enabled, the equivalent subjects are modified to: + +- `plgd.owners.*.devices.{deviceId}.resources.{hrefId}.{eventType}.leadrt.{leadResourceType}` +- `plgd.owners.*.devices.{deviceId}.resources.*.{eventType}.>` +- `plgd.owners.*.devices.*.resources.*.{eventType}.>` +- `plgd.owners.*.devices.*.resources.{hrefId}.{eventType}.>` +- `plgd.owners.{ownerId}.devices.*.resources.*.{eventType}.>` +- `plgd.owners.{ownerId}.devices.*.resources.{hrefId}.{eventType}.>` + +These subscription options provide flexibility for consumers to filter events based on specific criteria, such as device ID, owner ID, and resource ID, and they also support wildcard subscriptions for broader event capture.