From 5bcf075106ff0749236d7f557d7390fa65cfc2c4 Mon Sep 17 00:00:00 2001 From: Jason Collins <47123298+jcollins-axway@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:32:56 -0700 Subject: [PATCH] APIGOV-28931 - custom unit ingestion and refactoring of metric storage/sending (#843) * udpates for custom unit metric ingestion * get event id from v4Event during acking * set status in metric * use pointers in metric event sent to central * code cleanup * add additional methods to get resource reference information * fix get key method * update to use counter for custom metrics * cleanup * add test for sending custom usage metrics * add name for generator docker container * resource generate * update models * APIGOV-28931 - fix generator * get product plan unit ref * use the plan unit ref while handing metrics --- Makefile | 2 +- .../apiserver/models/api/v1/resourcemeta.go | 20 + .../catalog/v1alpha1/SubscriptionInvoice.go | 17 + ..._mapping_status_resource_asset_resource.go | 3 +- ...del_asset_mapping_status_resource_stage.go | 3 +- ...ping_status_source_resource_api_service.go | 3 +- ...us_source_resource_api_service_instance.go | 3 +- ...us_source_resource_api_service_revision.go | 3 +- ..._source_resource_asset_mapping_template.go | 3 +- .../v1alpha1/model_product_plan_unit_spec.go | 2 +- .../model_release_tag_status_success_meta.go | 1 - .../model_subscription_invoice_references.go | 16 + .../model_access_request_references_quota.go | 2 + ...l_access_request_spec_additional_quotas.go | 20 + .../model_access_request_spec_quota.go | 2 +- .../v1alpha1/model_api_service_compliance.go | 7 +- .../model_api_service_instance_compliance.go | 1 - ..._api_service_instance_source_compliance.go | 1 - .../model_api_service_revision_compliance.go | 4 +- ..._mapping_status_resource_asset_resource.go | 3 +- ...del_asset_mapping_status_resource_stage.go | 3 +- ...ping_status_source_resource_api_service.go | 3 +- ...us_source_resource_api_service_instance.go | 3 +- ...us_source_resource_api_service_revision.go | 3 +- ..._source_resource_asset_mapping_template.go | 3 +- .../model_environment_compliancetasks.go | 4 +- .../model_identity_provider_secret_spec.go | 2 - .../v1alpha1/model_identity_provider_spec.go | 3 - pkg/transaction/metric/apimetric.go | 63 +++ pkg/transaction/metric/cachestorage.go | 92 ++-- pkg/transaction/metric/centralmetric.go | 119 +++++ pkg/transaction/metric/definition.go | 83 +--- pkg/transaction/metric/metricbatch.go | 30 +- pkg/transaction/metric/metricscollector.go | 423 ++++++++++-------- .../metric/metricscollector_test.go | 161 ++++++- pkg/transaction/metric/mockclient_test.go | 21 +- pkg/transaction/metric/reportcache.go | 6 +- pkg/transaction/metric/util.go | 81 ++++ pkg/transaction/models/definitions.go | 13 + scripts/apiserver/generate.js | 3 +- 40 files changed, 858 insertions(+), 377 deletions(-) create mode 100644 pkg/apic/apiserver/models/catalog/v1alpha1/model_subscription_invoice_references.go create mode 100644 pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_additional_quotas.go create mode 100644 pkg/transaction/metric/apimetric.go create mode 100644 pkg/transaction/metric/centralmetric.go create mode 100644 pkg/transaction/metric/util.go diff --git a/Makefile b/Makefile index 285b884de..4705a6350 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ test-sonar: dep @go test -short -coverpkg=./... -coverprofile=${WORKSPACE}/gocoverage.out -count=1 ${GO_PKG_LIST} -json > ${WORKSPACE}/goreport.json apiserver-generate: # generate api server resources, prod by default. ex: make apiserver-generate protocol=https host=apicentral.axway.com port=443 - docker run --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)' -e PORT='$(port)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest + docker run --name generator --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)' -e PORT='$(port)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest PROTOFILES := $(shell find $(WORKSPACE)/proto -type f -name '*.proto') PROTOTARGETS := $(PROTOFILES:.proto=.pb.go) diff --git a/pkg/apic/apiserver/models/api/v1/resourcemeta.go b/pkg/apic/apiserver/models/api/v1/resourcemeta.go index 196d7c029..220fa77a8 100644 --- a/pkg/apic/apiserver/models/api/v1/resourcemeta.go +++ b/pkg/apic/apiserver/models/api/v1/resourcemeta.go @@ -196,6 +196,26 @@ func (rm *ResourceMeta) GetReferenceByGVK(gvk GroupVersionKind) Reference { return Reference{} } +// GetReferenceByIDAndGVK returns the first found reference that matches the ID and GroupKind arguments. +func (rm *ResourceMeta) GetReferenceByIDAndGVK(id string, gvk GroupVersionKind) Reference { + for _, ref := range rm.Metadata.References { + if ref.Group == gvk.Group && ref.Kind == gvk.Kind && ref.ID == id { + return ref + } + } + return Reference{} +} + +// GetReferenceByNameAndGVK returns the first found reference that matches the Name and GroupKind arguments. +func (rm *ResourceMeta) GetReferenceByNameAndGVK(name string, gvk GroupVersionKind) Reference { + for _, ref := range rm.Metadata.References { + if ref.Group == gvk.Group && ref.Kind == gvk.Kind && ref.Name == name { + return ref + } + } + return Reference{} +} + // MarshalJSON marshals the ResourceMeta to properly set the SubResources func (rm *ResourceMeta) MarshalJSON() ([]byte, error) { rawSubs := map[string]interface{}{} diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/SubscriptionInvoice.go b/pkg/apic/apiserver/models/catalog/v1alpha1/SubscriptionInvoice.go index 6daac76f9..52b8babce 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/SubscriptionInvoice.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/SubscriptionInvoice.go @@ -30,6 +30,7 @@ const ( SubscriptionInvoiceResourceName = "subscriptioninvoices" SubscriptionInvoiceBillingSubResourceName = "billing" SubscriptionInvoiceMarketplaceSubResourceName = "marketplace" + SubscriptionInvoiceReferencesSubResourceName = "references" SubscriptionInvoiceStateSubResourceName = "state" SubscriptionInvoiceStatusSubResourceName = "status" ) @@ -49,6 +50,7 @@ type SubscriptionInvoice struct { Billing SubscriptionInvoiceBilling `json:"billing"` Marketplace SubscriptionInvoiceMarketplace `json:"marketplace"` Owner *apiv1.Owner `json:"owner"` + References SubscriptionInvoiceReferences `json:"references"` Spec SubscriptionInvoiceSpec `json:"spec"` State SubscriptionInvoiceState `json:"state"` // Status SubscriptionInvoiceStatus `json:"status"` @@ -140,6 +142,7 @@ func (res *SubscriptionInvoice) MarshalJSON() ([]byte, error) { out["billing"] = res.Billing out["marketplace"] = res.Marketplace out["owner"] = res.Owner + out["references"] = res.References out["spec"] = res.Spec out["state"] = res.State out["status"] = res.Status @@ -200,6 +203,20 @@ func (res *SubscriptionInvoice) UnmarshalJSON(data []byte) error { } } + // marshalling subresource References + if v, ok := aux.SubResources["references"]; ok { + sr, err = json.Marshal(v) + if err != nil { + return err + } + + delete(aux.SubResources, "references") + err = json.Unmarshal(sr, &res.References) + if err != nil { + return err + } + } + // marshalling subresource State if v, ok := aux.SubResources["state"]; ok { sr, err = json.Marshal(v) diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_asset_resource.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_asset_resource.go index b9499b7c9..1ad10e3c2 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_asset_resource.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_asset_resource.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusResourceAssetResource (catalog.v1alpha1.AssetMapping) type AssetMappingStatusResourceAssetResource struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_stage.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_stage.go index f65e70600..16b5786a2 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_stage.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_stage.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusResourceStage (catalog.v1alpha1.AssetMapping) type AssetMappingStatusResourceStage struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service.go index 17cacf60a..bbae0e6a0 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusSourceResourceApiService (catalog.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiService struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go index 4c47fb7df..fd8167af7 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusSourceResourceApiServiceInstance (catalog.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiServiceInstance struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go index 0946c0f4c..07de3127f 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusSourceResourceApiServiceRevision (catalog.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiServiceRevision struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go index a6564cd67..90631284a 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go @@ -11,7 +11,6 @@ package catalog // AssetMappingStatusSourceResourceAssetMappingTemplate (catalog.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceAssetMappingTemplate struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_product_plan_unit_spec.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_product_plan_unit_spec.go index 378d68359..339c7d822 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_product_plan_unit_spec.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_product_plan_unit_spec.go @@ -11,6 +11,6 @@ package catalog // ProductPlanUnitSpec (catalog.v1alpha1.ProductPlanUnit) type ProductPlanUnitSpec struct { - // description of the Plan. + // description of the Product Plan Unit. Description string `json:"description,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_release_tag_status_success_meta.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_release_tag_status_success_meta.go index 224f09a15..f0958d37a 100644 --- a/pkg/apic/apiserver/models/catalog/v1alpha1/model_release_tag_status_success_meta.go +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_release_tag_status_success_meta.go @@ -11,6 +11,5 @@ package catalog // ReleaseTagStatusSuccessMeta struct for ReleaseTagStatusSuccessMeta type ReleaseTagStatusSuccessMeta struct { - // GENERATE: The following code has been modified after code generation Reference []ReleaseTagStatusReference `json:"reference,omitempty"` } diff --git a/pkg/apic/apiserver/models/catalog/v1alpha1/model_subscription_invoice_references.go b/pkg/apic/apiserver/models/catalog/v1alpha1/model_subscription_invoice_references.go new file mode 100644 index 000000000..c3d193f57 --- /dev/null +++ b/pkg/apic/apiserver/models/catalog/v1alpha1/model_subscription_invoice_references.go @@ -0,0 +1,16 @@ +/* + * API Server specification. + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * API version: SNAPSHOT + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package catalog + +// SubscriptionInvoiceReferences (catalog.v1alpha1.SubscriptionInvoice) +type SubscriptionInvoiceReferences struct { + // Reference to product. + Product string `json:"product,omitempty"` +} diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_references_quota.go b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_references_quota.go index 8b25bbc98..ef44160a9 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_references_quota.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_references_quota.go @@ -13,4 +13,6 @@ package management type AccessRequestReferencesQuota struct { Kind string `json:"kind"` Name string `json:"name,omitempty"` + // The name of the unit for the included quota. + Unit string `json:"unit,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_additional_quotas.go b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_additional_quotas.go new file mode 100644 index 000000000..e700928ab --- /dev/null +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_additional_quotas.go @@ -0,0 +1,20 @@ +/* + * API Server specification. + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * API version: SNAPSHOT + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package management + +// AccessRequestSpecAdditionalQuotas Quota information for accessing the api. (management.v1alpha1.AccessRequest) +type AccessRequestSpecAdditionalQuotas struct { + // The logical name of the quota + Name string `json:"name"` + // The limit of the allowed quota for the access request. + Limit int32 `json:"limit,omitempty"` + // The interval for the applied quota. + Interval string `json:"interval,omitempty"` +} diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_quota.go b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_quota.go index 88b4e9105..f8dc4e8e8 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_quota.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_quota.go @@ -9,7 +9,7 @@ package management -// AccessRequestSpecQuota Quota information for accessing the api. (management.v1alpha1.AccessRequest) +// AccessRequestSpecQuota Quota information for api transactions in accessing the api. (management.v1alpha1.AccessRequest) type AccessRequestSpecQuota struct { // The limit of the allowed quota for the access request. Limit int32 `json:"limit"` diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_compliance.go b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_compliance.go index d2b293fd7..bd50bc621 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_compliance.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_compliance.go @@ -11,10 +11,7 @@ package management // ApiServiceCompliance (management.v1alpha1.APIService) type ApiServiceCompliance struct { - // GENERATE: The following code has been modified after code generation - Design ApiServiceComplianceLintingStatus `json:"design,omitempty"` - // GENERATE: The following code has been modified after code generation + Design ApiServiceComplianceLintingStatus `json:"design,omitempty"` Security ApiServiceComplianceLintingStatus `json:"security,omitempty"` - // GENERATE: The following code has been modified after code generation - Runtime ApiServiceComplianceRuntimeStatus `json:"runtime,omitempty"` + Runtime ApiServiceComplianceRuntimeStatus `json:"runtime,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_compliance.go b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_compliance.go index 7c9056fc5..6139a88e4 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_compliance.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_compliance.go @@ -11,6 +11,5 @@ package management // ApiServiceInstanceCompliance (management.v1alpha1.APIServiceInstance) type ApiServiceInstanceCompliance struct { - // GENERATE: The following code has been modified after code generation Runtime ApiServiceInstanceComplianceRuntimeStatus `json:"runtime,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_source_compliance.go b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_source_compliance.go index 9c74f1322..82d52363f 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_source_compliance.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_source_compliance.go @@ -11,6 +11,5 @@ package management // ApiServiceInstanceSourceCompliance (management.v1alpha1.APIServiceInstance) type ApiServiceInstanceSourceCompliance struct { - // GENERATE: The following code has been modified after code generation Runtime ApiServiceInstanceSourceRuntimeStatus `json:"runtime,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_revision_compliance.go b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_revision_compliance.go index f493e456b..f1b779755 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_revision_compliance.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_api_service_revision_compliance.go @@ -11,8 +11,6 @@ package management // ApiServiceRevisionCompliance (management.v1alpha1.APIServiceRevision) type ApiServiceRevisionCompliance struct { - // GENERATE: The following code has been modified after code generation - Design ApiServiceRevisionComplianceLintingStatus `json:"design,omitempty"` - // GENERATE: The following code has been modified after code generation + Design ApiServiceRevisionComplianceLintingStatus `json:"design,omitempty"` Security ApiServiceRevisionComplianceLintingStatus `json:"security,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_asset_resource.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_asset_resource.go index 8fb411957..a3ca6df43 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_asset_resource.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_asset_resource.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusResourceAssetResource (management.v1alpha1.AssetMapping) type AssetMappingStatusResourceAssetResource struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_stage.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_stage.go index 028c933b9..99c1d8a53 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_stage.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_stage.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusResourceStage (management.v1alpha1.AssetMapping) type AssetMappingStatusResourceStage struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service.go index f8d5ce97f..94553198a 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusSourceResourceApiService (management.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiService struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go index 3ed5207f5..9b061cd44 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusSourceResourceApiServiceInstance (management.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiServiceInstance struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go index 51ba28567..5c70ccfd8 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusSourceResourceApiServiceRevision (management.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceApiServiceRevision struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go index ebfc24ceb..62b3a5956 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go @@ -11,7 +11,6 @@ package management // AssetMappingStatusSourceResourceAssetMappingTemplate (management.v1alpha1.AssetMapping) type AssetMappingStatusSourceResourceAssetMappingTemplate struct { - Ref string `json:"ref,omitempty"` - // GENERATE: The following code has been modified after code generation + Ref string `json:"ref,omitempty"` OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_environment_compliancetasks.go b/pkg/apic/apiserver/models/management/v1alpha1/model_environment_compliancetasks.go index 7890f1c5f..8fcf177f5 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_environment_compliancetasks.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_environment_compliancetasks.go @@ -11,8 +11,6 @@ package management // EnvironmentCompliancetasks Subresource that is only accessible by the backend. Used by compliance-controller to auto-start/cancel linting jobs based on changes made to \"spec.compliance\" properties detected via mutation hook. (management.v1alpha1.Environment) type EnvironmentCompliancetasks struct { - // GENERATE: The following code has been modified after code generation - Design EnvironmentCompliancetasksLinting `json:"design,omitempty"` - // GENERATE: The following code has been modified after code generation + Design EnvironmentCompliancetasksLinting `json:"design,omitempty"` Security EnvironmentCompliancetasksLinting `json:"security,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_secret_spec.go b/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_secret_spec.go index ded621979..4472f1bdb 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_secret_spec.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_secret_spec.go @@ -16,9 +16,7 @@ type IdentityProviderSecretSpec struct { // GENERATE: The following code has been modified after code generation Config interface{} `json:"config"` // The list of request headers to be sent in the call to get token from Identity provider - // GENERATE: The following code has been modified after code generation RequestHeaders []IdentityProviderSecretSpecKeyValuePair `json:"requestHeaders,omitempty"` // The list of query parameters to be sent in the call to get token from Identity provider - // GENERATE: The following code has been modified after code generation QueryParameters []IdentityProviderSecretSpecKeyValuePair `json:"queryParameters,omitempty"` } diff --git a/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_spec.go b/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_spec.go index 4f85536a3..015ccda9c 100644 --- a/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_spec.go +++ b/pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_spec.go @@ -18,13 +18,10 @@ type IdentityProviderSpec struct { // The flag to configure agent to use registration access token for removing/updating the client created by agent in Identity provider UseRegistrationAccessToken bool `json:"useRegistrationAccessToken,omitempty"` // The list of request headers to be sent in the call to Identity provider for creating/updating and removing clients - // GENERATE: The following code has been modified after code generation RequestHeaders []IdentityProviderSpecKeyValuePair `json:"requestHeaders,omitempty"` // The list of query parameters to be sent in the call to Identity provider for creating/updating and removing clients - // GENERATE: The following code has been modified after code generation QueryParameters []IdentityProviderSpecKeyValuePair `json:"queryParameters,omitempty"` // The list of additional metadata properties that will be set when registering the client in Identity provider - // GENERATE: The following code has been modified after code generation AdditionalClientProperties []IdentityProviderSpecKeyValuePair `json:"additionalClientProperties,omitempty"` // Defines the timeout interval in seconds for Identity provider http client. Defaults to 30 seconds ClientTimeout int32 `json:"clientTimeout,omitempty"` diff --git a/pkg/transaction/metric/apimetric.go b/pkg/transaction/metric/apimetric.go new file mode 100644 index 000000000..2cfca0968 --- /dev/null +++ b/pkg/transaction/metric/apimetric.go @@ -0,0 +1,63 @@ +package metric + +import ( + "time" + + "github.com/Axway/agent-sdk/pkg/transaction/models" + "github.com/sirupsen/logrus" +) + +// APIMetric - struct to hold metric aggregated for subscription,application,api,statuscode +type APIMetric struct { + Subscription models.Subscription `json:"subscription,omitempty"` + App models.AppDetails `json:"application,omitempty"` + Product models.Product `json:"product,omitempty"` + API models.APIDetails `json:"api"` + AssetResource models.AssetResource `json:"assetResource,omitempty"` + ProductPlan models.ProductPlan `json:"productPlan,omitempty"` + Quota models.Quota `json:"quota,omitempty"` + Unit models.Unit `json:"unit,omitempty"` + StatusCode string `json:"statusCode,omitempty"` + Status string `json:"status,omitempty"` + Count int64 `json:"count"` + Response ResponseMetrics `json:"response,omitempty"` + Observation ObservationDetails `json:"observation"` + EventID string `json:"-"` + StartTime time.Time `json:"-"` +} + +// GetStartTime - Returns the start time for subscription metric +func (a *APIMetric) GetStartTime() time.Time { + return a.StartTime +} + +// GetType - Returns APIMetric +func (a *APIMetric) GetType() string { + return "APIMetric" +} + +// GetType - Returns APIMetric +func (a *APIMetric) GetEventID() string { + return a.EventID +} + +func (a *APIMetric) GetLogFields() logrus.Fields { + fields := logrus.Fields{ + "id": a.EventID, + "count": a.Count, + "status": a.StatusCode, + "minResponse": a.Response.Min, + "maxResponse": a.Response.Max, + "avgResponse": a.Response.Avg, + "startTimestamp": a.Observation.Start, + "endTimestamp": a.Observation.End, + } + fields = a.Subscription.GetLogFields(fields) + fields = a.App.GetLogFields(fields) + fields = a.Product.GetLogFields(fields) + fields = a.API.GetLogFields(fields) + fields = a.AssetResource.GetLogFields(fields) + fields = a.ProductPlan.GetLogFields(fields) + fields = a.Quota.GetLogFields(fields) + return fields +} diff --git a/pkg/transaction/metric/cachestorage.go b/pkg/transaction/metric/cachestorage.go index 1779c23ad..20019e755 100644 --- a/pkg/transaction/metric/cachestorage.go +++ b/pkg/transaction/metric/cachestorage.go @@ -2,6 +2,7 @@ package metric import ( "encoding/json" + "fmt" "os" "os/signal" "strings" @@ -13,13 +14,13 @@ import ( "github.com/Axway/agent-sdk/pkg/cache" "github.com/Axway/agent-sdk/pkg/traceability" "github.com/Axway/agent-sdk/pkg/util" - metrics "github.com/rcrowley/go-metrics" + "github.com/rcrowley/go-metrics" ) const ( appUsagePrefix = "app_usage." cacheFileName = "agent-usagemetric.json" - metricKeyPrefix = "metric." + metricKeyPrefix = "metric" metricStartTimeKey = "metric_start_time" usageStartTimeKey = "usage_start_time" usageCountKey = "usage_count" @@ -31,8 +32,8 @@ type storageCache interface { updateUsage(usageCount int) updateVolume(bytes int64) updateAppUsage(usageCount int, appID string) - updateMetric(apiStatusMetric metrics.Histogram, metric *APIMetric) - removeMetric(metric *APIMetric) + updateMetric(cachedMetric cachedMetricInterface, metric *centralMetricEvent) + removeMetric(metric *centralMetricEvent) save() } @@ -148,7 +149,7 @@ func (c *cacheStorage) updateAppUsage(usageCount int, appID string) { func (c *cacheStorage) loadMetrics(storageCache cache.Cache) { cacheKeys := storageCache.GetKeys() for _, cacheKey := range cacheKeys { - if strings.Contains(cacheKey, metricKeyPrefix) { + if strings.HasPrefix(cacheKey, fmt.Sprintf("%s.", metricKeyPrefix)) { if agent.GetCentralConfig().GetUsageReportingConfig().IsOfflineMode() { // delete metrics from cache in offline mode storageCache.Delete(cacheKey) @@ -160,18 +161,23 @@ func (c *cacheStorage) loadMetrics(storageCache cache.Cache) { var cm cachedMetric json.Unmarshal(buffer, &cm) - var metric *APIMetric + var metric *centralMetricEvent for _, duration := range cm.Values { + unitID := "" + if cm.Unit != nil { + unitID = cm.Unit.ID + } metricDetail := Detail{ - APIDetails: cm.API, - AppDetails: cm.App, + APIDetails: *cm.API, + AppDetails: *cm.App, + UnitName: unitID, StatusCode: cm.StatusCode, Duration: duration, } metric = c.collector.createOrUpdateMetric(metricDetail) } - newKey := c.getKey(metric) + newKey := metric.getKey() if newKey != cacheKey { c.storageLock.Lock() storageCache.Delete(cacheKey) @@ -185,7 +191,7 @@ func (c *cacheStorage) loadMetrics(storageCache cache.Cache) { } } -func (c *cacheStorage) updateMetric(histogram metrics.Histogram, metric *APIMetric) { +func (c *cacheStorage) updateMetric(cached cachedMetricInterface, metric *centralMetricEvent) { if !c.isInitialized { return } @@ -193,39 +199,18 @@ func (c *cacheStorage) updateMetric(histogram metrics.Histogram, metric *APIMetr c.storageLock.Lock() defer c.storageLock.Unlock() - cachedMetric := cachedMetric{ - Subscription: metric.Subscription, - App: metric.App, - Product: metric.Product, - AssetResource: metric.AssetResource, - ProductPlan: metric.ProductPlan, - Quota: metric.Quota, - API: metric.API, - StatusCode: metric.StatusCode, - Count: histogram.Count(), - Values: histogram.Sample().Values(), - StartTime: metric.StartTime, - } - - c.storage.Set(c.getKey(metric), cachedMetric) + c.storage.Set(metric.getKey(), metric.createdCachedMetric(cached)) } -func (c *cacheStorage) removeMetric(metric *APIMetric) { +func (c *cacheStorage) removeMetric(metric *centralMetricEvent) { if !c.isInitialized { return } + c.storageLock.Lock() defer c.storageLock.Unlock() - c.storage.Delete(c.getKey(metric)) -} - -func (c *cacheStorage) getKey(metric *APIMetric) string { - return metricKeyPrefix + - metric.Subscription.ID + "." + - metric.App.ID + "." + - metric.API.ID + "." + - metric.StatusCode + c.storage.Delete(metric.getKey()) } func (c *cacheStorage) save() { @@ -271,3 +256,40 @@ func parseTimeFromCache(storage cache.Cache, key string) (time.Time, error) { } return resultTime, nil } + +type cachedMetricInterface interface { + Count() int64 + Values() []int64 +} + +type customCounter struct { + c metrics.Counter +} + +func newCustomCounter(c metrics.Counter) *customCounter { + return &customCounter{c: c} +} + +func (c customCounter) Count() int64 { + return c.c.Count() +} + +func (c customCounter) Values() []int64 { + return nil +} + +type customHistogram struct { + h metrics.Histogram +} + +func newCustomHistogram(h metrics.Histogram) *customHistogram { + return &customHistogram{h: h} +} + +func (c customHistogram) Count() int64 { + return c.h.Count() +} + +func (c customHistogram) Values() []int64 { + return c.h.Sample().Values() +} diff --git a/pkg/transaction/metric/centralmetric.go b/pkg/transaction/metric/centralmetric.go new file mode 100644 index 000000000..0a9394133 --- /dev/null +++ b/pkg/transaction/metric/centralmetric.go @@ -0,0 +1,119 @@ +package metric + +import ( + "strings" + "time" + + "github.com/Axway/agent-sdk/pkg/transaction/models" + "github.com/sirupsen/logrus" +) + +// metricInfo - the base object holding the metricInfo +type metricInfo struct { + Subscription *models.Subscription `json:"subscription,omitempty"` + App *models.AppDetails `json:"app,omitempty"` + Product *models.Product `json:"product,omitempty"` + API *models.APIDetails `json:"api,omitempty"` + AssetResource *models.AssetResource `json:"assetResource,omitempty"` + ProductPlan *models.ProductPlan `json:"productPlan,omitempty"` + Quota *models.Quota `json:"quota,omitempty"` + Unit *models.Unit `json:"unit,omitempty"` + StatusCode string `json:"statusCode,omitempty"` +} + +// centralMetricEvent - the event that is actually sent to platform +type centralMetricEvent struct { + metricInfo + Status string `json:"status,omitempty"` + Count int64 `json:"count"` + Response *ResponseMetrics `json:"response,omitempty"` + Observation *ObservationDetails `json:"observation"` + EventID string `json:"-"` + StartTime time.Time `json:"-"` +} + +// GetStartTime - Returns the start time for subscription metric +func (a *centralMetricEvent) GetStartTime() time.Time { + return a.StartTime +} + +// GetType - Returns APIMetric +func (a *centralMetricEvent) GetType() string { + return "APIMetric" +} + +// GetType - Returns APIMetric +func (a *centralMetricEvent) GetEventID() string { + return a.EventID +} + +func (a *centralMetricEvent) GetLogFields() logrus.Fields { + fields := logrus.Fields{ + "id": a.EventID, + "count": a.Count, + "status": a.StatusCode, + "minResponse": a.Response.Min, + "maxResponse": a.Response.Max, + "avgResponse": a.Response.Avg, + "startTimestamp": a.Observation.Start, + "endTimestamp": a.Observation.End, + } + if a.Subscription != nil { + fields = a.Subscription.GetLogFields(fields) + } + if a.App != nil { + fields = a.App.GetLogFields(fields) + } + if a.Product != nil { + fields = a.Product.GetLogFields(fields) + } + if a.API != nil { + fields = a.API.GetLogFields(fields) + } + if a.AssetResource != nil { + fields = a.AssetResource.GetLogFields(fields) + } + if a.ProductPlan != nil { + fields = a.ProductPlan.GetLogFields(fields) + } + if a.Quota != nil { + fields = a.Quota.GetLogFields(fields) + } + if a.Unit != nil { + fields = a.Unit.GetLogFields(fields) + } + return fields +} + +// getKey - returns the cache key for the metric +func (a *centralMetricEvent) getKey() string { + subID := unknown + if a.Subscription != nil { + subID = a.Subscription.ID + } + appID := unknown + if a.App != nil { + appID = a.App.ID + } + apiID := unknown + if a.API != nil { + apiID = a.API.ID + } + uniqueKey := unknown + if a.StatusCode != "" { + uniqueKey = a.StatusCode + } else if a.Unit != nil { + uniqueKey = a.Unit.ID + } + + return strings.Join([]string{metricKeyPrefix, subID, appID, apiID, uniqueKey}, ".") +} + +func (a *centralMetricEvent) createdCachedMetric(cached cachedMetricInterface) cachedMetric { + return cachedMetric{ + metricInfo: a.metricInfo, + StartTime: a.StartTime, + Count: cached.Count(), + Values: cached.Values(), + } +} diff --git a/pkg/transaction/metric/definition.go b/pkg/transaction/metric/definition.go index ebfeb39f0..c6ad4bcca 100644 --- a/pkg/transaction/metric/definition.go +++ b/pkg/transaction/metric/definition.go @@ -25,10 +25,11 @@ const ( unknown = "unknown" ) -type TransactionContext struct { +type transactionContext struct { APIDetails models.APIDetails AppDetails models.AppDetails - StatusCode string + Status string + UnitName string } // Detail - holds the details for computing metrics @@ -37,6 +38,7 @@ type Detail struct { APIDetails models.APIDetails AppDetails models.AppDetails StatusCode string + UnitName string Duration int64 Bytes int64 } @@ -50,6 +52,14 @@ type MetricDetail struct { Observation ObservationDetails } +type CustomMetricDetail struct { + APIDetails models.APIDetails + AppDetails models.AppDetails + UnitDetails models.Unit + Count int64 + Observation ObservationDetails +} + // ResponseMetrics - Holds metrics API response type ResponseMetrics struct { Max int64 `json:"max"` @@ -63,73 +73,12 @@ type ObservationDetails struct { End int64 `json:"end,omitempty"` } -// APIMetric - struct to hold metric aggregated for subscription,application,api,statuscode -type APIMetric struct { - Subscription models.Subscription `json:"subscription"` - App models.AppDetails `json:"application"` - Product models.Product `json:"product,omitempty"` - API models.APIDetails `json:"api"` - AssetResource models.AssetResource `json:"assetResource,omitempty"` - ProductPlan models.ProductPlan `json:"productPlan,omitempty"` - Quota models.Quota `json:"quota,omitempty"` - StatusCode string `json:"statusCode"` - Status string `json:"status"` - Count int64 `json:"count"` - Response ResponseMetrics `json:"response"` - Observation ObservationDetails `json:"observation"` - EventID string `json:"eventID"` - StartTime time.Time `json:"-"` -} - -// GetStartTime - Returns the start time for subscription metric -func (a *APIMetric) GetStartTime() time.Time { - return a.StartTime -} - -// GetType - Returns APIMetric -func (a *APIMetric) GetType() string { - return "APIMetric" -} - -// GetType - Returns APIMetric -func (a *APIMetric) GetEventID() string { - return a.EventID -} - -func (a *APIMetric) GetLogFields() logrus.Fields { - fields := logrus.Fields{ - "id": a.EventID, - "count": a.Count, - "status": a.StatusCode, - "minResponse": a.Response.Min, - "maxResponse": a.Response.Max, - "avgResponse": a.Response.Avg, - "startTimestamp": a.Observation.Start, - "endTimestamp": a.Observation.End, - } - fields = a.Subscription.GetLogFields(fields) - fields = a.App.GetLogFields(fields) - fields = a.Product.GetLogFields(fields) - fields = a.API.GetLogFields(fields) - fields = a.AssetResource.GetLogFields(fields) - fields = a.ProductPlan.GetLogFields(fields) - fields = a.Quota.GetLogFields(fields) - return fields -} - // cachedMetric - struct to hold metric specific that gets cached and used for agent recovery type cachedMetric struct { - Subscription models.Subscription `json:"subscription,omitempty"` - App models.AppDetails `json:"app,omitempty"` - Product models.Product `json:"product,omitempty"` - API models.APIDetails `json:"api"` - AssetResource models.AssetResource `json:"assetResource,omitempty"` - ProductPlan models.ProductPlan `json:"productPlan,omitempty"` - Quota models.Quota `json:"quota,omitempty"` - StatusCode string `json:"statusCode"` - Count int64 `json:"count"` - Values []int64 `json:"values"` - StartTime time.Time `json:"startTime"` + metricInfo + Count int64 `json:"count"` + Values []int64 `json:"values,omitempty"` + StartTime time.Time `json:"startTime"` } // V4EventDistribution - represents V4 distribution diff --git a/pkg/transaction/metric/metricbatch.go b/pkg/transaction/metric/metricbatch.go index e8fc76689..c362a21f9 100644 --- a/pkg/transaction/metric/metricbatch.go +++ b/pkg/transaction/metric/metricbatch.go @@ -133,10 +133,16 @@ func (b *EventBatch) logEvents(status string, events []beatPub.Event) { func (b *EventBatch) ackEvents(events []beatPub.Event) { for _, event := range events { metric := getMetricFromEvent(event) - if metric != nil { - b.collector.logMetric("published", metric) - b.collector.cleanupMetricCounter(b.histograms[metric.EventID], metric) + if metric == nil { + continue + } + b.collector.logMetric("published", metric) + histogram, found := b.histograms[metric.EventID] + if !found { + b.collector.metricLogger.WithField("eventID", metric.EventID).Warn("could not clean cached metric") + continue } + b.collector.cleanupMetricCounter(histogram, metric) } } @@ -152,10 +158,19 @@ func NewEventBatch(c *collector) *EventBatch { func getEventsToAck(retryEvents []beatPub.Event, events []beatPub.Event) []beatPub.Event { ackEvents := make([]beatPub.Event, 0) for _, e := range events { - eID := getMetricFromEvent(e).EventID + eID := "" + if m := getMetricFromEvent(e); m != nil { + eID = m.EventID + } + if eID == "" { + continue + } found := false for _, rE := range retryEvents { - rEID := getMetricFromEvent(rE).EventID + rEID := "" + if m := getMetricFromEvent(rE); m != nil { + rEID = m.EventID + } if rEID == eID { found = true break @@ -176,6 +191,10 @@ func getMetricFromEvent(event beatPub.Event) *APIMetric { if err != nil { return nil } + eventID, ok := v4Event["id"] + if !ok { + return nil + } eventType, ok := v4Event["event"] if !ok { return nil @@ -192,6 +211,7 @@ func getMetricFromEvent(event beatPub.Event) *APIMetric { if err != nil { return nil } + metric.EventID = eventID.(string) return metric } return nil diff --git a/pkg/transaction/metric/metricscollector.go b/pkg/transaction/metric/metricscollector.go index 613c43773..3a43c453c 100644 --- a/pkg/transaction/metric/metricscollector.go +++ b/pkg/transaction/metric/metricscollector.go @@ -52,6 +52,7 @@ func ExitMetricInit() { type Collector interface { InitializeBatch() AddMetric(apiDetails models.APIDetails, statusCode string, duration, bytes int64, appName string) + AddCustomMetricDetail(metric CustomMetricDetail) AddMetricDetail(metricDetail Detail) AddAPIMetricDetail(metric MetricDetail) AddAPIMetric(apiMetric *APIMetric) @@ -71,7 +72,7 @@ type collector struct { batchLock *sync.Mutex registry metrics.Registry metricBatch *EventBatch - metricMap map[string]map[string]map[string]map[string]*APIMetric + metricMap map[string]map[string]map[string]map[string]*centralMetricEvent metricMapLock *sync.Mutex publishItemQueue []publishQueueItem jobID string @@ -95,7 +96,6 @@ type usageEventPublishItem interface { } type usageEventQueueItem struct { - usageEventPublishItem event UsageEvent usageMetric metrics.Counter volumeMetric metrics.Counter @@ -164,7 +164,7 @@ func createMetricCollector() Collector { batchLock: &sync.Mutex{}, metricMapLock: &sync.Mutex{}, registry: metrics.NewRegistry(), - metricMap: make(map[string]map[string]map[string]map[string]*APIMetric), + metricMap: make(map[string]map[string]map[string]map[string]*centralMetricEvent), publishItemQueue: make([]publishQueueItem, 0), metricConfig: agent.GetCentralConfig().GetMetricReportingConfig(), usageConfig: agent.GetCentralConfig().GetUsageReportingConfig(), @@ -265,24 +265,88 @@ func (c *collector) AddMetricDetail(metricDetail Detail) { c.createOrUpdateMetric(metricDetail) } -// AddMetricDetailSet - add metric details for several response codes and transactions +// AddAPIMetricDetail - add metric details for several response codes and transactions func (c *collector) AddAPIMetricDetail(detail MetricDetail) { if !c.metricConfig.CanPublish() || c.usageConfig.IsOfflineMode() { return } - newMetric, _ := c.createMetric(TransactionContext{detail.APIDetails, detail.AppDetails, detail.StatusCode}) + transactionCtx := transactionContext{ + APIDetails: detail.APIDetails, + AppDetails: detail.AppDetails, + Status: detail.StatusCode, + } + newMetric := c.createMetric(transactionCtx) // update the new metric with all the necessary details newMetric.Count = detail.Count - newMetric.Response = detail.Response + newMetric.Response = &detail.Response newMetric.StartTime = time.UnixMilli(detail.Observation.Start) - newMetric.Observation = detail.Observation + newMetric.Observation = &detail.Observation + newMetric.StatusCode = detail.StatusCode + newMetric.Status = c.getStatusText(detail.StatusCode) - c.AddAPIMetric(newMetric) + c.addMetric(newMetric) +} + +// AddCustomMetricDetail - add custom unit metric details for an api/app combo +func (c *collector) AddCustomMetricDetail(detail CustomMetricDetail) { + if !c.metricConfig.CanPublish() || c.usageConfig.IsOfflineMode() { + return + } + + logger := c.logger.WithField("handler", "customMetric"). + WithField("apiID", detail.APIDetails.ID). + WithField("appID", detail.AppDetails.ID). + WithField("unitID", detail.UnitDetails.ID). + WithField("unitName", detail.UnitDetails.Name) + + if detail.APIDetails.ID == "" { + logger.Error("custom units require API information") + return + } + + if detail.AppDetails.ID == "" { + logger.Error("custom units require App information") + return + } + + if detail.UnitDetails.ID == "" || detail.UnitDetails.Name == "" { + logger.Error("custom units require Unit information") + return + } + logger.WithField("count", detail.Count).Debug("received custom unit report") + + transactionCtx := transactionContext{ + APIDetails: detail.APIDetails, + AppDetails: detail.AppDetails, + UnitName: detail.UnitDetails.ID, + } + + metric := c.createMetric(transactionCtx) + metric.StartTime = time.UnixMilli(detail.Observation.Start) + metric.Observation = &detail.Observation + + if m := c.getExistingMetric(metric); m != nil { + // use the cached metric + metric = m + } + + // add the count + metric.Count += detail.Count + + counter := c.getOrRegisterCounter(metric.getKey()) + counter.Inc(detail.Count) + + c.updateMetricWithCachedMetric(metric, newCustomCounter(counter)) } // AddAPIMetric - add api metric for API transaction func (c *collector) AddAPIMetric(metric *APIMetric) { + c.addMetric(centralMetricFromAPIMetric(metric)) +} + +// addMetric - add central metric event +func (c *collector) addMetric(metric *centralMetricEvent) { if metric.EventID == "" { metric.EventID = uuid.NewString() } @@ -327,60 +391,88 @@ func (c *collector) updateUsage(count int64) { c.storage.updateUsage(int(transactionCount.Count())) } -func (c *collector) createMetric(detail TransactionContext) (*APIMetric, []string) { +func (c *collector) createMetric(detail transactionContext) *centralMetricEvent { // Go get the access request and managed app accessRequest, managedApp := c.getAccessRequestAndManagedApp(agent.GetCacheManager(), detail) - // Update consumer details - subRef := v1.Reference{ - ID: unknown, - Name: unknown, - } - if accessRequest != nil { - accessReqSub := accessRequest.GetReferenceByGVK(catalog.SubscriptionGVK()) - if accessReqSub.ID != "" { - subRef = accessReqSub - } + cme := ¢ralMetricEvent{ + metricInfo: metricInfo{ + Subscription: c.createSubscriptionDetail(accessRequest), + App: c.createAppDetail(managedApp), + Product: c.getProduct(accessRequest), + API: c.createAPIDetail(detail.APIDetails), + AssetResource: c.getAssetResource(accessRequest), + ProductPlan: c.getProductPlan(accessRequest), + Quota: c.getQuota(accessRequest, detail.UnitName), + Unit: c.getProductPlanUnit(accessRequest, detail.UnitName), + }, + StartTime: now(), + EventID: uuid.NewString(), } - appDetail := c.createAppDetail(managedApp) - - // consumer, subscriptionID, appID, apiID, status - histogramKeyParts := []string{"consumer", subRef.ID, appDetail.ID, strings.ReplaceAll(detail.APIDetails.ID, ".", "#"), detail.StatusCode} + if detail.Status != "" { + cme.StatusCode = detail.Status + cme.Status = c.getStatusText(detail.Status) + } - return &APIMetric{ - Subscription: c.createSubscriptionDetail(subRef), - App: appDetail, - Product: c.getProduct(accessRequest, c.logger), - API: c.createAPIDetail(detail.APIDetails, accessRequest), - AssetResource: c.getAssetResource(accessRequest, c.logger), - ProductPlan: c.getProductPlan(accessRequest, c.logger), - Quota: c.getQuota(accessRequest, c.logger), - StatusCode: detail.StatusCode, - Status: c.getStatusText(detail.StatusCode), - StartTime: now(), - EventID: uuid.NewString(), - }, histogramKeyParts + return cme } -func (c *collector) createOrUpdateMetric(detail Detail) *APIMetric { +func (c *collector) createOrUpdateMetric(detail Detail) *centralMetricEvent { if !c.metricConfig.CanPublish() || c.usageConfig.IsOfflineMode() { return nil // no need to update metrics with publish off } - metric, keyParts := c.createMetric(TransactionContext{detail.APIDetails, detail.AppDetails, detail.StatusCode}) - histogram := c.getOrRegisterHistogram(strings.Join(keyParts, ".")) + transactionCtx := transactionContext{ + APIDetails: detail.APIDetails, + AppDetails: detail.AppDetails, + Status: detail.StatusCode, + UnitName: detail.UnitName, + } + + metric := c.createMetric(transactionCtx) + + histogram := c.getOrRegisterHistogram(metric.getKey()) + histogram.Update(detail.Duration) + + return c.updateMetricWithCachedMetric(metric, newCustomHistogram(histogram)) +} + +func (c *collector) getExistingMetric(metric *centralMetricEvent) *centralMetricEvent { + keyParts := strings.Split(metric.getKey(), ".") + + c.metricMapLock.Lock() + defer c.metricMapLock.Unlock() + + if _, ok := c.metricMap[keyParts[1]]; !ok { + return nil + } + if _, ok := c.metricMap[keyParts[1]][keyParts[2]]; !ok { + return nil + } + if _, ok := c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]]; !ok { + return nil + } + if _, ok := c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]]; !ok { + return nil + } + return c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]] +} + +func (c *collector) updateMetricWithCachedMetric(metric *centralMetricEvent, cached cachedMetricInterface) *centralMetricEvent { + keyParts := strings.Split(metric.getKey(), ".") c.metricMapLock.Lock() defer c.metricMapLock.Unlock() + if _, ok := c.metricMap[keyParts[1]]; !ok { - c.metricMap[keyParts[1]] = make(map[string]map[string]map[string]*APIMetric) + c.metricMap[keyParts[1]] = make(map[string]map[string]map[string]*centralMetricEvent) } if _, ok := c.metricMap[keyParts[1]][keyParts[2]]; !ok { - c.metricMap[keyParts[1]][keyParts[2]] = make(map[string]map[string]*APIMetric) + c.metricMap[keyParts[1]][keyParts[2]] = make(map[string]map[string]*centralMetricEvent) } if _, ok := c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]]; !ok { - c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]] = make(map[string]*APIMetric) + c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]] = make(map[string]*centralMetricEvent) } if _, ok := c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]]; !ok { // First api metric for sub+app+api+statuscode, @@ -388,13 +480,12 @@ func (c *collector) createOrUpdateMetric(detail Detail) *APIMetric { c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]] = metric } - histogram.Update(detail.Duration) - c.storage.updateMetric(histogram, c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]]) + c.storage.updateMetric(cached, c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]]) return c.metricMap[keyParts[1]][keyParts[2]][keyParts[3]][keyParts[4]] } // getAccessRequest - -func (c *collector) getAccessRequestAndManagedApp(cacheManager cache.Manager, detail TransactionContext) (*management.AccessRequest, *v1.ResourceInstance) { +func (c *collector) getAccessRequestAndManagedApp(cacheManager cache.Manager, detail transactionContext) (*management.AccessRequest, *v1.ResourceInstance) { if detail.AppDetails.Name == "" { return nil, nil } @@ -414,7 +505,7 @@ func (c *collector) getAccessRequestAndManagedApp(cacheManager cache.Manager, de } c.logger. WithField("appName", detail.AppDetails.Name). - WithField("managed-app-name", managedApp.Name). + WithField("managedAppName", managedApp.Name). Trace("managed application info") // get the access request @@ -425,159 +516,150 @@ func (c *collector) getAccessRequestAndManagedApp(cacheManager cache.Manager, de return nil, nil } c.logger. - WithField("managed-app-name", managedApp.Name). + WithField("managedAppName", managedApp.Name). WithField("apiID", detail.APIDetails.ID). WithField("stage", detail.APIDetails.Stage). - WithField("access-request-name", accessRequest.Name). + WithField("accessRequestName", accessRequest.Name). Trace("managed application info") return accessRequest, managedApp } -func (c *collector) createSubscriptionDetail(subRef v1.Reference) models.Subscription { - detail := models.Subscription{ - ID: unknown, - Name: unknown, +func (c *collector) createSubscriptionDetail(accessRequest *management.AccessRequest) *models.Subscription { + if accessRequest == nil { + return nil + } + + subRef := accessRequest.GetReferenceByGVK(catalog.SubscriptionGVK()) + if subRef.ID != "" { + return nil } - if subRef.ID != "" && subRef.Name != "" { - detail.ID = subRef.ID - detail.Name = subRef.Name + return &models.Subscription{ + ID: subRef.ID, + Name: subRef.Name, } - return detail } -func (c *collector) createAppDetail(app *v1.ResourceInstance) models.AppDetails { - detail := models.AppDetails{ - ID: unknown, - Name: unknown, +func (c *collector) createAppDetail(appRI *v1.ResourceInstance) *models.AppDetails { + if appRI == nil { + + // TODO remove the following when product plan unit ready + // return &models.AppDetails{ + // ID: "app-id", + // Name: "app-name", + // } + return nil } - if app != nil { - detail.ID, detail.Name = c.getConsumerApplication(app) - detail.ConsumerOrgID = c.getConsumerOrgID(app) + app := &management.ManagedApplication{} + app.FromInstance(appRI) + + orgID := "" + if app.Marketplace.Resource.Owner != nil { + orgID = app.Marketplace.Resource.Owner.Organization.ID } - return detail -} -func (c *collector) createAPIDetail(api models.APIDetails, accessReq *management.AccessRequest) models.APIDetails { - detail := models.APIDetails{ - ID: api.ID, - Name: api.Name, - Revision: api.Revision, - TeamID: api.TeamID, - APIServiceInstance: unknown, + appRef := app.GetReferenceByGVK(catalog.ApplicationGVK()) + if appRef.ID == "" { + return nil } - if accessReq != nil { - detail.APIServiceInstance = accessReq.Spec.ApiServiceInstance + return &models.AppDetails{ + ID: appRef.ID, + Name: appRef.Name, + ConsumerOrgID: orgID, } - return detail } -func (c *collector) getAssetResource(accessRequest *management.AccessRequest, log log.FieldLogger) models.AssetResource { - // Set default to provider details in case access request or managed apps comes back nil - assetResource := models.AssetResource{ - ID: unknown, - Name: unknown, +func (c *collector) createAPIDetail(api models.APIDetails) *models.APIDetails { + return &models.APIDetails{ + ID: api.ID, + Name: api.Name, + Revision: api.Revision, + TeamID: api.TeamID, } +} +func (c *collector) getAssetResource(accessRequest *management.AccessRequest) *models.AssetResource { if accessRequest == nil { - log.Trace("access request is nil. Setting default values to unknown") - return assetResource + return nil } assetResourceRef := accessRequest.GetReferenceByGVK(catalog.AssetResourceGVK()) - if assetResourceRef.ID == "" || assetResourceRef.Name == "" { - log.Trace("could not get asset resource, setting asset resource to unknown") - } else { - assetResource.ID = assetResourceRef.ID - assetResource.Name = assetResourceRef.Name + if assetResourceRef.ID == "" { + return nil } - log.WithField("asset-resource-id", assetResource.ID). - WithField("asset-resource-name", assetResource.Name). - Trace("asset resource information") - - return assetResource -} - -func (c *collector) getProduct(accessRequest *management.AccessRequest, log log.FieldLogger) models.Product { - product := models.Product{ - ID: unknown, - Name: unknown, - VersionID: unknown, - VersionName: unknown, + return &models.AssetResource{ + ID: assetResourceRef.ID, + Name: assetResourceRef.Name, } +} +func (c *collector) getProduct(accessRequest *management.AccessRequest) *models.Product { if accessRequest == nil { - log.Trace("access request is nil. Setting default values to unknown") - return product + return nil } productRef := accessRequest.GetReferenceByGVK(catalog.ProductGVK()) - if productRef.ID == "" || productRef.Name == "" { - log.Trace("could not get product information, setting product to unknown") - } else { - product.ID = productRef.ID - product.Name = productRef.Name - } + releaseRef := accessRequest.GetReferenceByGVK(catalog.ProductReleaseGVK()) - productReleaseRef := accessRequest.GetReferenceByGVK(catalog.ProductReleaseGVK()) - if productReleaseRef.ID == "" || productReleaseRef.Name == "" { - log.Trace("could not get product release information, setting product release to unknown") - } else { - product.VersionID = productReleaseRef.ID - product.VersionName = productReleaseRef.Name + if productRef.ID == "" || releaseRef.ID == "" { + return nil } - log.WithField("product-id", product.ID). - WithField("product-name", product.Name). - WithField("product-version-id", product.VersionID). - WithField("product-version-name", product.VersionName). - Trace("product information") - return product - -} -func (c *collector) getProductPlan(accessRequest *management.AccessRequest, log log.FieldLogger) models.ProductPlan { - productPlan := models.ProductPlan{ - ID: unknown, + return &models.Product{ + ID: productRef.ID, + Name: productRef.Name, + VersionID: releaseRef.ID, + VersionName: releaseRef.Name, } +} +func (c *collector) getProductPlan(accessRequest *management.AccessRequest) *models.ProductPlan { if accessRequest == nil { - log.Trace("access request is nil. Setting default values to unknown") - return productPlan + return nil } productPlanRef := accessRequest.GetReferenceByGVK(catalog.ProductPlanGVK()) if productPlanRef.ID == "" { - log.Debug("could not get product plan ID, setting product plan to unknown") - } else { - productPlan.ID = productPlanRef.ID + return nil } - log.WithField("product-plan-id", productPlan.ID). - Trace("product plan ID information") - return productPlan + return &models.ProductPlan{ + ID: productPlanRef.ID, + } } -func (c *collector) getQuota(accessRequest *management.AccessRequest, log log.FieldLogger) models.Quota { - quota := models.Quota{ - ID: unknown, - } +func (c *collector) getQuota(accessRequest *management.AccessRequest, id string) *models.Quota { if accessRequest == nil { - log.Trace("access request or managed app is nil. Setting default values to unknown") - return quota + return nil } - quotaRef := accessRequest.GetReferenceByGVK(catalog.QuotaGVK()) + + quotaRef := accessRequest.GetReferenceByIDAndGVK(id, catalog.QuotaGVK()) if quotaRef.ID == "" { - log.Debug("could not get quota ID, setting quota to unknown") - } else { - quota.ID = quotaRef.ID + return nil } - log.WithField("quota-id", quota.ID). - Trace("quota ID information") - return quota + return &models.Quota{ + ID: quotaRef.ID, + } +} + +func (c *collector) getProductPlanUnit(accessRequest *management.AccessRequest, name string) *models.Unit { + if accessRequest == nil { + return nil + } + + unitRef := accessRequest.GetReferenceByNameAndGVK(name, catalog.ProductPlanUnitGVK()) + if unitRef.ID == "" { + return nil + } + + return &models.Unit{ + ID: unitRef.ID, + Name: unitRef.Name, + } } func (c *collector) cleanup() { @@ -733,20 +815,25 @@ func (c *collector) processMetric(metricName string, metric interface{}) { } } -func (c *collector) setMetricsFromHistogram(metrics *APIMetric, histogram metrics.Histogram) { +func (c *collector) setMetricsFromHistogram(metrics *centralMetricEvent, histogram metrics.Histogram) { metrics.Count = histogram.Count() - metrics.Response.Max = histogram.Max() - metrics.Response.Min = histogram.Min() - metrics.Response.Avg = histogram.Mean() + metrics.Response = &ResponseMetrics{ + Max: histogram.Max(), + Min: histogram.Min(), + Avg: histogram.Mean(), + } } -func (c *collector) generateMetricEvent(histogram metrics.Histogram, metric *APIMetric) { +func (c *collector) generateMetricEvent(histogram metrics.Histogram, metric *centralMetricEvent) { if metric.Count == 0 { + c.logger.Trace("skipping registry entry with no reported quantity") return } + metric.Observation = &ObservationDetails{ + Start: util.ConvertTimeToMillis(c.metricStartTime), + End: util.ConvertTimeToMillis(c.metricEndTime), + } - metric.Observation.Start = util.ConvertTimeToMillis(c.metricStartTime) - metric.Observation.End = util.ConvertTimeToMillis(c.metricEndTime) // Generate app subscription metric c.generateV4Event(histogram, metric) } @@ -884,33 +971,3 @@ func (c *collector) getStatusText(statusCode string) string { return "Exception" } } - -func (c *collector) getConsumerOrgID(ri *v1.ResourceInstance) string { - if ri == nil { - return "" - } - - // Lookup Subscription - app := &management.ManagedApplication{} - app.FromInstance(ri) - - if app.Marketplace.Resource.Owner != nil { - return app.Marketplace.Resource.Owner.Organization.ID - } - return "" -} - -func (c *collector) getConsumerApplication(ri *v1.ResourceInstance) (string, string) { - if ri == nil { - return "", "" - } - - for _, ref := range ri.Metadata.References { - // get the ID of the Catalog Application - if ref.Kind == catalog.ApplicationGVK().Kind { - return ref.ID, ref.Name - } - } - - return ri.Metadata.ID, ri.Name // default to the managed app id -} diff --git a/pkg/transaction/metric/metricscollector_test.go b/pkg/transaction/metric/metricscollector_test.go index 8aa891415..eb2cd1fb0 100644 --- a/pkg/transaction/metric/metricscollector_test.go +++ b/pkg/transaction/metric/metricscollector_test.go @@ -35,8 +35,8 @@ var ( Version: "", } apiDetails2 = models.APIDetails{ - ID: "111", - Name: "111", + ID: "222", + Name: "222", Revision: 1, TeamID: teamID, APIServiceInstance: "", @@ -44,6 +44,16 @@ var ( Version: "", } traceStatus = healthcheck.OK + appDetails1 = models.AppDetails{ + ID: "111", + Name: "111", + ConsumerOrgID: "org-id-111", + } + appDetails2 = models.AppDetails{ + ID: "222", + Name: "222", + ConsumerOrgID: "org-id-222", + } ) func getFutureTime() time.Time { @@ -180,7 +190,7 @@ func generateMockReports(transactionPerReport []int) UsageEvent { return mockEvent } -func cleanUpReportfiles() { +func cleanUpReportFiles() { os.RemoveAll("./reports") } @@ -312,7 +322,23 @@ func TestMetricCollector(t *testing.T) { appName string publishPrior bool hcStatus healthcheck.StatusLevel + skipWaitForPub bool }{ + // Success case with no usage report + { + name: "WithUsageNoUsageReport", + loopCount: 1, + retryBatchCount: 0, + apiTransactionCount: []int{0}, + failUsageEventOnServer: []bool{false}, + failUsageResponseOnServer: []*UsageResponse{nil}, + expectedLHEvents: []int{0}, + expectedTransactionCount: []int{0}, + trackVolume: false, + expectedTransactionVolume: []int{0}, + expectedMetricEventsAcked: 0, + skipWaitForPub: true, + }, // Success case with no app detail { name: "WithUsage", @@ -371,20 +397,6 @@ func TestMetricCollector(t *testing.T) { expectedMetricEventsAcked: 1, // API metric + Provider + Consumer subscription metric appName: "managed-app-2", }, - // Success case with no usage report - { - name: "WithUsageNoUsageReport", - loopCount: 1, - retryBatchCount: 0, - apiTransactionCount: []int{0}, - failUsageEventOnServer: []bool{false}, - failUsageResponseOnServer: []*UsageResponse{nil}, - expectedLHEvents: []int{0}, - expectedTransactionCount: []int{0}, - trackVolume: false, - expectedTransactionVolume: []int{0}, - expectedMetricEventsAcked: 0, - }, // Test case with failing request to LH, the subsequent successful request should contain the total count since initial failure { name: "WithUsageWithFailure", @@ -464,6 +476,7 @@ func TestMetricCollector(t *testing.T) { expectedMetricEventsAcked: 0, // API metric + Provider subscription metric appName: "managed-app-1", hcStatus: healthcheck.FAIL, + skipWaitForPub: true, }, } @@ -473,9 +486,12 @@ func TestMetricCollector(t *testing.T) { traceStatus = test.hcStatus } runTestHealthcheck() - + myCollector.InitializeBatch() + metricCollector.metricMap = make(map[string]map[string]map[string]map[string]*centralMetricEvent) cfg.SetAxwayManaged(test.trackVolume) - setupMockClient(test.retryBatchCount) + testClient := setupMockClient(test.retryBatchCount) + mockClient := testClient.(*MockClient) + for l := 0; l < test.loopCount; l++ { fmt.Printf("\n\nTransaction Info: %+v\n\n", test.apiTransactionCount[l]) for i := 0; i < test.apiTransactionCount[l]; i++ { @@ -500,8 +516,8 @@ func TestMetricCollector(t *testing.T) { metricCollector.Execute() metricCollector.usagePublisher.Execute() } - assert.Equal(t, test.expectedMetricEventsAcked, myMockClient.(*MockClient).eventsAcked) } + assert.Equal(t, test.expectedMetricEventsAcked, mockClient.eventsAcked) s.resetConfig() }) } @@ -595,6 +611,16 @@ func TestMetricCollectorUsageAggregation(t *testing.T) { cmd.BuildDataPlaneType = "Azure" agent.Initialize(cfg) + // setup the cache for handling custom metrics + cm := agent.GetCacheManager() + cm.AddAPIServiceInstance(createAPIServiceInstance("inst-1", "instance-1", "111")) + + cm.AddManagedApplication(createManagedApplication("app-1", "managed-app-1", "")) + cm.AddManagedApplication(createManagedApplication("app-2", "managed-app-2", "test-consumer-org")) + + cm.AddAccessRequest(createAccessRequest("ac-1", "access-req-1", "managed-app-1", "inst-1", "instance-1", "subscription-1")) + cm.AddAccessRequest(createAccessRequest("ac-2", "access-req-2", "managed-app-2", "inst-1", "instance-1", "subscription-2")) + traceStatus = healthcheck.OK runTestHealthcheck() @@ -649,7 +675,7 @@ func TestMetricCollectorUsageAggregation(t *testing.T) { s.resetConfig() }) } - cleanUpReportfiles() + cleanUpReportFiles() } func TestMetricCollectorCache(t *testing.T) { @@ -836,7 +862,7 @@ func TestOfflineMetricCollector(t *testing.T) { publisher := metricCollector.usagePublisher for testLoops < test.loopCount { for i := 0; i < test.apiTransactionCount[testLoops]; i++ { - metricCollector.AddMetric(apiDetails1, "200", 10, 10, "") + myCollector.AddMetric(apiDetails1, "200", 10, 10, "") } metricCollector.Execute() testLoops++ @@ -874,5 +900,94 @@ func TestOfflineMetricCollector(t *testing.T) { s.resetOffline(myCollector) }) } - cleanUpReportfiles() + cleanUpReportFiles() +} + +func TestCustomMetrics(t *testing.T) { + defer cleanUpCachedMetricFile() + s := &testHTTPServer{} + defer s.closeServer() + s.startServer() + + traceStatus = healthcheck.OK + traceability.SetDataDirPath(".") + runTestHealthcheck() + + cfg := createCentralCfg(s.server.URL, "demo") + cfg.UsageReporting.(*config.UsageReportingConfiguration).URL = s.server.URL + "/usage" + cfg.SetEnvironmentID("267bd671-e5e2-4679-bcc3-bbe7b70f30fd") + cmd.BuildDataPlaneType = "Azure" + agent.Initialize(cfg) + + cm := agent.GetCacheManager() + cm.AddAPIServiceInstance(createAPIServiceInstance("inst-1", "instance-1", "111")) + + cm.AddManagedApplication(createManagedApplication("app-1", "managed-app-1", "")) + cm.AddManagedApplication(createManagedApplication("app-2", "managed-app-2", "test-consumer-org")) + + cm.AddAccessRequest(createAccessRequest("ac-1", "access-req-1", "managed-app-1", "inst-1", "instance-1", "subscription-1")) + cm.AddAccessRequest(createAccessRequest("ac-2", "access-req-2", "managed-app-2", "inst-1", "instance-1", "subscription-2")) + + myCollector := createMetricCollector() + metricCollector := myCollector.(*collector) + + base := CustomMetricDetail{ + APIDetails: apiDetails1, + AppDetails: appDetails1, + Count: 5, + UnitDetails: models.Unit{ + ID: "unit-id", + Name: "unit-name", + }, + } + _ = base + + testCases := map[string]struct { + skip bool + metricEvent1 CustomMetricDetail + metricEvent2 CustomMetricDetail + expectedMetrics int + }{ + "no custom metric when api details not in event": { + skip: false, + metricEvent1: CustomMetricDetail{}, + }, + "no custom metric when app details not in event": { + skip: false, + metricEvent1: CustomMetricDetail{ + APIDetails: apiDetails1, + }, + }, + "no custom metric when unit details not in event": { + skip: false, + metricEvent1: CustomMetricDetail{ + APIDetails: apiDetails1, + AppDetails: appDetails1, + }, + }, + "expect custom metric when all needed data given": { + skip: false, + metricEvent1: base, + expectedMetrics: 1, + }, + "expect 1 metric when multiple updates for same unit and detials": { + skip: false, + metricEvent1: base, + metricEvent2: base, + expectedMetrics: 1, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.skip { + return + } + metricCollector.metricMap = map[string]map[string]map[string]map[string]*centralMetricEvent{} + metricCollector.AddCustomMetricDetail(tc.metricEvent1) + if tc.metricEvent2.Count > 0 { + metricCollector.AddCustomMetricDetail(tc.metricEvent2) + } + assert.Equal(t, tc.expectedMetrics, len(metricCollector.metricMap)) + }) + } } diff --git a/pkg/transaction/metric/mockclient_test.go b/pkg/transaction/metric/mockclient_test.go index 926b40ef7..22c9bba25 100644 --- a/pkg/transaction/metric/mockclient_test.go +++ b/pkg/transaction/metric/mockclient_test.go @@ -37,20 +37,17 @@ func (m *MockClient) String() string { return "" } -var myMockClient outputs.Client - -func mockGetClient() (*traceability.Client, error) { - tpClient := &traceability.Client{} - tpClient.SetTransportClient(myMockClient) - tpClient.SetLogger(log.NewFieldLogger()) - return tpClient, nil -} - -func setupMockClient(retries int) { - myMockClient = &MockClient{ +func setupMockClient(retries int) outputs.Client { + testClient := &MockClient{ pubCount: 0, retry: retries, eventsAcked: 0, } - traceability.GetClient = mockGetClient + traceability.GetClient = func() (*traceability.Client, error) { + tpClient := &traceability.Client{} + tpClient.SetTransportClient(testClient) + tpClient.SetLogger(log.NewFieldLogger()) + return tpClient, nil + } + return testClient } diff --git a/pkg/transaction/metric/reportcache.go b/pkg/transaction/metric/reportcache.go index 95ae212bc..60c6f1a76 100644 --- a/pkg/transaction/metric/reportcache.go +++ b/pkg/transaction/metric/reportcache.go @@ -281,9 +281,5 @@ func (c *usageReportCache) shouldPublish(schedule string) bool { } // publish if last scheduled time is past nextPublishTime := cronSchedule.Next(lastPublishTimestamp) - if nextPublishTime.Before(currentTime) { - return true - } - - return false + return nextPublishTime.Before(currentTime) } diff --git a/pkg/transaction/metric/util.go b/pkg/transaction/metric/util.go new file mode 100644 index 000000000..4f35ef03f --- /dev/null +++ b/pkg/transaction/metric/util.go @@ -0,0 +1,81 @@ +package metric + +import ( + "github.com/Axway/agent-sdk/pkg/transaction/models" +) + +func centralMetricFromAPIMetric(in *APIMetric) *centralMetricEvent { + out := ¢ralMetricEvent{ + metricInfo: metricInfo{ + StatusCode: in.StatusCode, + }, + Count: in.Count, + Status: in.Status, + EventID: in.EventID, + StartTime: in.StartTime, + } + + if in.Subscription.ID != unknown && in.Subscription.ID != "" { + out.Subscription = &models.Subscription{ + ID: in.Subscription.ID, + Name: in.Subscription.Name, + } + } + + if in.App.ID != unknown && in.App.ID != "" { + out.App = &models.AppDetails{ + ID: in.App.ID, + Name: in.App.Name, + ConsumerOrgID: in.App.ConsumerOrgID, + } + } + + if in.Product.ID != unknown && in.Product.ID != "" { + out.Product = &models.Product{ + ID: in.Product.ID, + Name: in.Product.Name, + VersionName: in.Product.VersionName, + VersionID: in.Product.VersionID, + } + } + + if in.API.ID != unknown && in.API.ID != "" { + out.API = &models.APIDetails{ + ID: in.API.ID, + Name: in.API.Name, + Revision: in.API.Revision, + TeamID: in.API.TeamID, + APIServiceInstance: in.API.APIServiceInstance, + Stage: in.API.Stage, + Version: in.API.Version, + } + } + + if in.AssetResource.ID != unknown && in.AssetResource.ID != "" { + out.AssetResource = &models.AssetResource{ + ID: in.AssetResource.ID, + Name: in.AssetResource.Name, + } + } + + if in.ProductPlan.ID != unknown && in.ProductPlan.ID != "" { + out.ProductPlan = &models.ProductPlan{ + ID: in.ProductPlan.ID, + } + } + + if in.Quota.ID != unknown && in.Quota.ID != "" { + out.Quota = &models.Quota{ + ID: in.Quota.ID, + } + } + + if in.Unit.ID != unknown && in.Unit.ID != "" { + out.Unit = &models.Unit{ + ID: in.Unit.ID, + Name: in.Unit.Name, + } + } + + return out +} diff --git a/pkg/transaction/models/definitions.go b/pkg/transaction/models/definitions.go index b7a6e9dc6..7408a081e 100644 --- a/pkg/transaction/models/definitions.go +++ b/pkg/transaction/models/definitions.go @@ -106,3 +106,16 @@ func (a APIDetails) GetLogFields(fields logrus.Fields) logrus.Fields { } return fields } + +// Unit - struct for custom unit details to report +type Unit struct { + ID string `json:"id"` + Name string `json:"name"` +} + +func (a Unit) GetLogFields(fields logrus.Fields) logrus.Fields { + if a.ID != "unknown" { + fields["apiID"] = a.ID + } + return fields +} diff --git a/scripts/apiserver/generate.js b/scripts/apiserver/generate.js index d5ff9577f..7b7436659 100644 --- a/scripts/apiserver/generate.js +++ b/scripts/apiserver/generate.js @@ -88,7 +88,8 @@ const writeSubResources = (subResources) => { for (let groupKey in subResources) { const groupObj = subResources[groupKey]; for (let versionKey in groupObj) { - const data = JSON.stringify(groupObj[versionKey]); + // stringify the group and version data, update all references to drop group and version within path + const data = JSON.stringify(groupObj[versionKey]).replaceAll(`components/schemas/${groupKey}.${versionKey}.`, `components/schemas/`); const res = execSync( `openapi-generator-cli generate -g go -i /dev/stdin --package-name ${groupKey} --output ${modelsPath}${groupKey}/${versionKey} --global-property modelDocs=false,models << 'EOF'\n${data}\nEOF` );