APIGOV-28931 - custom unit ingestion and refactoring of metric storag…

…e/sending (#843)

* udpates for custom unit metric ingestion

* get event id from v4Event during acking

* set status in metric

* use pointers in metric event sent to central

* code cleanup

* add additional methods to get resource reference information

* fix get key method

* update to use counter for custom metrics

* cleanup

* add test for sending custom usage metrics

* add name for generator docker container

* resource generate

* update models

* APIGOV-28931 - fix generator

* get product plan unit ref

* use the plan unit ref while handing metrics

Makefile

@@ -36,7 +36,7 @@ test-sonar: dep
 	@go test -short -coverpkg=./... -coverprofile=${WORKSPACE}/gocoverage.out -count=1 ${GO_PKG_LIST} -json > ${WORKSPACE}/goreport.json
 
 apiserver-generate: # generate api server resources, prod by default. ex: make apiserver-generate protocol=https host=apicentral.axway.com port=443
-	docker run --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)'  -e PORT='$(port)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest
+	docker run --name generator --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)'  -e PORT='$(port)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest
 
 PROTOFILES := $(shell find $(WORKSPACE)/proto -type f -name '*.proto')
 PROTOTARGETS := $(PROTOFILES:.proto=.pb.go)

pkg/apic/apiserver/models/api/v1/resourcemeta.go

@@ -196,6 +196,26 @@ func (rm *ResourceMeta) GetReferenceByGVK(gvk GroupVersionKind) Reference {
 	return Reference{}
 }
 
+// GetReferenceByIDAndGVK returns the first found reference that matches the ID and GroupKind arguments.
+func (rm *ResourceMeta) GetReferenceByIDAndGVK(id string, gvk GroupVersionKind) Reference {
+	for _, ref := range rm.Metadata.References {
+		if ref.Group == gvk.Group && ref.Kind == gvk.Kind && ref.ID == id {
+			return ref
+		}
+	}
+	return Reference{}
+}
+
+// GetReferenceByNameAndGVK returns the first found reference that matches the Name and GroupKind arguments.
+func (rm *ResourceMeta) GetReferenceByNameAndGVK(name string, gvk GroupVersionKind) Reference {
+	for _, ref := range rm.Metadata.References {
+		if ref.Group == gvk.Group && ref.Kind == gvk.Kind && ref.Name == name {
+			return ref
+		}
+	}
+	return Reference{}
+}
+
 // MarshalJSON marshals the ResourceMeta to properly set the SubResources
 func (rm *ResourceMeta) MarshalJSON() ([]byte, error) {
 	rawSubs := map[string]interface{}{}

pkg/apic/apiserver/models/catalog/v1alpha1/SubscriptionInvoice.go

@@ -30,6 +30,7 @@ const (
 	SubscriptionInvoiceResourceName               = "subscriptioninvoices"
 	SubscriptionInvoiceBillingSubResourceName     = "billing"
 	SubscriptionInvoiceMarketplaceSubResourceName = "marketplace"
+	SubscriptionInvoiceReferencesSubResourceName  = "references"
 	SubscriptionInvoiceStateSubResourceName       = "state"
 	SubscriptionInvoiceStatusSubResourceName      = "status"
 )
@@ -49,6 +50,7 @@ type SubscriptionInvoice struct {
 	Billing     SubscriptionInvoiceBilling     `json:"billing"`
 	Marketplace SubscriptionInvoiceMarketplace `json:"marketplace"`
 	Owner       *apiv1.Owner                   `json:"owner"`
+	References  SubscriptionInvoiceReferences  `json:"references"`
 	Spec        SubscriptionInvoiceSpec        `json:"spec"`
 	State       SubscriptionInvoiceState       `json:"state"`
 	// Status      SubscriptionInvoiceStatus      `json:"status"`
@@ -140,6 +142,7 @@ func (res *SubscriptionInvoice) MarshalJSON() ([]byte, error) {
 	out["billing"] = res.Billing
 	out["marketplace"] = res.Marketplace
 	out["owner"] = res.Owner
+	out["references"] = res.References
 	out["spec"] = res.Spec
 	out["state"] = res.State
 	out["status"] = res.Status
@@ -200,6 +203,20 @@ func (res *SubscriptionInvoice) UnmarshalJSON(data []byte) error {
 		}
 	}
 
+	// marshalling subresource References
+	if v, ok := aux.SubResources["references"]; ok {
+		sr, err = json.Marshal(v)
+		if err != nil {
+			return err
+		}
+
+		delete(aux.SubResources, "references")
+		err = json.Unmarshal(sr, &res.References)
+		if err != nil {
+			return err
+		}
+	}
+
 	// marshalling subresource State
 	if v, ok := aux.SubResources["state"]; ok {
 		sr, err = json.Marshal(v)

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_asset_resource.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusResourceAssetResource  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusResourceAssetResource struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_resource_stage.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusResourceStage  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusResourceStage struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusSourceResourceApiService  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiService struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusSourceResourceApiServiceInstance  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiServiceInstance struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusSourceResourceApiServiceRevision  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiServiceRevision struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go

@@ -11,7 +11,6 @@ package catalog
 
 // AssetMappingStatusSourceResourceAssetMappingTemplate  (catalog.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceAssetMappingTemplate struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_product_plan_unit_spec.go

@@ -11,6 +11,6 @@ package catalog
 
 // ProductPlanUnitSpec  (catalog.v1alpha1.ProductPlanUnit)
 type ProductPlanUnitSpec struct {
-	// description of the Plan.
+	// description of the Product Plan Unit.
 	Description string `json:"description,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_release_tag_status_success_meta.go

@@ -11,6 +11,5 @@ package catalog
 
 // ReleaseTagStatusSuccessMeta struct for ReleaseTagStatusSuccessMeta
 type ReleaseTagStatusSuccessMeta struct {
-	// GENERATE: The following code has been modified after code generation
 	Reference []ReleaseTagStatusReference `json:"reference,omitempty"`
 }

pkg/apic/apiserver/models/catalog/v1alpha1/model_subscription_invoice_references.go

@@ -0,0 +1,16 @@
+/*
+ * API Server specification.
+ *
+ * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+ *
+ * API version: SNAPSHOT
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package catalog
+
+// SubscriptionInvoiceReferences  (catalog.v1alpha1.SubscriptionInvoice)
+type SubscriptionInvoiceReferences struct {
+	// Reference to product.
+	Product string `json:"product,omitempty"`
+}

pkg/apic/apiserver/models/management/v1alpha1/model_access_request_references_quota.go

@@ -13,4 +13,6 @@ package management
 type AccessRequestReferencesQuota struct {
 	Kind string `json:"kind"`
 	Name string `json:"name,omitempty"`
+	// The name of the unit for the included quota.
+	Unit string `json:"unit,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_additional_quotas.go

@@ -0,0 +1,20 @@
+/*
+ * API Server specification.
+ *
+ * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+ *
+ * API version: SNAPSHOT
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package management
+
+// AccessRequestSpecAdditionalQuotas Quota information for accessing the api. (management.v1alpha1.AccessRequest)
+type AccessRequestSpecAdditionalQuotas struct {
+	// The logical name of the quota
+	Name string `json:"name"`
+	// The limit of the allowed quota for the access request.
+	Limit int32 `json:"limit,omitempty"`
+	// The interval for the applied quota.
+	Interval string `json:"interval,omitempty"`
+}

pkg/apic/apiserver/models/management/v1alpha1/model_access_request_spec_quota.go

@@ -9,7 +9,7 @@
 
 package management
 
-// AccessRequestSpecQuota Quota information for accessing the api. (management.v1alpha1.AccessRequest)
+// AccessRequestSpecQuota Quota information for api transactions in accessing the api. (management.v1alpha1.AccessRequest)
 type AccessRequestSpecQuota struct {
 	// The limit of the allowed quota for the access request.
 	Limit    int32  `json:"limit"`

pkg/apic/apiserver/models/management/v1alpha1/model_api_service_compliance.go

@@ -11,10 +11,7 @@ package management
 
 // ApiServiceCompliance  (management.v1alpha1.APIService)
 type ApiServiceCompliance struct {
-	// GENERATE: The following code has been modified after code generation
-	Design ApiServiceComplianceLintingStatus `json:"design,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Design   ApiServiceComplianceLintingStatus `json:"design,omitempty"`
 	Security ApiServiceComplianceLintingStatus `json:"security,omitempty"`
-	// GENERATE: The following code has been modified after code generation
-	Runtime ApiServiceComplianceRuntimeStatus `json:"runtime,omitempty"`
+	Runtime  ApiServiceComplianceRuntimeStatus `json:"runtime,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_compliance.go

@@ -11,6 +11,5 @@ package management
 
 // ApiServiceInstanceCompliance  (management.v1alpha1.APIServiceInstance)
 type ApiServiceInstanceCompliance struct {
-	// GENERATE: The following code has been modified after code generation
 	Runtime ApiServiceInstanceComplianceRuntimeStatus `json:"runtime,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_api_service_instance_source_compliance.go

@@ -11,6 +11,5 @@ package management
 
 // ApiServiceInstanceSourceCompliance  (management.v1alpha1.APIServiceInstance)
 type ApiServiceInstanceSourceCompliance struct {
-	// GENERATE: The following code has been modified after code generation
 	Runtime ApiServiceInstanceSourceRuntimeStatus `json:"runtime,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_api_service_revision_compliance.go

@@ -11,8 +11,6 @@ package management
 
 // ApiServiceRevisionCompliance  (management.v1alpha1.APIServiceRevision)
 type ApiServiceRevisionCompliance struct {
-	// GENERATE: The following code has been modified after code generation
-	Design ApiServiceRevisionComplianceLintingStatus `json:"design,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Design   ApiServiceRevisionComplianceLintingStatus `json:"design,omitempty"`
 	Security ApiServiceRevisionComplianceLintingStatus `json:"security,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_asset_resource.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusResourceAssetResource  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusResourceAssetResource struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_resource_stage.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusResourceStage  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusResourceStage struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusSourceResourceApiService  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiService struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_instance.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusSourceResourceApiServiceInstance  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiServiceInstance struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_api_service_revision.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusSourceResourceApiServiceRevision  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceApiServiceRevision struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_asset_mapping_status_source_resource_asset_mapping_template.go

@@ -11,7 +11,6 @@ package management
 
 // AssetMappingStatusSourceResourceAssetMappingTemplate  (management.v1alpha1.AssetMapping)
 type AssetMappingStatusSourceResourceAssetMappingTemplate struct {
-	Ref string `json:"ref,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Ref           string                          `json:"ref,omitempty"`
 	OperationType AssetMappingStatusOperationType `json:"operationType,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_environment_compliancetasks.go

@@ -11,8 +11,6 @@ package management
 
 // EnvironmentCompliancetasks Subresource that is only accessible by the backend. Used by compliance-controller to auto-start/cancel linting jobs based on changes made to \"spec.compliance\" properties detected via mutation hook. (management.v1alpha1.Environment)
 type EnvironmentCompliancetasks struct {
-	// GENERATE: The following code has been modified after code generation
-	Design EnvironmentCompliancetasksLinting `json:"design,omitempty"`
-	// GENERATE: The following code has been modified after code generation
+	Design   EnvironmentCompliancetasksLinting `json:"design,omitempty"`
 	Security EnvironmentCompliancetasksLinting `json:"security,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_secret_spec.go

@@ -16,9 +16,7 @@ type IdentityProviderSecretSpec struct {
 	// GENERATE: The following code has been modified after code generation
 	Config interface{} `json:"config"`
 	// The list of request headers to be sent in the call to get token from Identity provider
-	// GENERATE: The following code has been modified after code generation
 	RequestHeaders []IdentityProviderSecretSpecKeyValuePair `json:"requestHeaders,omitempty"`
 	// The list of query parameters to be sent in the call to get token from Identity provider
-	// GENERATE: The following code has been modified after code generation
 	QueryParameters []IdentityProviderSecretSpecKeyValuePair `json:"queryParameters,omitempty"`
 }

pkg/apic/apiserver/models/management/v1alpha1/model_identity_provider_spec.go

@@ -18,13 +18,10 @@ type IdentityProviderSpec struct {
 	// The flag to configure agent to use registration access token for removing/updating the client created by agent in Identity provider
 	UseRegistrationAccessToken bool `json:"useRegistrationAccessToken,omitempty"`
 	// The list of request headers to be sent in the call to Identity provider for creating/updating and removing clients
-	// GENERATE: The following code has been modified after code generation
 	RequestHeaders []IdentityProviderSpecKeyValuePair `json:"requestHeaders,omitempty"`
 	// The list of query parameters to be sent in the call to Identity provider for creating/updating and removing clients
-	// GENERATE: The following code has been modified after code generation
 	QueryParameters []IdentityProviderSpecKeyValuePair `json:"queryParameters,omitempty"`
 	// The list of additional metadata properties that will be set when registering the client in Identity provider
-	// GENERATE: The following code has been modified after code generation
 	AdditionalClientProperties []IdentityProviderSpecKeyValuePair `json:"additionalClientProperties,omitempty"`
 	// Defines the timeout interval in seconds for Identity provider http client. Defaults to 30 seconds
 	ClientTimeout int32 `json:"clientTimeout,omitempty"`

pkg/transaction/metric/apimetric.go

@@ -0,0 +1,63 @@
+package metric
+
+import (
+	"time"
+
+	"github.com/Axway/agent-sdk/pkg/transaction/models"
+	"github.com/sirupsen/logrus"
+)
+
+// APIMetric - struct to hold metric aggregated for subscription,application,api,statuscode
+type APIMetric struct {
+	Subscription  models.Subscription  `json:"subscription,omitempty"`
+	App           models.AppDetails    `json:"application,omitempty"`
+	Product       models.Product       `json:"product,omitempty"`
+	API           models.APIDetails    `json:"api"`
+	AssetResource models.AssetResource `json:"assetResource,omitempty"`
+	ProductPlan   models.ProductPlan   `json:"productPlan,omitempty"`
+	Quota         models.Quota         `json:"quota,omitempty"`
+	Unit          models.Unit          `json:"unit,omitempty"`
+	StatusCode    string               `json:"statusCode,omitempty"`
+	Status        string               `json:"status,omitempty"`
+	Count         int64                `json:"count"`
+	Response      ResponseMetrics      `json:"response,omitempty"`
+	Observation   ObservationDetails   `json:"observation"`
+	EventID       string               `json:"-"`
+	StartTime     time.Time            `json:"-"`
+}
+
+// GetStartTime - Returns the start time for subscription metric
+func (a *APIMetric) GetStartTime() time.Time {
+	return a.StartTime
+}
+
+// GetType - Returns APIMetric
+func (a *APIMetric) GetType() string {
+	return "APIMetric"
+}
+
+// GetType - Returns APIMetric
+func (a *APIMetric) GetEventID() string {
+	return a.EventID
+}
+
+func (a *APIMetric) GetLogFields() logrus.Fields {
+	fields := logrus.Fields{
+		"id":             a.EventID,
+		"count":          a.Count,
+		"status":         a.StatusCode,
+		"minResponse":    a.Response.Min,
+		"maxResponse":    a.Response.Max,
+		"avgResponse":    a.Response.Avg,
+		"startTimestamp": a.Observation.Start,
+		"endTimestamp":   a.Observation.End,
+	}
+	fields = a.Subscription.GetLogFields(fields)
+	fields = a.App.GetLogFields(fields)
+	fields = a.Product.GetLogFields(fields)
+	fields = a.API.GetLogFields(fields)
+	fields = a.AssetResource.GetLogFields(fields)
+	fields = a.ProductPlan.GetLogFields(fields)
+	fields = a.Quota.GetLogFields(fields)
+	return fields
+}