diff --git a/apis/vshn/v1/dbaas_vshn_keycloak.go b/apis/vshn/v1/dbaas_vshn_keycloak.go index f75d5d2d6..18c533fb1 100644 --- a/apis/vshn/v1/dbaas_vshn_keycloak.go +++ b/apis/vshn/v1/dbaas_vshn_keycloak.go @@ -190,6 +190,10 @@ func (v *VSHNKeycloak) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNKeycloak) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNKeycloak) GetInstanceNamespace() string { return fmt.Sprintf("vshn-keycloak-%s", v.GetName()) } @@ -341,3 +345,11 @@ func (v *VSHNKeycloak) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMa func (v *VSHNKeycloak) GetWorkloadName() string { return v.GetName() + "-keycloakx" } + +func (v *VSHNKeycloak) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNKeycloak) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_mariadb.go b/apis/vshn/v1/dbaas_vshn_mariadb.go index 643af44ed..187c4a60d 100644 --- a/apis/vshn/v1/dbaas_vshn_mariadb.go +++ b/apis/vshn/v1/dbaas_vshn_mariadb.go @@ -133,6 +133,10 @@ func (v *VSHNMariaDB) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNMariaDB) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNMariaDB) GetInstanceNamespace() string { return fmt.Sprintf("vshn-mariadb-%s", v.GetName()) } @@ -280,3 +284,11 @@ func (v *VSHNMariaDB) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsMan func (v *VSHNMariaDB) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNMariaDB) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNMariaDB) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_postgresql.go b/apis/vshn/v1/dbaas_vshn_postgresql.go index 4a5ec52a9..d7e2b0ae9 100644 --- a/apis/vshn/v1/dbaas_vshn_postgresql.go +++ b/apis/vshn/v1/dbaas_vshn_postgresql.go @@ -248,6 +248,10 @@ func (v *VSHNPostgreSQL) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNPostgreSQL) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + // +kubebuilder:object:root=true // VSHNPostgreSQLList defines a list of VSHNPostgreSQL @@ -404,3 +408,11 @@ func (v *VSHNPostgreSQL) GetWorkloadPodTemplateLabelsManager() PodTemplateLabels func (v *VSHNPostgreSQL) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNPostgreSQL) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNPostgreSQL) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/dbaas_vshn_redis.go b/apis/vshn/v1/dbaas_vshn_redis.go index 3703fb2cb..f8dafcd25 100644 --- a/apis/vshn/v1/dbaas_vshn_redis.go +++ b/apis/vshn/v1/dbaas_vshn_redis.go @@ -150,6 +150,10 @@ func (v *VSHNRedis) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNRedis) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + // +kubebuilder:object:generate=true // +kubebuilder:object:root=true @@ -305,3 +309,11 @@ func (v *VSHNRedis) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag func (v *VSHNRedis) GetWorkloadName() string { return "redis-master" } + +func (v *VSHNRedis) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNRedis) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/apis/vshn/v1/vshn_minio.go b/apis/vshn/v1/vshn_minio.go index e18726728..881386ebc 100644 --- a/apis/vshn/v1/vshn_minio.go +++ b/apis/vshn/v1/vshn_minio.go @@ -106,6 +106,10 @@ func (v *VSHNMinio) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNMinio) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNMinio) GetInstanceNamespace() string { return fmt.Sprintf("vshn-minio-%s", v.GetName()) } @@ -257,3 +261,11 @@ func (v *VSHNMinio) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsManag func (v *VSHNMinio) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNMinio) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNMinio) GetSLA() string { + return string(BestEffort) +} diff --git a/apis/vshn/v1/vshn_nextcloud.go b/apis/vshn/v1/vshn_nextcloud.go index 4b846a49a..6701022f2 100644 --- a/apis/vshn/v1/vshn_nextcloud.go +++ b/apis/vshn/v1/vshn_nextcloud.go @@ -158,6 +158,10 @@ func (v *VSHNNextcloud) GetClaimNamespace() string { return v.GetLabels()["crossplane.io/claim-namespace"] } +func (v *VSHNNextcloud) GetClaimName() string { + return v.GetLabels()["crossplane.io/claim-name"] +} + func (v *VSHNNextcloud) GetInstanceNamespace() string { return fmt.Sprintf("vshn-nextcloud-%s", v.GetName()) } @@ -311,3 +315,11 @@ func (v *VSHNNextcloud) GetWorkloadPodTemplateLabelsManager() PodTemplateLabelsM func (v *VSHNNextcloud) GetWorkloadName() string { return v.GetName() } + +func (v *VSHNNextcloud) GetBillingName() string { + return "appcat-" + v.GetServiceName() +} + +func (v *VSHNNextcloud) GetSLA() string { + return string(v.Spec.Parameters.Service.ServiceLevel) +} diff --git a/pkg/comp-functions/functions/common/billing.go b/pkg/comp-functions/functions/common/billing.go index d4f5d74a7..f44d1eec4 100644 --- a/pkg/comp-functions/functions/common/billing.go +++ b/pkg/comp-functions/functions/common/billing.go @@ -1,24 +1,30 @@ package common import ( + "bytes" "context" + _ "embed" "fmt" v12 "github.com/crossplane/crossplane-runtime/apis/common/v1" xfnproto "github.com/crossplane/function-sdk-go/proto/v1beta1" + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" xkube "github.com/vshn/appcat/v4/apis/kubernetes/v1alpha2" "github.com/vshn/appcat/v4/pkg/comp-functions/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "reflect" controllerruntime "sigs.k8s.io/controller-runtime" "strings" + "text/template" ) +var rawExpr = "sum_over_time (vector({{.}})[60m:1m])/60" + const billingLabel = "appcat.io/billing" +// TODO Remove the label from workloads when automatic release pipeline is implemented // InjectBillingLabelToService adds billing label to a service (StatefulSet or Deployment). // It uses a kube Object to achieve post provisioning labelling func InjectBillingLabelToService(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result { - log := controllerruntime.LoggerFrom(ctx) - log.Info("Enabling billing for service", "service", comp.GetName()) s := comp.GetWorkloadPodTemplateLabelsManager() s.SetName(comp.GetWorkloadName()) @@ -51,3 +57,100 @@ func InjectBillingLabelToService(ctx context.Context, svc *runtime.ServiceRuntim func getType(myvar interface{}) (res string) { return strings.ToLower(reflect.TypeOf(myvar).Elem().Field(0).Name) } + +// CreateBillingRecord creates a new prometheus rule per each instance namespace +// The rule is skipped for any secondary service such as postgresql instance for nextcloud +// The skipping is based on whether label appuio.io/billing-name is set or not on instance namespace +func CreateBillingRecord(ctx context.Context, svc *runtime.ServiceRuntime, comp InfoGetter) *xfnproto.Result { + log := controllerruntime.LoggerFrom(ctx) + log.Info("Enabling billing for service", "service", comp.GetName()) + + expr, err := getExprFromTemplate(comp.GetInstances()) + if err != nil { + runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + org, err := getOrg(comp.GetName(), svc) + if err != nil { + log.Error(err, "billing not working, cannot get organization", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + controlNS, ok := svc.Config.Data["controlNamespace"] + if !ok { + log.Error(err, "billing not working, control namespace missing", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + disabled, err := isBillingDisabled(controlNS, comp.GetInstanceNamespace(), comp.GetName(), svc) + if err != nil { + log.Error(err, "billing not working, cannot determine if primary service", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", comp.GetName())) + } + + if disabled { + log.Info("secondary service, skipping billing", "service", comp.GetName()) + return runtime.NewNormalResult(fmt.Sprintf("billing disabled for instance %s", comp.GetName())) + } + + p := &v1.PrometheusRule{ + Spec: v1.PrometheusRuleSpec{ + Groups: []v1.RuleGroup{ + { + Name: "appcat-metering-rules", + Rules: []v1.Rule{ + { + Record: "appcat:metering", + Expr: intstr.FromString(expr), + Labels: getLabels(org, comp, svc), + }, + }, + }, + }, + }, + } + p.SetName(comp.GetName() + "-billing") + p.SetNamespace(comp.GetInstanceNamespace()) + kubeName := comp.GetName() + "-billing" + + err = svc.SetDesiredKubeObject(p, kubeName) + + if err != nil { + log.Error(err, "cannot add billing to service, cannot set desired object", "service", comp.GetName()) + return runtime.NewWarningResult(fmt.Sprintf("cannot add billing to service %s", p.GetName())) + } + + return runtime.NewNormalResult(fmt.Sprintf("billing enabled for instance %s", comp.GetName())) +} + +func getLabels(org string, comp InfoGetter, svc *runtime.ServiceRuntime) map[string]string { + labels := map[string]string{ + "label_appcat_vshn_io_claim_name": comp.GetClaimName(), + "label_appcat_vshn_io_claim_namespace": comp.GetClaimNamespace(), + "label_appcat_vshn_io_sla": comp.GetSLA(), + "label_appuio_io_billing_name": comp.GetBillingName(), + "label_appuio_io_organization": org, + } + + so := svc.Config.Data["salesOrder"] + // if appuio managed then add the sales order + if so != "" { + labels["sales_order"] = so + } + return labels +} + +func getExprFromTemplate(i int) (string, error) { + var buf bytes.Buffer + tmpl, err := template.New("billing").Parse(rawExpr) + if err != nil { + return "", err + } + + err = tmpl.Execute(&buf, i) + if err != nil { + return "", err + } + + return buf.String(), err +} diff --git a/pkg/comp-functions/functions/common/instance_namespace.go b/pkg/comp-functions/functions/common/instance_namespace.go index 5f0190623..37a72e251 100644 --- a/pkg/comp-functions/functions/common/instance_namespace.go +++ b/pkg/comp-functions/functions/common/instance_namespace.go @@ -29,6 +29,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names compositionName := comp.GetName() instanceNs := comp.GetInstanceNamespace() claimName, ok := comp.GetLabels()[claimNameLabel] + billingName := comp.GetBillingName() if !ok { return errors.New("no claim name available in composite labels") } @@ -40,7 +41,7 @@ func BootstrapInstanceNs(ctx context.Context, comp Composite, serviceName, names } l.Info("Creating namespace for " + serviceName + " instance") - err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, svc) + err = createInstanceNamespace(serviceName, compositionName, claimNs, instanceNs, namespaceResName, claimName, billingName, svc) if err != nil { return fmt.Errorf("cannot create %s namespace: %w", serviceName, err) } @@ -93,7 +94,7 @@ func createNamespaceObserver(claimNs string, instance string, svc *runtime.Servi } // Create the namespace for the service instance -func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName string, svc *runtime.ServiceRuntime) error { +func createInstanceNamespace(serviceName, compName, claimNamespace, instanceNamespace, namespaceResName, claimName, billingName string, svc *runtime.ServiceRuntime) error { org, err := getOrg(compName, svc) if err != nil { @@ -115,7 +116,7 @@ func createInstanceNamespace(serviceName, compName, claimNamespace, instanceName "appcat.vshn.io/claim-namespace": claimNamespace, "appcat.vshn.io/claim-name": claimName, "appuio.io/no-rbac-creation": "true", - "appuio.io/billing-name": "appcat-" + serviceName, + "appuio.io/billing-name": billingName, "appuio.io/organization": org, }, }, diff --git a/pkg/comp-functions/functions/common/interfaces.go b/pkg/comp-functions/functions/common/interfaces.go index 31ba70c5f..f4ff98b8e 100644 --- a/pkg/comp-functions/functions/common/interfaces.go +++ b/pkg/comp-functions/functions/common/interfaces.go @@ -20,6 +20,9 @@ type InfoGetter interface { GetPDBLabels() map[string]string GetWorkloadPodTemplateLabelsManager() vshnv1.PodTemplateLabelsManager GetWorkloadName() string + GetClaimName() string + GetSLA() string + GetBillingName() string } // InstanceNamespaceInfo provides all the necessary information to create diff --git a/pkg/comp-functions/functions/vshnkeycloak/billing.go b/pkg/comp-functions/functions/vshnkeycloak/billing.go index 0d81ee5d7..66c30e313 100644 --- a/pkg/comp-functions/functions/vshnkeycloak/billing.go +++ b/pkg/comp-functions/functions/vshnkeycloak/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNKeycloak, svc *run return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnmariadb/billing.go b/pkg/comp-functions/functions/vshnmariadb/billing.go index ed50df386..5eda049e2 100644 --- a/pkg/comp-functions/functions/vshnmariadb/billing.go +++ b/pkg/comp-functions/functions/vshnmariadb/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMariaDB, svc *runt return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go index 82d1adda9..fd200c0c0 100644 --- a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go +++ b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go @@ -48,7 +48,7 @@ func DeployMariadb(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.S } l.Info("Bootstrapping instance namespace and rbac rules") - err = common.BootstrapInstanceNs(ctx, comp, "mariadb", comp.GetName()+"-instanceNs", svc) + err = common.BootstrapInstanceNs(ctx, comp, comp.GetServiceName(), comp.GetName()+"-instanceNs", svc) if err != nil { return runtime.NewWarningResult(fmt.Errorf("cannot bootstrap instance namespace: %w", err).Error()) } diff --git a/pkg/comp-functions/functions/vshnminio/billing.go b/pkg/comp-functions/functions/vshnminio/billing.go index feefc28e6..a9a99a349 100644 --- a/pkg/comp-functions/functions/vshnminio/billing.go +++ b/pkg/comp-functions/functions/vshnminio/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNMinio, svc *runtim return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnnextcloud/billing.go b/pkg/comp-functions/functions/vshnnextcloud/billing.go index 0fe1d7249..d5a0f63c2 100644 --- a/pkg/comp-functions/functions/vshnnextcloud/billing.go +++ b/pkg/comp-functions/functions/vshnnextcloud/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNNextcloud, svc *ru return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnpostgres/billing.go b/pkg/comp-functions/functions/vshnpostgres/billing.go index 40efc4708..a56f557d5 100644 --- a/pkg/comp-functions/functions/vshnpostgres/billing.go +++ b/pkg/comp-functions/functions/vshnpostgres/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNPostgreSQL, svc *r return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) } diff --git a/pkg/comp-functions/functions/vshnredis/billing.go b/pkg/comp-functions/functions/vshnredis/billing.go index af57a4681..eb75ac3ea 100644 --- a/pkg/comp-functions/functions/vshnredis/billing.go +++ b/pkg/comp-functions/functions/vshnredis/billing.go @@ -16,5 +16,7 @@ func AddServiceBillingLabel(ctx context.Context, comp *v1.VSHNRedis, svc *runtim return runtime.NewFatalResult(fmt.Errorf("can't get composite: %w", err)) } - return common.InjectBillingLabelToService(ctx, svc, comp) + common.InjectBillingLabelToService(ctx, svc, comp) + + return common.CreateBillingRecord(ctx, svc, comp) }