diff --git a/charts/cinder-csi-plugin/Chart.yaml b/charts/cinder-csi-plugin/Chart.yaml index 376efab11a..c21b962414 100644 --- a/charts/cinder-csi-plugin/Chart.yaml +++ b/charts/cinder-csi-plugin/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: v1.31.0 description: Cinder CSI Chart for OpenStack name: openstack-cinder-csi -version: 2.31.2 +version: 2.31.3 home: https://github.com/kubernetes/cloud-provider-openstack icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png maintainers: diff --git a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml b/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml index 5b681e4c4d..0942c24dc5 100644 --- a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml +++ b/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml @@ -5,6 +5,9 @@ metadata: spec: attachRequired: true podInfoOnMount: true + {{- if .Values.csi.provisioner.capacity }} + storageCapacity: true + {{- end }} volumeLifecycleModes: - Persistent - Ephemeral diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml index 03cc4f1b2e..691ab242ea 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml @@ -73,6 +73,10 @@ spec: - "--default-fstype=ext4" - "--feature-gates=Topology={{ .Values.csi.provisioner.topology }}" - "--extra-create-metadata" + {{- if .Values.csi.provisioner.capacity }} + - "--enable-capacity" + - "--capacity-ownerref-level=2" + {{- end }} {{- if .Values.csi.provisioner.extraArgs }} {{- with .Values.csi.provisioner.extraArgs }} {{- tpl . $ | trim | nindent 12 }} diff --git a/charts/cinder-csi-plugin/values.yaml b/charts/cinder-csi-plugin/values.yaml index ec87bb341a..7318c858d9 100644 --- a/charts/cinder-csi-plugin/values.yaml +++ b/charts/cinder-csi-plugin/values.yaml @@ -15,6 +15,7 @@ csi: extraEnv: [] provisioner: topology: "true" + capacity: false image: repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.6.2 diff --git a/docs/cinder-csi-plugin/features.md b/docs/cinder-csi-plugin/features.md index b033ff791f..dea1cd5e33 100644 --- a/docs/cinder-csi-plugin/features.md +++ b/docs/cinder-csi-plugin/features.md @@ -5,6 +5,7 @@ - [Plugin Features](#plugin-features) - [Dynamic Provisioning](#dynamic-provisioning) - [Topology](#topology) + - [Storage capacity](#storage-capacity) - [Block Volume](#block-volume) - [Volume Expansion](#volume-expansion) - [Rescan on in-use volume resize](#rescan-on-in-use-volume-resize) @@ -24,7 +25,7 @@ Dynamic Provisioning uses persistence volume claim (PVC) to request the Kubernetes to create the Cinder volume on behalf of user and consumes the volume from inside container. -For usage, refer [sample app](./examples.md#dynamic-volume-provisioning) +For usage, refer [sample app](./examples.md#dynamic-volume-provisioning) ## Topology @@ -35,10 +36,22 @@ This feature enables driver to consider the topology constraints while creating `topology.cinder.csi.openstack.org/zone` : Availability by Zone * `allowedTopologies` can be specified in storage class to restrict the topology of provisioned volumes to specific zones and should be used as replacement of `availability` parameter. * To disable: set `--feature-gates=Topology=false` in external-provisioner (container `csi-provisioner` of `csi-cinder-controllerplugin`). - * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"` + * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"` For usage, refer [sample app](./examples.md#use-topology) +## Storage Capacity + +This feature enables driver to consider the storage capacity constraints while creating the volume. For more info, refer [Capacity Support](https://github.com/kubernetes-csi/external-provisioner/blob/master/README.md#capacity-support) + +The driver will expose the storage capacity limits of the openstack project. +The capacity is calculated based on the quota, which is a sum of all the volumes and snapshots created by the csi project account. + +* Disabled by default +* To enable: set `--enable-capacity` and `--capacity-ownerref-level=2` in external-provisioner (container `csi-provisioner` of `csi-cinder-controllerplugin`), you need to have `StorageCapacity: true` in CSIDriver object. + * If using Helm, it can be enabled by setting `Values.csi.provisioner.capacity: "true"` + * To change frequency of capacity check, set `--capacity-check-interval` in external-provisioner + ## Block Volume Cinder volumes to be exposed inside containers as a block device instead of as a mounted file system. The corresponding CSI feature (CSIBlockVolume) is GA since Kubernetes 1.18. @@ -51,7 +64,7 @@ For usage, refer [sample app](./examples.md#using-block-volume) ## Volume Expansion -Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion. +Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion. The same should be supported by underlying OpenStack Cloud to avail the feature. * As of kubernetes v1.16, Volume Expansion is a beta feature and enabled by default. @@ -81,7 +94,7 @@ Two different Kubernetes features allow volumes to follow the Pod's lifecycle: C This feature allows CSI volumes to be directly embedded in the Pod specification instead of a PersistentVolume. Volumes specified in this way are ephemeral and do not persist across Pod restarts. -* As of Kubernetes v1.16 this feature is beta so enabled by default. +* As of Kubernetes v1.16 this feature is beta so enabled by default. * To enable this feature for CSI Driver, `volumeLifecycleModes` needs to be specified in [CSIDriver](../../manifests/cinder-csi-plugin/csi-cinder-driver.yaml) object. The driver can run in `Persistent` mode, `Ephemeral` or in both modes. * `podInfoOnMount` must be `true` to use this feature. * For usage, refer [sample app](./examples.md#deploy-app-using-inline-volumes) diff --git a/docs/cinder-csi-plugin/using-cinder-csi-plugin.md b/docs/cinder-csi-plugin/using-cinder-csi-plugin.md index 3e1a3c3e03..f2c07d1ceb 100644 --- a/docs/cinder-csi-plugin/using-cinder-csi-plugin.md +++ b/docs/cinder-csi-plugin/using-cinder-csi-plugin.md @@ -248,6 +248,7 @@ helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugi * [Dynamic Provisioning](./features.md#dynamic-provisioning) * [Topology](./features.md#topology) +* [Storage capacity](./features.md#storage-capacity) * [Raw Block Volume](./features.md#block-volume) * [Volume Expansion](./features.md#volume-expansion) * [Volume Cloning](./features.md#volume-cloning) diff --git a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml index 8b871d7988..cdb8aa9713 100644 --- a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml +++ b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml @@ -45,6 +45,8 @@ spec: - "--timeout=3m" - "--default-fstype=ext4" - "--feature-gates=Topology=true" + - "--enable-capacity" + - "--capacity-ownerref-level=2" - "--extra-create-metadata" - "--leader-election=true" env: diff --git a/manifests/cinder-csi-plugin/csi-cinder-driver.yaml b/manifests/cinder-csi-plugin/csi-cinder-driver.yaml index 5b681e4c4d..7f5555b3d9 100644 --- a/manifests/cinder-csi-plugin/csi-cinder-driver.yaml +++ b/manifests/cinder-csi-plugin/csi-cinder-driver.yaml @@ -5,6 +5,7 @@ metadata: spec: attachRequired: true podInfoOnMount: true + storageCapacity: true volumeLifecycleModes: - Persistent - Ephemeral diff --git a/pkg/csi/cinder/controllerserver.go b/pkg/csi/cinder/controllerserver.go index c69171aada..dd756fa1de 100644 --- a/pkg/csi/cinder/controllerserver.go +++ b/pkg/csi/cinder/controllerserver.go @@ -33,6 +33,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" + corev1 "k8s.io/api/core/v1" "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" "k8s.io/cloud-provider-openstack/pkg/util" cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" @@ -949,7 +950,28 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req } func (cs *controllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - return nil, status.Error(codes.Unimplemented, "GetCapacity is not yet implemented") + klog.V(4).Infof("GetCapacity: called with args %+v", protosanitizer.StripSecrets(*req)) + + topology := req.GetAccessibleTopology() + if topology == nil { + return &csi.GetCapacityResponse{}, nil + } + + region := topology.GetSegments()[corev1.LabelTopologyRegion] + + cloud, cloudExist := cs.Clouds[region] + if !cloudExist { + return nil, status.Error(codes.InvalidArgument, "[GetCapacity] specified cloud undefined") + } + + availableCapacity, err := cloud.GetFreeQuotaStorageSpace() + if err != nil { + return nil, status.Errorf(codes.Internal, "[GetCapacity]: failed with error %v", err) + } + + return &csi.GetCapacityResponse{ + AvailableCapacity: int64(availableCapacity * 1024 * 1024 * 1024), + }, nil } func (cs *controllerServer) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { diff --git a/pkg/csi/cinder/driver.go b/pkg/csi/cinder/driver.go index 1040a09bdb..123e6c11a0 100644 --- a/pkg/csi/cinder/driver.go +++ b/pkg/csi/cinder/driver.go @@ -99,6 +99,7 @@ func NewDriver(o *DriverOpts) *Driver { csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + csi.ControllerServiceCapability_RPC_GET_CAPACITY, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, diff --git a/pkg/csi/cinder/openstack/openstack.go b/pkg/csi/cinder/openstack/openstack.go index 8b4a028452..ce86efb37b 100644 --- a/pkg/csi/cinder/openstack/openstack.go +++ b/pkg/csi/cinder/openstack/openstack.go @@ -70,6 +70,7 @@ type IOpenStack interface { GetInstanceByID(instanceID string) (*servers.Server, error) ExpandVolume(volumeID string, status string, size int) error GetMaxVolLimit() int64 + GetFreeQuotaStorageSpace() (int, error) GetMetadataOpts() metadata.Opts GetBlockStorageOpts() BlockStorageOpts } diff --git a/pkg/csi/cinder/openstack/openstack_mock.go b/pkg/csi/cinder/openstack/openstack_mock.go index 53263e1f8e..fa7e18c221 100644 --- a/pkg/csi/cinder/openstack/openstack_mock.go +++ b/pkg/csi/cinder/openstack/openstack_mock.go @@ -460,6 +460,10 @@ func (_m *OpenStackMock) GetMaxVolLimit() int64 { return 256 } +func (_m *OpenStackMock) GetFreeQuotaStorageSpace() (int, error) { + return 100, nil +} + func (_m *OpenStackMock) BackupsAreEnabled() (bool, error) { return true, nil } diff --git a/pkg/csi/cinder/openstack/openstack_volumes.go b/pkg/csi/cinder/openstack/openstack_volumes.go index 5ac62fc73f..bb4515f1a6 100644 --- a/pkg/csi/cinder/openstack/openstack_volumes.go +++ b/pkg/csi/cinder/openstack/openstack_volumes.go @@ -23,6 +23,7 @@ import ( "time" "github.com/gophercloud/gophercloud/v2/openstack" + "github.com/gophercloud/gophercloud/v2/openstack/blockstorage/v3/limits" "github.com/gophercloud/gophercloud/v2/openstack/blockstorage/v3/volumes" "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/volumeattach" "github.com/gophercloud/gophercloud/v2/pagination" @@ -396,6 +397,22 @@ func (os *OpenStack) GetMaxVolLimit() int64 { return defaultMaxVolAttachLimit } +// GetFreeQuotaStorageSpace returns the tenant quota capacity of the block storage, in GB +func (os *OpenStack) GetFreeQuotaStorageSpace() (int, error) { + mc := metrics.NewMetricContext("limits", "get") + + res, err := limits.Get(context.TODO(), os.blockstorage).Extract() + if mc.ObserveRequest(err) != nil { + return 0, err + } + + capacity := res.Absolute.MaxTotalVolumeGigabytes - res.Absolute.TotalGigabytesUsed + if capacity < 0 { + capacity = 0 + } + return capacity, nil +} + // diskIsAttached queries if a volume is attached to a compute instance func (os *OpenStack) diskIsAttached(instanceID, volumeID string) (bool, error) { volume, err := os.GetVolume(volumeID) diff --git a/tests/sanity/cinder/fakecloud.go b/tests/sanity/cinder/fakecloud.go index 53c0dcba4f..f3e87d34fe 100644 --- a/tests/sanity/cinder/fakecloud.go +++ b/tests/sanity/cinder/fakecloud.go @@ -332,6 +332,10 @@ func (cloud *cloud) GetMaxVolLimit() int64 { return 256 } +func (cloud *cloud) GetFreeQuotaStorageSpace() (int, error) { + return 100, nil +} + func (cloud *cloud) GetMetadataOpts() metadata.Opts { var m metadata.Opts m.SearchOrder = fmt.Sprintf("%s,%s", "configDrive", "metadataService")