From 772257b703ab2758cb706f3b5ffd44ef2039cebf Mon Sep 17 00:00:00 2001 From: parth-gr Date: Mon, 22 Jul 2024 20:22:36 +0530 Subject: [PATCH] logrotate: add logrotate functionality for csi 1) Make main container and csi addons container log to a file(dependency on klog) 2) Add a log-rotate sidecar container, so it can rotate the logs 3) Added other volume and volumemounts as needed 4) Added the privileged option for controllerplugin 5) Add resources to the logrotate container 6) make the api change in the merge function Signed-off-by: parth-gr --- internal/controller/driver_controller.go | 160 +++++++++++++++++++++-- internal/utils/csi.go | 39 ++++++ 2 files changed, 185 insertions(+), 14 deletions(-) diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 9ac9e4c4..90745605 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -26,6 +26,7 @@ import ( "reflect" "regexp" "slices" + "strconv" "strings" "github.com/go-logr/logr" @@ -436,6 +437,19 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.LeaderElectionRetryPeriodContainerArg(leaderElectionSpec.RetryPeriod), } + var logRotationContainerArgs []string + var logRotationCsiAddonsSidecarContainerArgs []string + securityContext := &corev1.SecurityContext{Privileged: ptr.To(false)} + // choosing a different approach as need to check two different nil values + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, fmt.Sprintf("csi-%splugin", r.driverType))) + logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, "csi-addons")) + securityContext = &corev1.SecurityContext{ + Privileged: ptr.To(true), + } + } + } deploy.Spec = appsv1.DeploymentSpec{ Replicas: pluginSpec.Replicas, Selector: &appSelector, @@ -461,7 +475,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, - Args: []string{ + SecurityContext: securityContext, + Args: append( + slices.Clone(logRotationContainerArgs), utils.TypeContainerArg(string(r.driverType)), utils.LogLevelContainerArg(logLevel), utils.EndpointContainerArg, @@ -477,7 +493,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsEndpointContainerArg, "", ), - }, + ), Env: []corev1.EnvVar{ utils.PodIpEnvVar, utils.NodeIdEnvVar, @@ -500,6 +516,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.KeysTmpDirVolumeMount, utils.CsiConfigVolumeMount, ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name)) + } + } if r.driver.Spec.Encryption != nil { mounts = append(mounts, utils.KmsConfigVolumeMount) } @@ -612,8 +633,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: append( - slices.Clone(leaderElectionArgs), + slices.Concat(leaderElectionArgs, logRotationCsiAddonsSidecarContainerArgs), utils.LogLevelContainerArg(logLevel), utils.NodeIdContainerArg, utils.PodContainerArg, @@ -631,9 +653,17 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodNameEnvVar, utils.PodNamespaceEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.SocketDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.SocketDirVolumeMount, + } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name)) + } + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -693,7 +723,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), }) } - + // CSI Logrotate Container + if r.driver.Spec.Log.Rotation != nil { + containers = append(containers, r.logRotateContainer(imagePullPolicy, deploy.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}))) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -714,6 +747,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.OidcTokenVolume, utils.CsiConfigVolume, ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName()) + } + } if r.driver.Spec.Encryption != nil { volumes = append( volumes, @@ -757,6 +795,16 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath) forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient + var logRotationContainerArgs []string + var logRotationCsiAddonsSidecarContainerArgs []string + // choosing a different approach as need to check two different nil values + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, fmt.Sprintf("csi-%splugin", r.driverType))) + logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, "csi-addons")) + } + } + daemonSet.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": appName}, @@ -798,7 +846,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }, AllowPrivilegeEscalation: ptr.To(true), }, - Args: []string{ + Args: append( + slices.Clone(logRotationContainerArgs), utils.LogLevelContainerArg(logLevel), utils.TypeContainerArg(string(r.driverType)), utils.NodeServerContainerArg, @@ -812,7 +861,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""), utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""), // TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info - }, + ), Env: []corev1.EnvVar{ utils.PodIpEnvVar, utils.NodeIdEnvVar, @@ -838,6 +887,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name)) + } + } return mounts }), Resources: ptr.Deref( @@ -886,7 +940,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Drop: []corev1.Capability{"All"}, }, }, - Args: []string{ + Args: append( + slices.Clone(logRotationCsiAddonsSidecarContainerArgs), utils.NodeIdContainerArg, utils.LogLevelContainerArg(logLevel), utils.CsiAddonsAddressContainerArg, @@ -895,7 +950,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.NamespaceContainerArg, utils.PodUidContainerArg, utils.StagingPathContainerArg(kubeletDirPath), - }, + ), Ports: []corev1.ContainerPort{ utils.CsiAddonsContainerPort, }, @@ -905,9 +960,17 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodNamespaceEnvVar, utils.PodUidEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.PluginDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.PluginDirVolumeMount, + } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name)) + } + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -946,6 +1009,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }) } } + // CSI Logrotate Container + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + containers = append(containers, r.logRotateContainer(imagePullPolicy, daemonSet.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}))) + } + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -968,6 +1037,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodsMountDirVolume(pluginSpec.KubeletDirPath), utils.RegistrationDirVolume(pluginSpec.KubeletDirPath), ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName()) + } + } if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) { volumes = append( volumes, @@ -1183,6 +1257,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if src.ControllerPlugin != nil { @@ -1238,6 +1315,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if dest.AttachRequired == nil { @@ -1259,3 +1339,55 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { dest.CephFsClientType = src.CephFsClientType } } + +func (r *driverReconcile) logRotateContainer(imagePullPolicy corev1.PullPolicy, csiComponentName string, resources corev1.ResourceRequirements) corev1.Container { + rotation := r.driver.Spec.Log.Rotation + return corev1.Container{ + Name: "log-collector", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{ + "/bin/bash", + "-c", // Command to run + fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, csiComponentName), + }, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName), + utils.LogRotateDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName), + }, + } +} + +var cronLogRotate = ` +echo "Starting the csi-logrotate-sidecar" +PERIODICITY=%s +LOG_MAX_SIZE=%s +ROTATE=%s +CsiLogHostPath=%s +CsiComponentName=%s + +mkdir -p $CsiLogHostPath/logrotate-config/$CsiComponentName +cat < ${CsiLogHostPath}/logrotate-config/${CsiComponentName}/csi +$CsiLogHostPath/log/$CsiComponentName/*.log { + $PERIODICITY + missingok + rotate $ROTATE + compress + copytruncate + notifempty +} +EOF + +echo "File creation container completed" + +LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi +if [ "$LOG_MAX_SIZE" != "0" ]; then + sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE" +fi + +while true; do + logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE" + sleep 15m +done +` diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 24f45dc7..14d078c1 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -33,6 +33,8 @@ const ( pluginDirVolumeName = "plugin-dir" podsMountDirVolumeName = "pods-mount-dir" pluginMountDirVolumeName = "plugin-mount-dir" + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" ) // Ceph CSI common volumes @@ -119,6 +121,26 @@ var EtcSelinuxVolume = corev1.Volume{ }, } +func LogsDirVolume(LogHostPath string) corev1.Volume { + return corev1.Volume{ + Name: logsDirVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: fmt.Sprintf("%s/log/", LogHostPath), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + } +} +func LogRotateDirVolumeName() corev1.Volume { + return corev1.Volume{ + Name: logRotateDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + func KmsConfigVolume(configRef *corev1.LocalObjectReference) corev1.Volume { return corev1.Volume{ VolumeSource: corev1.VolumeSource{ @@ -234,6 +256,18 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{ ReadOnly: true, } +func LogsDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logsDirVolumeName, + MountPath: fmt.Sprintf("%s/log/%s", LogHostPath, CsiComponentName), + } +} +func LogRotateDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logRotateDirVolumeName, + MountPath: fmt.Sprintf("%s/logrotate-config/%s", LogHostPath, CsiComponentName), + } +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{ Name: podsMountDirVolumeName, @@ -331,10 +365,15 @@ var TopologyContainerArg = "--feature-gates=Topology=true" var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true" var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true" var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true" +var LogToStdErrContainerArg = "--logtostderr=false" +var AlsoLogToStdErrContainerArg = "--alsologtostderr=true" func LogLevelContainerArg(level int) string { return fmt.Sprintf("--v=%d", Clamp(level, 0, 5)) } +func LogFileContainerArg(LogHostPath, CsiComponentName, containerName string) string { + return fmt.Sprintf("--log_file=%s/log/%s/%s.log", LogHostPath, CsiComponentName, containerName) +} func TypeContainerArg(t string) string { switch t { case "rbd", "cephfs", "nfs", "controller", "liveness":