diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 9ac9e4c4..90745605 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -26,6 +26,7 @@ import ( "reflect" "regexp" "slices" + "strconv" "strings" "github.com/go-logr/logr" @@ -436,6 +437,19 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.LeaderElectionRetryPeriodContainerArg(leaderElectionSpec.RetryPeriod), } + var logRotationContainerArgs []string + var logRotationCsiAddonsSidecarContainerArgs []string + securityContext := &corev1.SecurityContext{Privileged: ptr.To(false)} + // choosing a different approach as need to check two different nil values + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, fmt.Sprintf("csi-%splugin", r.driverType))) + logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, "csi-addons")) + securityContext = &corev1.SecurityContext{ + Privileged: ptr.To(true), + } + } + } deploy.Spec = appsv1.DeploymentSpec{ Replicas: pluginSpec.Replicas, Selector: &appSelector, @@ -461,7 +475,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, - Args: []string{ + SecurityContext: securityContext, + Args: append( + slices.Clone(logRotationContainerArgs), utils.TypeContainerArg(string(r.driverType)), utils.LogLevelContainerArg(logLevel), utils.EndpointContainerArg, @@ -477,7 +493,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsEndpointContainerArg, "", ), - }, + ), Env: []corev1.EnvVar{ utils.PodIpEnvVar, utils.NodeIdEnvVar, @@ -500,6 +516,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.KeysTmpDirVolumeMount, utils.CsiConfigVolumeMount, ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name)) + } + } if r.driver.Spec.Encryption != nil { mounts = append(mounts, utils.KmsConfigVolumeMount) } @@ -612,8 +633,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: append( - slices.Clone(leaderElectionArgs), + slices.Concat(leaderElectionArgs, logRotationCsiAddonsSidecarContainerArgs), utils.LogLevelContainerArg(logLevel), utils.NodeIdContainerArg, utils.PodContainerArg, @@ -631,9 +653,17 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodNameEnvVar, utils.PodNamespaceEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.SocketDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.SocketDirVolumeMount, + } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name)) + } + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -693,7 +723,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), }) } - + // CSI Logrotate Container + if r.driver.Spec.Log.Rotation != nil { + containers = append(containers, r.logRotateContainer(imagePullPolicy, deploy.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}))) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -714,6 +747,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.OidcTokenVolume, utils.CsiConfigVolume, ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName()) + } + } if r.driver.Spec.Encryption != nil { volumes = append( volumes, @@ -757,6 +795,16 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath) forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient + var logRotationContainerArgs []string + var logRotationCsiAddonsSidecarContainerArgs []string + // choosing a different approach as need to check two different nil values + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, fmt.Sprintf("csi-%splugin", r.driverType))) + logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, "csi-addons")) + } + } + daemonSet.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": appName}, @@ -798,7 +846,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }, AllowPrivilegeEscalation: ptr.To(true), }, - Args: []string{ + Args: append( + slices.Clone(logRotationContainerArgs), utils.LogLevelContainerArg(logLevel), utils.TypeContainerArg(string(r.driverType)), utils.NodeServerContainerArg, @@ -812,7 +861,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""), utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""), // TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info - }, + ), Env: []corev1.EnvVar{ utils.PodIpEnvVar, utils.NodeIdEnvVar, @@ -838,6 +887,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name)) + } + } return mounts }), Resources: ptr.Deref( @@ -886,7 +940,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Drop: []corev1.Capability{"All"}, }, }, - Args: []string{ + Args: append( + slices.Clone(logRotationCsiAddonsSidecarContainerArgs), utils.NodeIdContainerArg, utils.LogLevelContainerArg(logLevel), utils.CsiAddonsAddressContainerArg, @@ -895,7 +950,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.NamespaceContainerArg, utils.PodUidContainerArg, utils.StagingPathContainerArg(kubeletDirPath), - }, + ), Ports: []corev1.ContainerPort{ utils.CsiAddonsContainerPort, }, @@ -905,9 +960,17 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodNamespaceEnvVar, utils.PodUidEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.PluginDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.PluginDirVolumeMount, + } + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name)) + } + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -946,6 +1009,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }) } } + // CSI Logrotate Container + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + containers = append(containers, r.logRotateContainer(imagePullPolicy, daemonSet.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}))) + } + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -968,6 +1037,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodsMountDirVolume(pluginSpec.KubeletDirPath), utils.RegistrationDirVolume(pluginSpec.KubeletDirPath), ) + if r.driver.Spec.Log != nil { + if r.driver.Spec.Log.Rotation != nil { + volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName()) + } + } if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) { volumes = append( volumes, @@ -1183,6 +1257,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if src.ControllerPlugin != nil { @@ -1238,6 +1315,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if dest.AttachRequired == nil { @@ -1259,3 +1339,55 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { dest.CephFsClientType = src.CephFsClientType } } + +func (r *driverReconcile) logRotateContainer(imagePullPolicy corev1.PullPolicy, csiComponentName string, resources corev1.ResourceRequirements) corev1.Container { + rotation := r.driver.Spec.Log.Rotation + return corev1.Container{ + Name: "log-collector", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{ + "/bin/bash", + "-c", // Command to run + fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, csiComponentName), + }, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName), + utils.LogRotateDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName), + }, + } +} + +var cronLogRotate = ` +echo "Starting the csi-logrotate-sidecar" +PERIODICITY=%s +LOG_MAX_SIZE=%s +ROTATE=%s +CsiLogHostPath=%s +CsiComponentName=%s + +mkdir -p $CsiLogHostPath/logrotate-config/$CsiComponentName +cat < ${CsiLogHostPath}/logrotate-config/${CsiComponentName}/csi +$CsiLogHostPath/log/$CsiComponentName/*.log { + $PERIODICITY + missingok + rotate $ROTATE + compress + copytruncate + notifempty +} +EOF + +echo "File creation container completed" + +LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi +if [ "$LOG_MAX_SIZE" != "0" ]; then + sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE" +fi + +while true; do + logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE" + sleep 15m +done +` diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 24f45dc7..14d078c1 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -33,6 +33,8 @@ const ( pluginDirVolumeName = "plugin-dir" podsMountDirVolumeName = "pods-mount-dir" pluginMountDirVolumeName = "plugin-mount-dir" + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" ) // Ceph CSI common volumes @@ -119,6 +121,26 @@ var EtcSelinuxVolume = corev1.Volume{ }, } +func LogsDirVolume(LogHostPath string) corev1.Volume { + return corev1.Volume{ + Name: logsDirVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: fmt.Sprintf("%s/log/", LogHostPath), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + } +} +func LogRotateDirVolumeName() corev1.Volume { + return corev1.Volume{ + Name: logRotateDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + func KmsConfigVolume(configRef *corev1.LocalObjectReference) corev1.Volume { return corev1.Volume{ VolumeSource: corev1.VolumeSource{ @@ -234,6 +256,18 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{ ReadOnly: true, } +func LogsDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logsDirVolumeName, + MountPath: fmt.Sprintf("%s/log/%s", LogHostPath, CsiComponentName), + } +} +func LogRotateDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logRotateDirVolumeName, + MountPath: fmt.Sprintf("%s/logrotate-config/%s", LogHostPath, CsiComponentName), + } +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{ Name: podsMountDirVolumeName, @@ -331,10 +365,15 @@ var TopologyContainerArg = "--feature-gates=Topology=true" var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true" var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true" var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true" +var LogToStdErrContainerArg = "--logtostderr=false" +var AlsoLogToStdErrContainerArg = "--alsologtostderr=true" func LogLevelContainerArg(level int) string { return fmt.Sprintf("--v=%d", Clamp(level, 0, 5)) } +func LogFileContainerArg(LogHostPath, CsiComponentName, containerName string) string { + return fmt.Sprintf("--log_file=%s/log/%s/%s.log", LogHostPath, CsiComponentName, containerName) +} func TypeContainerArg(t string) string { switch t { case "rbd", "cephfs", "nfs", "controller", "liveness":