Skip to content

Commit

Permalink
logrotate: add logrotate functionality for csi
Browse files Browse the repository at this point in the history 
1) Make main container and csi addons container
   log to a file(dependency on klog)

2) Add a log-rotate sidecar container,
   so it can rotate the logs

3) Added other volume and volumemounts as needed

4) Added the privileged option for controllerplugin

5) Add resources to the logrotate container

6) make the api change in the merge function

Signed-off-by: parth-gr <[email protected]>
  • Loading branch information
@parth-gr
parth-gr committed Jul 25, 2024
1 parent badf2d6 commit 772257b
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 14 deletions.
160 changes: 146 additions & 14 deletions internal/controller/driver_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"reflect"
"regexp"
"slices"
"strconv"
"strings"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -436,6 +437,19 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.LeaderElectionRetryPeriodContainerArg(leaderElectionSpec.RetryPeriod),
}

var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
securityContext := &corev1.SecurityContext{Privileged: ptr.To(false)}
// choosing a different approach as need to check two different nil values
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, fmt.Sprintf("csi-%splugin", r.driverType)))
logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, "csi-addons"))
securityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
}
}
deploy.Spec = appsv1.DeploymentSpec{
Replicas: pluginSpec.Replicas,
Selector: &appSelector,
Expand All @@ -461,7 +475,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: fmt.Sprintf("csi-%splugin", r.driverType),
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Args: []string{
SecurityContext: securityContext,
Args: append(
slices.Clone(logRotationContainerArgs),
utils.TypeContainerArg(string(r.driverType)),
utils.LogLevelContainerArg(logLevel),
utils.EndpointContainerArg,
Expand All @@ -477,7 +493,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.CsiAddonsEndpointContainerArg,
"",
),
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -500,6 +516,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.KeysTmpDirVolumeMount,
utils.CsiConfigVolumeMount,
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name))
}
}
if r.driver.Spec.Encryption != nil {
mounts = append(mounts, utils.KmsConfigVolumeMount)
}
Expand Down Expand Up @@ -612,8 +633,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: "csi-addons",
Image: r.images["addons"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: securityContext,
Args: append(
slices.Clone(leaderElectionArgs),
slices.Concat(leaderElectionArgs, logRotationCsiAddonsSidecarContainerArgs),
utils.LogLevelContainerArg(logLevel),
utils.NodeIdContainerArg,
utils.PodContainerArg,
Expand All @@ -631,9 +653,17 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.PodNameEnvVar,
utils.PodNamespaceEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.SocketDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.SocketDirVolumeMount,
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -693,7 +723,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
),
})
}

// CSI Logrotate Container
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logRotateContainer(imagePullPolicy, deploy.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})))
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -714,6 +747,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.OidcTokenVolume,
utils.CsiConfigVolume,
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName())
}
}
if r.driver.Spec.Encryption != nil {
volumes = append(
volumes,
Expand Down Expand Up @@ -757,6 +795,16 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath)
forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient

var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
// choosing a different approach as need to check two different nil values
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, fmt.Sprintf("csi-%splugin", r.driverType)))
logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, "csi-addons"))
}
}

daemonSet.Spec = appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": appName},
Expand Down Expand Up @@ -798,7 +846,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
},
AllowPrivilegeEscalation: ptr.To(true),
},
Args: []string{
Args: append(
slices.Clone(logRotationContainerArgs),
utils.LogLevelContainerArg(logLevel),
utils.TypeContainerArg(string(r.driverType)),
utils.NodeServerContainerArg,
Expand All @@ -812,7 +861,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""),
utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""),
// TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -838,6 +887,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
if r.isRdbDriver() {
mounts = append(mounts, utils.OidcTokenVolumeMount)
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
Expand Down Expand Up @@ -886,7 +940,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
Drop: []corev1.Capability{"All"},
},
},
Args: []string{
Args: append(
slices.Clone(logRotationCsiAddonsSidecarContainerArgs),
utils.NodeIdContainerArg,
utils.LogLevelContainerArg(logLevel),
utils.CsiAddonsAddressContainerArg,
Expand All @@ -895,7 +950,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
},
),
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
},
Expand All @@ -905,9 +960,17 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodNamespaceEnvVar,
utils.PodUidEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.PluginDirVolumeMount,
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -946,6 +1009,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
})
}
}
// CSI Logrotate Container
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logRotateContainer(imagePullPolicy, daemonSet.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})))
}
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -968,6 +1037,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodsMountDirVolume(pluginSpec.KubeletDirPath),
utils.RegistrationDirVolume(pluginSpec.KubeletDirPath),
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName())
}
}
if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) {
volumes = append(
volumes,
Expand Down Expand Up @@ -1183,6 +1257,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if src.ControllerPlugin != nil {
Expand Down Expand Up @@ -1238,6 +1315,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if dest.AttachRequired == nil {
Expand All @@ -1259,3 +1339,55 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
dest.CephFsClientType = src.CephFsClientType
}
}

func (r *driverReconcile) logRotateContainer(imagePullPolicy corev1.PullPolicy, csiComponentName string, resources corev1.ResourceRequirements) corev1.Container {
rotation := r.driver.Spec.Log.Rotation
return corev1.Container{
Name: "log-collector",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: resources,
Command: []string{
"/bin/bash",
"-c", // Command to run
fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, csiComponentName),
},
VolumeMounts: []corev1.VolumeMount{
utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
utils.LogRotateDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
},
}
}

var cronLogRotate = `
echo "Starting the csi-logrotate-sidecar"
PERIODICITY=%s
LOG_MAX_SIZE=%s
ROTATE=%s
CsiLogHostPath=%s
CsiComponentName=%s
mkdir -p $CsiLogHostPath/logrotate-config/$CsiComponentName
cat <<EOF > ${CsiLogHostPath}/logrotate-config/${CsiComponentName}/csi
$CsiLogHostPath/log/$CsiComponentName/*.log {
$PERIODICITY
missingok
rotate $ROTATE
compress
copytruncate
notifempty
}
EOF
echo "File creation container completed"
LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi
while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
`
39 changes: 39 additions & 0 deletions internal/utils/csi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ const (
pluginDirVolumeName = "plugin-dir"
podsMountDirVolumeName = "pods-mount-dir"
pluginMountDirVolumeName = "plugin-mount-dir"
logsDirVolumeName = "logs-dir"
logRotateDirVolumeName = "log-rotate-dir"
)

// Ceph CSI common volumes
Expand Down Expand Up @@ -119,6 +121,26 @@ var EtcSelinuxVolume = corev1.Volume{
},
}

func LogsDirVolume(LogHostPath string) corev1.Volume {
return corev1.Volume{
Name: logsDirVolumeName,
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: fmt.Sprintf("%s/log/", LogHostPath),
Type: ptr.To(corev1.HostPathDirectoryOrCreate),
},
},
}
}
func LogRotateDirVolumeName() corev1.Volume {
return corev1.Volume{
Name: logRotateDirVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}
}

func KmsConfigVolume(configRef *corev1.LocalObjectReference) corev1.Volume {
return corev1.Volume{
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -234,6 +256,18 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{
ReadOnly: true,
}

func LogsDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: logsDirVolumeName,
MountPath: fmt.Sprintf("%s/log/%s", LogHostPath, CsiComponentName),
}
}
func LogRotateDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: logRotateDirVolumeName,
MountPath: fmt.Sprintf("%s/logrotate-config/%s", LogHostPath, CsiComponentName),
}
}
func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: podsMountDirVolumeName,
Expand Down Expand Up @@ -331,10 +365,15 @@ var TopologyContainerArg = "--feature-gates=Topology=true"
var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true"
var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true"
var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true"
var LogToStdErrContainerArg = "--logtostderr=false"
var AlsoLogToStdErrContainerArg = "--alsologtostderr=true"

func LogLevelContainerArg(level int) string {
return fmt.Sprintf("--v=%d", Clamp(level, 0, 5))
}
func LogFileContainerArg(LogHostPath, CsiComponentName, containerName string) string {
return fmt.Sprintf("--log_file=%s/log/%s/%s.log", LogHostPath, CsiComponentName, containerName)
}
func TypeContainerArg(t string) string {
switch t {
case "rbd", "cephfs", "nfs", "controller", "liveness":
Expand Down

0 comments on commit 772257b

Please sign in to comment.