Skip to content

Commit

Permalink
logrotate: add logrotate functionality for csi
Browse files Browse the repository at this point in the history 
1) Make main container and csi addons container
   log to a file(dependency on klog)

2) Add a log-rotate sidecar container,
   so it can rotate the logs

3) Added other volume and volumemounts as needed

4) Added the privileged option for controllerplugin

5) Add resources to the logrotate container

6) make the api change in the merge function

Signed-off-by: parth-gr <[email protected]>
  • Loading branch information
@parth-gr
parth-gr committed Jul 25, 2024
1 parent badf2d6 commit 221e980
Show file tree
Hide file tree
Showing 2 changed files with 209 additions and 22 deletions.
199 changes: 177 additions & 22 deletions internal/controller/driver_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"reflect"
"regexp"
"slices"
"strconv"
"strings"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -436,6 +437,30 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.LeaderElectionRetryPeriodContainerArg(leaderElectionSpec.RetryPeriod),
}

logRotator := &csiv1a1.LogRotationSpec{}
if r.driver.Spec.Log != nil {
logRotator = r.driver.Spec.Log.Rotation
}
var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
securityContext := &corev1.SecurityContext{Privileged: ptr.To(false)}
// choosing a different approach as need to check nil values
if logRotator != nil {
logRotationContainerArgs = []string{
utils.LogToStdErrContainerArg,
utils.AlsoLogToStdErrContainerArg,
utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)),
}
logRotationCsiAddonsSidecarContainerArgs = []string{
utils.LogToStdErrContainerArg,
utils.AlsoLogToStdErrContainerArg,
utils.LogFileContainerArg("csi-addons"),
}
securityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
}

deploy.Spec = appsv1.DeploymentSpec{
Replicas: pluginSpec.Replicas,
Selector: &appSelector,
Expand All @@ -461,7 +486,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: fmt.Sprintf("csi-%splugin", r.driverType),
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Args: []string{
SecurityContext: securityContext,
Args: append(
slices.Clone(logRotationContainerArgs),
utils.TypeContainerArg(string(r.driverType)),
utils.LogLevelContainerArg(logLevel),
utils.EndpointContainerArg,
Expand All @@ -477,7 +504,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.CsiAddonsEndpointContainerArg,
"",
),
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -500,6 +527,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.KeysTmpDirVolumeMount,
utils.CsiConfigVolumeMount,
)
if logRotator != nil {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
if r.driver.Spec.Encryption != nil {
mounts = append(mounts, utils.KmsConfigVolumeMount)
}
Expand Down Expand Up @@ -612,15 +642,19 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: "csi-addons",
Image: r.images["addons"],
ImagePullPolicy: imagePullPolicy,
Args: append(
slices.Clone(leaderElectionArgs),
utils.LogLevelContainerArg(logLevel),
utils.NodeIdContainerArg,
utils.PodContainerArg,
utils.PodUidContainerArg,
utils.CsiAddonsAddressContainerArg,
utils.ControllerPortContainerArg,
utils.NamespaceContainerArg,
SecurityContext: securityContext,
Args: slices.Concat(
leaderElectionArgs,
logRotationCsiAddonsSidecarContainerArgs,
[]string{
utils.LogLevelContainerArg(logLevel),
utils.NodeIdContainerArg,
utils.PodContainerArg,
utils.PodUidContainerArg,
utils.CsiAddonsAddressContainerArg,
utils.ControllerPortContainerArg,
utils.NamespaceContainerArg,
},
),
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
Expand All @@ -631,9 +665,15 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.PodNameEnvVar,
utils.PodNamespaceEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.SocketDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.SocketDirVolumeMount,
}
if logRotator != nil {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -693,7 +733,26 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
),
})
}

// CSI Logrotate Container
if logRotator != nil {
rotation := r.driver.Spec.Log.Rotation
resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})
containers = append(containers, corev1.Container{
Name: "log-collector",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: resources,
Command: []string{
"/bin/bash",
"-c", // Command to run
fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, deploy.Name),
},
VolumeMounts: []corev1.VolumeMount{
utils.LogsDirVolumeMount,
utils.LogRotateDirVolumeMount,
},
})
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -714,6 +773,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.OidcTokenVolume,
utils.CsiConfigVolume,
)
if logRotator != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name), utils.LogRotateDirVolumeName)
}
if r.driver.Spec.Encryption != nil {
volumes = append(
volumes,
Expand Down Expand Up @@ -757,6 +819,26 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath)
forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient

logRotator := &csiv1a1.LogRotationSpec{}
if r.driver.Spec.Log != nil {
logRotator = r.driver.Spec.Log.Rotation
}
var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
// choosing a different approach as need to check nil values
if logRotator != nil {
logRotationContainerArgs = []string{
utils.LogToStdErrContainerArg,
utils.AlsoLogToStdErrContainerArg,
utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)),
}
logRotationCsiAddonsSidecarContainerArgs = []string{
utils.LogToStdErrContainerArg,
utils.AlsoLogToStdErrContainerArg,
utils.LogFileContainerArg("csi-addons"),
}
}

daemonSet.Spec = appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": appName},
Expand Down Expand Up @@ -798,7 +880,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
},
AllowPrivilegeEscalation: ptr.To(true),
},
Args: []string{
Args: append(
slices.Clone(logRotationContainerArgs),
utils.LogLevelContainerArg(logLevel),
utils.TypeContainerArg(string(r.driverType)),
utils.NodeServerContainerArg,
Expand All @@ -812,7 +895,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""),
utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""),
// TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -838,6 +921,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
if r.isRdbDriver() {
mounts = append(mounts, utils.OidcTokenVolumeMount)
}
if logRotator != nil {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
return mounts
}),
Resources: ptr.Deref(
Expand Down Expand Up @@ -886,7 +972,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
Drop: []corev1.Capability{"All"},
},
},
Args: []string{
Args: append(
slices.Clone(logRotationCsiAddonsSidecarContainerArgs),
utils.NodeIdContainerArg,
utils.LogLevelContainerArg(logLevel),
utils.CsiAddonsAddressContainerArg,
Expand All @@ -895,7 +982,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
},
),
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
},
Expand All @@ -905,9 +992,15 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodNamespaceEnvVar,
utils.PodUidEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.PluginDirVolumeMount,
}
if logRotator != nil {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -946,6 +1039,26 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
})
}
}
// CSI Logrotate Container
if logRotator != nil {
rotation := r.driver.Spec.Log.Rotation
resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})
containers = append(containers, corev1.Container{
Name: "log-collector",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: resources,
Command: []string{
"/bin/bash",
"-c", // Command to run
fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, daemonSet.Name),
},
VolumeMounts: []corev1.VolumeMount{
utils.LogsDirVolumeMount,
utils.LogRotateDirVolumeMount,
},
})
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -968,6 +1081,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodsMountDirVolume(pluginSpec.KubeletDirPath),
utils.RegistrationDirVolume(pluginSpec.KubeletDirPath),
)
if logRotator != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name), utils.LogRotateDirVolumeName)
}
if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) {
volumes = append(
volumes,
Expand Down Expand Up @@ -1183,6 +1299,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if src.ControllerPlugin != nil {
Expand Down Expand Up @@ -1238,6 +1357,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if dest.AttachRequired == nil {
Expand All @@ -1259,3 +1381,36 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
dest.CephFsClientType = src.CephFsClientType
}
}

var cronLogRotate = `
echo "Starting the csi-logrotate-sidecar"
PERIODICITY=%s
LOG_MAX_SIZE=%s
ROTATE=%s
CsiLogHostPath=%s
CsiComponentName=%s
mkdir -p logrotate-config
cat <<EOF > /logrotate-config/csi
$CsiLogHostPath/$CsiComponentName/*.log {
$PERIODICITY
missingok
rotate $ROTATE
compress
copytruncate
notifempty
}
EOF
echo "File creation container completed"
LOG_ROTATE_CEPH_CSI_FILE=/logrotate-config/csi
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi
while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
`
Loading

0 comments on commit 221e980

Please sign in to comment.