Skip to content

Commit

Permalink
logrotate: add logrotate functionality for csi
Browse files Browse the repository at this point in the history 
1) Make main container and csi addons container
   log to a file(dependency on klog)

2) Add a log-rotate sidecar container,
   so it can rotate the logs

3) Added other volume and volumemounts as needed

4) Added the privileged option for controllerplugin

5) Add resources to the logrotate container

6) make the api change in the merge function

Signed-off-by: parth-gr <[email protected]>
  • Loading branch information
@parth-gr
parth-gr committed Jul 24, 2024
1 parent badf2d6 commit d1764c6
Show file tree
Hide file tree
Showing 3 changed files with 217 additions and 14 deletions.
160 changes: 146 additions & 14 deletions internal/controller/driver_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"reflect"
"regexp"
"slices"
"strconv"
"strings"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -436,6 +437,19 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.LeaderElectionRetryPeriodContainerArg(leaderElectionSpec.RetryPeriod),
}

var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
securityContext := &corev1.SecurityContext{Privileged: ptr.To(false)}
// choosing a different approach as need to check two differnt nil values

Check failure on line 443 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / codespell 

differnt ==> different

Check failure on line 443 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

`differnt` is a misspelling of `different` (misspell)

Check failure on line 443 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / misspell 

[misspell] reported by reviewdog 🐶
"differnt" is a misspelling of "different"

Raw Output:
./internal/controller/driver_controller.go:443:56: "differnt" is a misspelling of "different"
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, fmt.Sprintf("csi-%splugin", r.driverType)))
logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, "csi-addons"))
securityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
}
}
deploy.Spec = appsv1.DeploymentSpec{
Replicas: pluginSpec.Replicas,
Selector: &appSelector,
Expand All @@ -461,7 +475,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: fmt.Sprintf("csi-%splugin", r.driverType),
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Args: []string{
SecurityContext: securityContext,
Args: append(
slices.Clone(logRotationContainerArgs),
utils.TypeContainerArg(string(r.driverType)),
utils.LogLevelContainerArg(logLevel),
utils.EndpointContainerArg,
Expand All @@ -477,7 +493,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.CsiAddonsEndpointContainerArg,
"",
),
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -500,6 +516,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.KeysTmpDirVolumeMount,
utils.CsiConfigVolumeMount,
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name))
}
}
if r.driver.Spec.Encryption != nil {
mounts = append(mounts, utils.KmsConfigVolumeMount)
}
Expand Down Expand Up @@ -612,8 +633,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: "csi-addons",
Image: r.images["addons"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: securityContext,
Args: append(
slices.Clone(leaderElectionArgs),
slices.Concat(leaderElectionArgs, logRotationCsiAddonsSidecarContainerArgs),
utils.LogLevelContainerArg(logLevel),
utils.NodeIdContainerArg,
utils.PodContainerArg,
Expand All @@ -631,9 +653,17 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.PodNameEnvVar,
utils.PodNamespaceEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.SocketDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.SocketDirVolumeMount,
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -693,7 +723,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
),
})
}

// CSI Logrotate Container
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logRotateContainer(imagePullPolicy, deploy.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})))
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -714,6 +747,11 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.OidcTokenVolume,
utils.CsiConfigVolume,
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName())
}
}
if r.driver.Spec.Encryption != nil {
volumes = append(
volumes,
Expand Down Expand Up @@ -757,6 +795,16 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath)
forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient

var logRotationContainerArgs []string
var logRotationCsiAddonsSidecarContainerArgs []string
// choosing a different approach as need to check two differnt nil values

Check failure on line 800 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / codespell 

differnt ==> different

Check failure on line 800 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

`differnt` is a misspelling of `different` (misspell)

Check failure on line 800 in internal/controller/driver_controller.go

View workflow job for this annotation

GitHub Actions / misspell 

[misspell] reported by reviewdog 🐶
"differnt" is a misspelling of "different"

Raw Output:
./internal/controller/driver_controller.go:800:56: "differnt" is a misspelling of "different"
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
logRotationContainerArgs = append(logRotationContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, fmt.Sprintf("csi-%splugin", r.driverType)))
logRotationCsiAddonsSidecarContainerArgs = append(logRotationCsiAddonsSidecarContainerArgs, utils.LogToStdErrContainerArg, utils.AlsoLogToStdErrContainerArg, utils.LogFileContainerArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, "csi-addons"))
}
}

daemonSet.Spec = appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": appName},
Expand Down Expand Up @@ -798,7 +846,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
},
AllowPrivilegeEscalation: ptr.To(true),
},
Args: []string{
Args: append(
slices.Clone(logRotationContainerArgs),
utils.LogLevelContainerArg(logLevel),
utils.TypeContainerArg(string(r.driverType)),
utils.NodeServerContainerArg,
Expand All @@ -812,7 +861,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""),
utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""),
// TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
utils.NodeIdEnvVar,
Expand All @@ -838,6 +887,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
if r.isRdbDriver() {
mounts = append(mounts, utils.OidcTokenVolumeMount)
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
Expand Down Expand Up @@ -886,7 +940,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
Drop: []corev1.Capability{"All"},
},
},
Args: []string{
Args: append(
slices.Clone(logRotationCsiAddonsSidecarContainerArgs),
utils.NodeIdContainerArg,
utils.LogLevelContainerArg(logLevel),
utils.CsiAddonsAddressContainerArg,
Expand All @@ -895,7 +950,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
},
),
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
},
Expand All @@ -905,9 +960,17 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodNamespaceEnvVar,
utils.PodUidEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.PluginDirVolumeMount,
}
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
mounts = append(mounts, utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name))
}
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
Expand Down Expand Up @@ -946,6 +1009,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
})
}
}
// CSI Logrotate Container
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logRotateContainer(imagePullPolicy, daemonSet.Name, ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})))
}
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -968,6 +1037,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PodsMountDirVolume(pluginSpec.KubeletDirPath),
utils.RegistrationDirVolume(pluginSpec.KubeletDirPath),
)
if r.driver.Spec.Log != nil {
if r.driver.Spec.Log.Rotation != nil {
volumes = append(volumes, utils.LogsDirVolume(r.driver.Spec.Log.Rotation.LogHostPath), utils.LogRotateDirVolumeName())
}
}
if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) {
volumes = append(
volumes,
Expand Down Expand Up @@ -1183,6 +1257,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if src.ControllerPlugin != nil {
Expand Down Expand Up @@ -1238,6 +1315,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if dest.AttachRequired == nil {
Expand All @@ -1259,3 +1339,55 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
dest.CephFsClientType = src.CephFsClientType
}
}

func (r *driverReconcile) logRotateContainer(imagePullPolicy corev1.PullPolicy, csiComponentName string, resources corev1.ResourceRequirements) corev1.Container {
rotation := r.driver.Spec.Log.Rotation
return corev1.Container{
Name: "log-collector",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: resources,
Command: []string{
"/bin/bash",
"-c", // Command to run
fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, csiComponentName),
},
VolumeMounts: []corev1.VolumeMount{
utils.LogsDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
utils.LogRotateDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
},
}
}

var cronLogRotate = `
echo "Starting the csi-logrotate-sidecar"
PERIODICITY=%s
LOG_MAX_SIZE=%s
ROTATE=%s
CsiLogHostPath=%s
CsiComponentName=%s
mkdir -p $CsiLogHostPath/logrotate-config/$CsiComponentName
cat <<EOF > ${CsiLogHostPath}/logrotate-config/${CsiComponentName}/csi
$CsiLogHostPath/log/$CsiComponentName/*.log {
$PERIODICITY
missingok
rotate $ROTATE
compress
copytruncate
notifempty
}
EOF
echo "File creation container completed"
LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi
while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
`
32 changes: 32 additions & 0 deletions internal/logrotate/logrotatesh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env -S bash -e

echo "Starting the csi-logrotate-sidecar"
PERIODICITY=%s
LOG_MAX_SIZE=%s
ROTATE=%s
CsiLogHostPath=%s
CsiComponentName=%s

mkdir -p "$CsiLogHostPath"/logrotate-config/"$CsiComponentName"
cat <<EOF > ${CsiLogHostPath}/logrotate-config/${CsiComponentName}/csi
"$CsiLogHostPath"/log/"$CsiComponentName"/*.log {
"$PERIODICITY"
missingok
rotate "$ROTATE"
compress
copytruncate
notifempty
}
EOF

echo "File creation container completed"

LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi

while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
Loading

0 comments on commit d1764c6

Please sign in to comment.