Skip to content

Commit fcf3f42

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit fcf3f42

File tree

4 files changed

+185
-130
lines changed

4 files changed

+185
-130
lines changed

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go‎

Lines changed: 92 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package controller
22

33
import (
4+
"context"
45
"encoding/json"
56
"errors"
67
"fmt"
@@ -27,7 +28,7 @@ const (
2728
annotationAgentName = "lightrun.com/lightrunjavaagent"
2829
)
2930

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
31+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3132
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3233
jsonString, err := json.Marshal(metadata)
3334
if err != nil {
@@ -52,26 +53,28 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5253
}
5354

5455
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5656
// init spec.template.spec
5757
deploymentApplyConfig.WithSpec(
5858
appsv1ac.DeploymentSpec().WithTemplate(
5959
corev1ac.PodTemplateSpec().WithSpec(
6060
corev1ac.PodSpec(),
6161
).WithAnnotations(map[string]string{
6262
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
63+
}),
6564
),
6665
).WithAnnotations(map[string]string{
6766
annotationAgentName: lightrunJavaAgent.Name,
6867
})
6968
r.addVolume(deploymentApplyConfig, lightrunJavaAgent)
7069
r.addInitContainer(deploymentApplyConfig, lightrunJavaAgent, secret)
71-
err = r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
70+
err := r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
7271
if err != nil {
7372
return err
7473
}
74+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
75+
corev1ac.PodSecurityContext().
76+
WithFSGroup(1000),
77+
)
7578
return nil
7679
}
7780

@@ -98,55 +101,73 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
98101
)
99102
}
100103

101-
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
104+
func (r *LightrunJavaAgentReconciler) createPinnedCertConfigMap(ctx context.Context, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (*corev1.ConfigMap, error) {
105+
configMap := &corev1.ConfigMap{
106+
ObjectMeta: metav1.ObjectMeta{
107+
Name: fmt.Sprintf("%s-pinned-cert", lightrunJavaAgent.Name),
108+
Namespace: lightrunJavaAgent.Namespace,
109+
},
110+
Data: map[string]string{
111+
"pinned_cert_hash": string(secret.Data["pinned_cert_hash"]),
112+
},
113+
}
102114

115+
err := r.Create(ctx, configMap)
116+
if err != nil {
117+
return nil, err
118+
}
119+
120+
return configMap, nil
121+
}
122+
123+
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
103124
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104125
corev1ac.Container().
105126
WithName(initContainerName).
106127
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107128
WithVolumeMounts(
108-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
129+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
109130
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
111-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112-
corev1ac.EnvVarSource().WithSecretKeyRef(
113-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
114-
),
115-
),
116-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
117-
corev1ac.EnvVarSource().WithSecretKeyRef(
118-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119-
),
120-
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
131+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
132+
).
133+
WithEnv(
134+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
135+
).
136+
WithSecurityContext(
137+
corev1ac.SecurityContext().
138+
WithReadOnlyRootFilesystem(true).
139+
WithAllowPrivilegeEscalation(false).
140+
WithRunAsNonRoot(true).
141+
WithRunAsUser(1000),
142+
).
123143
WithResources(
124144
corev1ac.ResourceRequirements().
125145
WithLimits(
126146
corev1.ResourceList{
127147
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
148+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
129149
},
130-
).WithRequests(
131-
corev1.ResourceList{
132-
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134-
},
135-
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141150
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
151+
WithRequests(
152+
corev1.ResourceList{
153+
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
154+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
155+
},
147156
),
148157
),
149158
)
159+
160+
// Add volume for secret with proper permissions
161+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(
162+
corev1ac.Volume().WithName("lightrun-secret").
163+
WithSecret(corev1ac.SecretVolumeSource().
164+
WithSecretName(secret.Name).
165+
WithItems(
166+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
167+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
168+
).
169+
WithDefaultMode(0440)),
170+
)
150171
}
151172

152173
func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration) error {
@@ -167,26 +188,22 @@ func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agen
167188
}
168189
}
169190
if !found {
170-
err = errors.New("unable to find matching container to patch")
171-
return err
191+
return errors.New("unable to find matching container to patch")
172192
}
173193
return nil
174194
}
175195

176196
// Client side patch, as we can't update value from 2 sources
177197
func (r *LightrunJavaAgentReconciler) patchJavaToolEnv(deplAnnotations map[string]string, container *corev1.Container, targetEnvVar string, agentArg string) error {
178-
// Check if some env was already patched before
179198
patchedEnv := deplAnnotations[annotationPatchedEnvName]
180199
patchedEnvValue := deplAnnotations[annotationPatchedEnvValue]
181200

182201
if patchedEnv != targetEnvVar || patchedEnvValue != agentArg {
183-
// If different env was patched before - unpatch it
184202
r.unpatchJavaToolEnv(deplAnnotations, container)
185203
}
186204

187205
targetEnvVarIndex := findEnvVarIndex(targetEnvVar, container.Env)
188206
if targetEnvVarIndex == -1 {
189-
// No such env - add new
190207
container.Env = append(container.Env, corev1.EnvVar{
191208
Name: targetEnvVar,
192209
Value: agentArg,
@@ -223,28 +240,22 @@ func (r *LightrunJavaAgentReconciler) unpatchJavaToolEnv(deplAnnotations map[str
223240

224241
// patchStatefulSet applies changes to a StatefulSet to inject the Lightrun agent
225242
func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration, cmDataHash uint64) error {
226-
// init spec.template.spec
227243
statefulSetApplyConfig.WithSpec(
228244
appsv1ac.StatefulSetSpec().WithTemplate(
229245
corev1ac.PodTemplateSpec().WithSpec(
230246
corev1ac.PodSpec(),
231247
).WithAnnotations(map[string]string{
232248
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
249+
}),
235250
),
236251
).WithAnnotations(map[string]string{
237252
annotationAgentName: lightrunJavaAgent.Name,
238253
})
239254

240-
// Add volumes to the StatefulSet
241255
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243-
// Add init container to the StatefulSet
244256
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245257

246-
// Patch app containers in the StatefulSet
247-
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
258+
err := r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248259
if err != nil {
249260
return err
250261
}
@@ -271,6 +282,15 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271282
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272283
),
273284
),
285+
).WithVolumes(
286+
corev1ac.Volume().WithName("lightrun-secret").
287+
WithSecret(corev1ac.SecretVolumeSource().
288+
WithSecretName(secret.Name).
289+
WithItems(
290+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
291+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
292+
).
293+
WithDefaultMode(0440)),
274294
)
275295
}
276296

@@ -280,48 +300,45 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
280300
WithName(initContainerName).
281301
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
282302
WithVolumeMounts(
283-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
303+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
284304
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
305+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
285306
).WithEnv(
286-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287-
corev1ac.EnvVarSource().WithSecretKeyRef(
288-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
289-
),
290-
),
291-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
292-
corev1ac.EnvVarSource().WithSecretKeyRef(
293-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294-
),
295-
),
296307
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297308
).
309+
WithSecurityContext(
310+
corev1ac.SecurityContext().
311+
WithReadOnlyRootFilesystem(true).
312+
WithAllowPrivilegeEscalation(false).
313+
WithRunAsNonRoot(true).
314+
WithRunAsUser(1000),
315+
).
298316
WithResources(
299317
corev1ac.ResourceRequirements().
300318
WithLimits(
301319
corev1.ResourceList{
302320
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
303-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
321+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
304322
},
305323
).WithRequests(
306324
corev1.ResourceList{
307325
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
308326
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
309327
},
310328
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323329
),
324330
)
331+
332+
statefulSetApplyConfig.Spec.Template.Spec.WithVolumes(
333+
corev1ac.Volume().WithName("lightrun-secret").
334+
WithSecret(corev1ac.SecretVolumeSource().
335+
WithSecretName(secret.Name).
336+
WithItems(
337+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
338+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
339+
).
340+
WithDefaultMode(0440)),
341+
)
325342
}
326343

327344
func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration) error {
@@ -342,15 +359,13 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
342359
}
343360
}
344361
if !found {
345-
err = errors.New("unable to find matching container to patch")
346-
return err
362+
return errors.New("unable to find matching container to patch")
347363
}
348364
return nil
349365
}
350366

351367
// configMapDataHash calculates a hash of the ConfigMap data to detect changes
352368
func configMapDataHash(cmData map[string]string) uint64 {
353-
// Combine all data values into a single string for hashing
354369
var hashString string
355370
for _, v := range cmData {
356371
hashString += v

‎lightrun-init-agent/Dockerfile‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
5-
6-
COPY lightrun-init-agent/$FILE /tmp/$FILE
4+
ARG FILE
75

86
RUN unzip -o /tmp/$FILE -d /agent ;\
97
rm -rf /tmp/$FILE && \
108
# Erase default values
119
sed -i.bak "s|com.lightrun.secret=.*|com.lightrun.secret=|" /agent/agent.config && rm /agent/agent.config.bak && \
1210
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
13-
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
14-
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
11+
# Set proper permissions for the agent directory
12+
chown -R 1000:1000 /agent && \
13+
chmod -R 750 /agent && \
14+
# Create secret directory with proper permissions
15+
mkdir -p /etc/lightrun/secret && \
16+
chown -R 1000:1000 /etc/lightrun/secret && \
17+
chmod -R 700 /etc/lightrun/secret
18+
19+
# Copy and set permissions for update_config.sh before switching user
20+
COPY update_config.sh /update_config.sh
21+
RUN chmod 750 /update_config.sh && \
22+
chown 1000:1000 /update_config.sh
1623

1724
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1925

2026
CMD [ "/bin/sh", "/update_config.sh" ]

0 commit comments

Comments
 (0)