Skip to content

Commit f343dcc

Browse files
author
eliranb
committed
Phase 2: StatefulSet Implementation
1 parent e9b798a commit f343dcc

File tree

6 files changed

+453
-9
lines changed

6 files changed

+453
-9
lines changed

‎examples/deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ spec:
1919
env:
2020
- name: JAVA_TOOL_OPTIONS
2121
value: -Djava.net.preferIPv4Stack=true
22-
image: lightruncom/operator-demo-app
22+
image: lightrun-k8s-operator-registry:5000/operator-demo-app:latest
2323
securityContext:
2424
allowPrivilegeEscalation: false
2525
capabilities:

‎examples/lightrunjavaagent.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ spec:
3030
# agent version - first part of the tag (1.7.0)
3131
# init container sub-version - last part of the tag (init.0)
3232
# List of available images in the README.md
33-
image: "lightruncom/k8s-operator-init-java-agent-linux:latest"
33+
image: "lightrun-k8s-operator-registry:5000/lightrun-init-agent:latest"
3434
# Volume name in case you have some convention in the names
3535
sharedVolumeName: lightrun-agent-init
3636
# Mount path where volume will be parked. Various distributions may have it's limitations.
@@ -66,8 +66,8 @@ metadata:
6666
name: lightrun-secrets
6767
stringData:
6868
# Lightrun key you can take from the server UI at the "setup agent" step
69-
lightrun_key: <lightrun_key_from_ui>
69+
lightrun_key: 65cf112e-03f5-42da-a2f3-3c46d368872b
7070
# Server certificate hash. It is ensuring that agent is connected to the right Lightrun server
71-
pinned_cert_hash: <pinned_cert_hash>
71+
pinned_cert_hash: ee80811b38e7e6c2dc4cc372cbea86bd86b446b012e427f2e19bf094afba5d12
7272
kind: Secret
7373
type: Opaque

‎examples/statefulset-example.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
apiVersion: apps/v1
2+
kind: StatefulSet
3+
metadata:
4+
name: sample-statefulset
5+
labels:
6+
app: stateful-app
7+
spec:
8+
serviceName: "stateful-app"
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: stateful-app
13+
template:
14+
metadata:
15+
labels:
16+
app: stateful-app
17+
spec:
18+
containers:
19+
- name: app
20+
env:
21+
- name: JAVA_TOOL_OPTIONS
22+
value: -Djava.net.preferIPv4Stack=true
23+
image: lightrun-k8s-operator-registry:5000/operator-demo-app:latest
24+
securityContext:
25+
allowPrivilegeEscalation: false
26+
capabilities:
27+
drop: ["ALL"]
28+
runAsNonRoot: true
29+
seccompProfile:
30+
type: RuntimeDefault
31+
# volumeMounts:
32+
# - name: data
33+
# mountPath: /data
34+
# Second container will be not patched, as not mentioned in the custom resource
35+
- name: non-patched-app
36+
image: lightruncom/operator-demo-app
37+
securityContext:
38+
allowPrivilegeEscalation: false
39+
capabilities:
40+
drop: ["ALL"]
41+
runAsNonRoot: true
42+
seccompProfile:
43+
type: RuntimeDefault
44+
# volumeMounts:
45+
# - name: data
46+
# mountPath: /data
47+
# volumeClaimTemplates:
48+
# - metadata:
49+
# name: data
50+
# spec:
51+
# accessModes: ["ReadWriteOnce"]
52+
# resources:
53+
# requests:
54+
# storage: 1Gi

‎examples/statefulset-lightrunjavaagent.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
apiVersion: agents.lightrun.com/v1beta
2+
kind: LightrunJavaAgent
3+
metadata:
4+
name: statefulset-agent
5+
spec:
6+
# Specify the StatefulSet to instrument
7+
statefulSetName: "sample-statefulset"
8+
# List of container names inside the pod to instrument
9+
containerSelector:
10+
- app
11+
# Init container with the Lightrun agent
12+
initContainer:
13+
image: "lightrun-k8s-operator-registry:5000/lightrun-init-agent:latest"
14+
sharedVolumeName: "lightrun-agent"
15+
sharedVolumeMountPath: "/lightrun"
16+
# Reference to the Secret with Lightrun credentials
17+
secretName: "lightrun-secrets"
18+
# Environment variable to patch with the agent path
19+
agentEnvVarName: "JAVA_TOOL_OPTIONS"
20+
# Lightrun server hostname
21+
serverHostname: "app.lightrun.com"
22+
# Optional agent configuration
23+
agentConfig:
24+
max_log_cpu_cost: "2"
25+
# Tags that will appear in the Lightrun UI
26+
agentTags:
27+
- operator
28+
- example
29+
- latest
30+
- statefulset
31+
# Optional agent name (if not set, pod name will be used)
32+
agentName: "stateful-java-app"

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 226 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,232 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
314314
// reconcileStatefulSet handles the reconciliation logic for StatefulSet workloads
315315
func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, namespace string) (ctrl.Result, error) {
316316
log := r.Log.WithValues("lightrunJavaAgent", lightrunJavaAgent.Name, "statefulSet", lightrunJavaAgent.Spec.StatefulSetName)
317+
fieldManager := "lightrun-controller"
317318

318-
// This is a placeholder for Phase 2 implementation
319-
log.Info("StatefulSet reconciliation not yet implemented", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
320-
return r.errorStatus(ctx, lightrunJavaAgent, errors.New("statefulset reconciliation not yet implemented"))
319+
stsNamespacedObj := client.ObjectKey{
320+
Name: lightrunJavaAgent.Spec.StatefulSetName,
321+
Namespace: namespace,
322+
}
323+
originalStatefulSet := &appsv1.StatefulSet{}
324+
err = r.Get(ctx, stsNamespacedObj, originalStatefulSet)
325+
if err != nil {
326+
// StatefulSet not found
327+
if client.IgnoreNotFound(err) == nil {
328+
log.Info("StatefulSet not found. Verify name/namespace", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
329+
// remove our finalizer from the list and update it.
330+
err = r.removeFinalizer(ctx, lightrunJavaAgent, finalizerName)
331+
if err != nil {
332+
return r.errorStatus(ctx, lightrunJavaAgent, err)
333+
}
334+
return r.errorStatus(ctx, lightrunJavaAgent, errors.New("statefulset not found"))
335+
} else {
336+
log.Error(err, "unable to fetch statefulset")
337+
return r.errorStatus(ctx, lightrunJavaAgent, err)
338+
}
339+
}
340+
341+
// Check if this LightrunJavaAgent is being deleted
342+
if !lightrunJavaAgent.ObjectMeta.DeletionTimestamp.IsZero() {
343+
// The object is being deleted
344+
if containsString(lightrunJavaAgent.ObjectMeta.Finalizers, finalizerName) {
345+
// our finalizer is present, so lets handle any cleanup operations
346+
347+
// Restore original StatefulSet (unpatch)
348+
// Volume and init container
349+
log.Info("Unpatching StatefulSet", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
350+
351+
originalStatefulSet = &appsv1.StatefulSet{}
352+
err = r.Get(ctx, stsNamespacedObj, originalStatefulSet)
353+
if err != nil {
354+
if client.IgnoreNotFound(err) == nil {
355+
log.Info("StatefulSet not found", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
356+
// remove our finalizer from the list and update it.
357+
log.Info("Removing finalizer")
358+
err = r.removeFinalizer(ctx, lightrunJavaAgent, finalizerName)
359+
if err != nil {
360+
return r.errorStatus(ctx, lightrunJavaAgent, err)
361+
}
362+
// Successfully removed finalizer and nothing to restore
363+
return r.successStatus(ctx, lightrunJavaAgent, reconcileTypeReady)
364+
}
365+
log.Error(err, "unable to unpatch statefulset", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
366+
return r.errorStatus(ctx, lightrunJavaAgent, err)
367+
}
368+
369+
// Revert environment variable modifications
370+
clientSidePatch := client.MergeFrom(originalStatefulSet.DeepCopy())
371+
for i, container := range originalStatefulSet.Spec.Template.Spec.Containers {
372+
for _, targetContainer := range lightrunJavaAgent.Spec.ContainerSelector {
373+
if targetContainer == container.Name {
374+
r.unpatchJavaToolEnv(originalStatefulSet.Annotations, &originalStatefulSet.Spec.Template.Spec.Containers[i])
375+
}
376+
}
377+
}
378+
delete(originalStatefulSet.Annotations, annotationPatchedEnvName)
379+
delete(originalStatefulSet.Annotations, annotationPatchedEnvValue)
380+
delete(originalStatefulSet.Annotations, annotationAgentName)
381+
err = r.Patch(ctx, originalStatefulSet, clientSidePatch)
382+
if err != nil {
383+
log.Error(err, "failed to unpatch statefulset environment variables")
384+
return r.errorStatus(ctx, lightrunJavaAgent, err)
385+
}
386+
387+
// Remove Volumes and init container
388+
emptyApplyConfig := appsv1ac.StatefulSet(stsNamespacedObj.Name, stsNamespacedObj.Namespace)
389+
obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(emptyApplyConfig)
390+
if err != nil {
391+
log.Error(err, "failed to convert StatefulSet to unstructured")
392+
return r.errorStatus(ctx, lightrunJavaAgent, err)
393+
}
394+
patch := &unstructured.Unstructured{
395+
Object: obj,
396+
}
397+
err = r.Patch(ctx, patch, client.Apply, &client.PatchOptions{
398+
FieldManager: fieldManager,
399+
Force: pointer.Bool(true),
400+
})
401+
if err != nil {
402+
log.Error(err, "failed to unpatch statefulset")
403+
return r.errorStatus(ctx, lightrunJavaAgent, err)
404+
}
405+
406+
// remove our finalizer from the list and update it.
407+
log.Info("Removing finalizer")
408+
err = r.removeFinalizer(ctx, lightrunJavaAgent, finalizerName)
409+
if err != nil {
410+
return r.errorStatus(ctx, lightrunJavaAgent, err)
411+
}
412+
413+
log.Info("StatefulSet returned to original state", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
414+
return r.successStatus(ctx, lightrunJavaAgent, reconcileTypeProgressing)
415+
}
416+
// Nothing to do here
417+
return r.successStatus(ctx, lightrunJavaAgent, reconcileTypeProgressing)
418+
}
419+
420+
// Check if already patched by another LightrunJavaAgent
421+
if oldLrjaName, ok := originalStatefulSet.Annotations[annotationAgentName]; ok && oldLrjaName != lightrunJavaAgent.Name {
422+
log.Error(err, "StatefulSet already patched by LightrunJavaAgent", "Existing LightrunJavaAgent", oldLrjaName)
423+
return r.errorStatus(ctx, lightrunJavaAgent, errors.New("statefulset already patched"))
424+
}
425+
426+
// Add finalizer if not already present
427+
if !containsString(lightrunJavaAgent.ObjectMeta.Finalizers, finalizerName) {
428+
log.V(2).Info("Adding finalizer")
429+
err = r.addFinalizer(ctx, lightrunJavaAgent, finalizerName)
430+
if err != nil {
431+
log.Error(err, "unable to add finalizer")
432+
return r.errorStatus(ctx, lightrunJavaAgent, err)
433+
}
434+
}
435+
436+
// Get the secret
437+
secretObj := client.ObjectKey{
438+
Name: lightrunJavaAgent.Spec.SecretName,
439+
Namespace: namespace,
440+
}
441+
secret = &corev1.Secret{}
442+
err = r.Get(ctx, secretObj, secret)
443+
if err != nil {
444+
log.Error(err, "unable to fetch Secret", "Secret", lightrunJavaAgent.Spec.SecretName)
445+
return r.errorStatus(ctx, lightrunJavaAgent, err)
446+
}
447+
448+
// Verify that env var won't exceed 1024 chars
449+
agentArg, err := agentEnvVarArgument(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath, lightrunJavaAgent.Spec.AgentCliFlags)
450+
if err != nil {
451+
log.Error(err, "agentEnvVarArgument exceeds 1024 chars")
452+
return r.errorStatus(ctx, lightrunJavaAgent, err)
453+
}
454+
455+
// Create config map
456+
log.V(2).Info("Reconciling config map with agent configuration")
457+
configMap, err := r.createAgentConfig(lightrunJavaAgent)
458+
if err != nil {
459+
log.Error(err, "unable to create configMap")
460+
return r.errorStatus(ctx, lightrunJavaAgent, err)
461+
}
462+
applyOpts := []client.PatchOption{client.ForceOwnership, client.FieldOwner("lightrun-controller")}
463+
464+
err = r.Patch(ctx, &configMap, client.Apply, applyOpts...)
465+
if err != nil {
466+
log.Error(err, "unable to apply configMap")
467+
return r.errorStatus(ctx, lightrunJavaAgent, err)
468+
}
469+
470+
// Calculate ConfigMap data hash
471+
cmDataHash := configMapDataHash(configMap.Data)
472+
473+
// Extract StatefulSet for applying changes
474+
statefulSetApplyConfig, err := appsv1ac.ExtractStatefulSet(originalStatefulSet, fieldManager)
475+
if err != nil {
476+
log.Error(err, "failed to extract StatefulSet")
477+
return r.errorStatus(ctx, lightrunJavaAgent, err)
478+
}
479+
480+
// Server side apply for StatefulSet changes
481+
log.V(2).Info("Patching StatefulSet", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName, "LightunrJavaAgent", lightrunJavaAgent.Name)
482+
err = r.patchStatefulSet(lightrunJavaAgent, secret, originalStatefulSet, statefulSetApplyConfig, cmDataHash)
483+
if err != nil {
484+
log.Error(err, "failed to patch statefulset")
485+
return r.errorStatus(ctx, lightrunJavaAgent, err)
486+
}
487+
488+
obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(statefulSetApplyConfig)
489+
if err != nil {
490+
log.Error(err, "failed to convert StatefulSet to unstructured")
491+
return r.errorStatus(ctx, lightrunJavaAgent, err)
492+
}
493+
patch := &unstructured.Unstructured{
494+
Object: obj,
495+
}
496+
err = r.Patch(ctx, patch, client.Apply, &client.PatchOptions{
497+
FieldManager: fieldManager,
498+
Force: pointer.Bool(true),
499+
})
500+
if err != nil {
501+
log.Error(err, "failed to patch statefulset")
502+
return r.errorStatus(ctx, lightrunJavaAgent, err)
503+
}
504+
505+
// Client side patch (we can't rollback JAVA_TOOL_OPTIONS env with server side apply)
506+
log.V(2).Info("Patching Java Env", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName, "LightunrJavaAgent", lightrunJavaAgent.Name)
507+
originalStatefulSet = &appsv1.StatefulSet{}
508+
err = r.Get(ctx, stsNamespacedObj, originalStatefulSet)
509+
if err != nil {
510+
if client.IgnoreNotFound(err) == nil {
511+
log.Info("StatefulSet not found", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName)
512+
err = r.removeFinalizer(ctx, lightrunJavaAgent, finalizerName)
513+
if err != nil {
514+
return r.errorStatus(ctx, lightrunJavaAgent, err)
515+
}
516+
return r.errorStatus(ctx, lightrunJavaAgent, errors.New("statefulset not found"))
517+
}
518+
return r.errorStatus(ctx, lightrunJavaAgent, err)
519+
}
520+
clientSidePatch := client.MergeFrom(originalStatefulSet.DeepCopy())
521+
for i, container := range originalStatefulSet.Spec.Template.Spec.Containers {
522+
for _, targetContainer := range lightrunJavaAgent.Spec.ContainerSelector {
523+
if targetContainer == container.Name {
524+
err = r.patchJavaToolEnv(originalStatefulSet.Annotations, &originalStatefulSet.Spec.Template.Spec.Containers[i], lightrunJavaAgent.Spec.AgentEnvVarName, agentArg)
525+
if err != nil {
526+
log.Error(err, "failed to patch "+lightrunJavaAgent.Spec.AgentEnvVarName)
527+
return r.errorStatus(ctx, lightrunJavaAgent, err)
528+
}
529+
}
530+
}
531+
}
532+
originalStatefulSet.Annotations[annotationPatchedEnvName] = lightrunJavaAgent.Spec.AgentEnvVarName
533+
originalStatefulSet.Annotations[annotationPatchedEnvValue] = agentArg
534+
err = r.Patch(ctx, originalStatefulSet, clientSidePatch)
535+
if err != nil {
536+
log.Error(err, "failed to patch "+lightrunJavaAgent.Spec.AgentEnvVarName)
537+
return r.errorStatus(ctx, lightrunJavaAgent, err)
538+
}
539+
540+
// Update status to Healthy
541+
log.V(1).Info("Reconciling finished successfully", "StatefulSet", lightrunJavaAgent.Spec.StatefulSetName, "LightunrJavaAgent", lightrunJavaAgent.Name)
542+
return r.successStatus(ctx, lightrunJavaAgent, reconcileTypeReady)
321543
}
322544

323545
// SetupWithManager sets up the controller with the Manager.
@@ -360,7 +582,7 @@ func (r *LightrunJavaAgentReconciler) SetupWithManager(mgr ctrl.Manager) error {
360582
return err
361583
}
362584

363-
// Add spec.container_selector.secret field to cache for future filtering
585+
// Add spec.secret field to cache for future filtering
364586
err = mgr.GetFieldIndexer().IndexField(
365587
context.Background(),
366588
&agentv1beta.LightrunJavaAgent{},
@@ -381,7 +603,6 @@ func (r *LightrunJavaAgentReconciler) SetupWithManager(mgr ctrl.Manager) error {
381603

382604
return ctrl.NewControllerManagedBy(mgr).
383605
For(&agentv1beta.LightrunJavaAgent{}).
384-
Owns(&corev1.ConfigMap{}).
385606
Watches(
386607
&appsv1.Deployment{},
387608
handler.EnqueueRequestsFromMapFunc(r.mapDeploymentToAgent),

0 commit comments

Comments
 (0)