Skip to content
This repository was archived by the owner on Oct 14, 2020. It is now read-only.

Commit 584e956

Browse files
J12934J12934
committed
Refactore Hook handling
1 parent 753a34e commit 584e956

File tree

1 file changed

+124
-163
lines changed

1 file changed

+124
-163
lines changed

‎operator/controllers/execution/scan_controller.go‎

Lines changed: 124 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -353,8 +353,7 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
353353
}
354354
log.Info("Matching ParseDefinition Found", "ParseDefinition", parseType)
355355

356-
bucketName := os.Getenv("S3_BUCKET")
357-
findingsUploadURL, err := r.MinioClient.PresignedPutObject(bucketName, fmt.Sprintf("scan-%s/findings.json", scan.UID), 12*time.Hour)
356+
findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json")
358357
if err != nil {
359358
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
360359
return err
@@ -423,7 +422,7 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
423422
},
424423
Args: []string{
425424
rawResultDownloadURL,
426-
findingsUploadURL.String(),
425+
findingsUploadURL,
427426
},
428427
ImagePullPolicy: "Always",
429428
},
@@ -486,10 +485,8 @@ func (r *ScanReconciler) checkIfParsingIsCompleted(scan *executionv1.Scan) error
486485
}
487486

488487
func (r *ScanReconciler) constructJobForScan(scan *executionv1.Scan, scanType *executionv1.ScanType) (*batch.Job, error) {
489-
bucketName := os.Getenv("S3_BUCKET")
490-
491488
filename := filepath.Base(scanType.Spec.ExtractResults.Location)
492-
resultUploadURL, err := r.MinioClient.PresignedPutObject(bucketName, fmt.Sprintf("scan-%s/%s", scan.UID, filename), 12*time.Hour)
489+
resultUploadURL, err := r.PresignedPutURL(scan.UID, filename)
493490
if err != nil {
494491
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
495492
return nil, err
@@ -575,7 +572,7 @@ func (r *ScanReconciler) constructJobForScan(scan *executionv1.Scan, scanType *e
575572
"--file",
576573
scanType.Spec.ExtractResults.Location,
577574
"--url",
578-
resultUploadURL.String(),
575+
resultUploadURL,
579576
},
580577
Env: []corev1.EnvVar{
581578
{
@@ -640,6 +637,18 @@ func (r *ScanReconciler) PresignedGetURL(scanID types.UID, filename string) (str
640637
return rawResultDownloadURL.String(), nil
641638
}
642639

640+
// PresignedPutURL returns a presigned URL from the s3 (or compatible) serice.
641+
func (r *ScanReconciler) PresignedPutURL(scanID types.UID, filename string) (string, error) {
642+
bucketName := os.Getenv("S3_BUCKET")
643+
644+
rawResultDownloadURL, err := r.MinioClient.PresignedPutObject(bucketName, fmt.Sprintf("scan-%s/%s", string(scanID), filename), 12*time.Hour)
645+
if err != nil {
646+
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
647+
return "", err
648+
}
649+
return rawResultDownloadURL.String(), nil
650+
}
651+
643652
func (r *ScanReconciler) startReadOnlyHooks(scan *executionv1.Scan) error {
644653
ctx := context.Background()
645654

@@ -698,71 +707,18 @@ func (r *ScanReconciler) startReadOnlyHooks(scan *executionv1.Scan) error {
698707
return err
699708
}
700709

701-
standardEnvVars := []corev1.EnvVar{
702-
{
703-
Name: "NAMESPACE",
704-
ValueFrom: &corev1.EnvVarSource{
705-
FieldRef: &corev1.ObjectFieldSelector{
706-
FieldPath: "metadata.namespace",
707-
},
708-
},
709-
},
710-
{
711-
Name: "SCAN_NAME",
712-
Value: scan.Name,
713-
},
714-
}
715-
716-
// Starting a new job based on the current ReadOnlyHook
717-
labels := scan.ObjectMeta.DeepCopy().Labels
718-
if labels == nil {
719-
labels = make(map[string]string)
720-
}
721-
labels["experimental.securecodebox.io/job-type"] = "read-only-hook"
722-
job := &batch.Job{
723-
ObjectMeta: metav1.ObjectMeta{
724-
Annotations: make(map[string]string),
725-
Name: fmt.Sprintf("%s-%s", hook.Name, scan.Name),
726-
Namespace: scan.Namespace,
727-
Labels: labels,
728-
},
729-
Spec: batch.JobSpec{
730-
Template: corev1.PodTemplateSpec{
731-
ObjectMeta: metav1.ObjectMeta{
732-
Annotations: map[string]string{
733-
"auto-discovery.experimental.securecodebox.io/ignore": "true",
734-
},
735-
},
736-
Spec: corev1.PodSpec{
737-
ServiceAccountName: serviceAccountName,
738-
RestartPolicy: corev1.RestartPolicyNever,
739-
Containers: []corev1.Container{
740-
{
741-
Name: "hook",
742-
Image: hook.Spec.Image,
743-
Args: []string{
744-
rawFileURL,
745-
findingsFileURL,
746-
},
747-
Env: append(hook.Spec.Env, standardEnvVars...),
748-
ImagePullPolicy: "IfNotPresent",
749-
},
750-
},
751-
},
752-
},
753-
TTLSecondsAfterFinished: nil,
710+
jobName, err := r.createJobForHook(
711+
&hook,
712+
scan,
713+
[]string{
714+
rawFileURL,
715+
findingsFileURL,
754716
},
755-
}
756-
if err := ctrl.SetControllerReference(scan, job, r.Scheme); err != nil {
757-
r.Log.Error(err, "Unable to set controllerReference on job", "job", job)
758-
return err
759-
}
760-
761-
if err := r.Create(ctx, job); err != nil {
762-
r.Log.Error(err, "Unable to create Job for ReadOnlyHook", "job", job)
717+
)
718+
if err != nil {
719+
r.Log.Error(err, "Unable to create Job for ReadOnlyHook", "job", jobName)
763720
return err
764721
}
765-
766722
}
767723
scan.Status.State = "ReadOnlyHookProcessing"
768724
if err := r.Status().Update(ctx, scan); err != nil {
@@ -1015,6 +971,88 @@ func (r *ScanReconciler) setHookStatus(scan *executionv1.Scan) error {
1015971
return nil
1016972
}
1017973

974+
func (r *ScanReconciler) createJobForHook(hook *executionv1.ScanCompletionHook, scan *executionv1.Scan, cliArgs []string) (string, error) {
975+
ctx := context.Background()
976+
rules := []rbacv1.PolicyRule{
977+
{
978+
APIGroups: []string{"execution.experimental.securecodebox.io"},
979+
Resources: []string{"scans"},
980+
Verbs: []string{"get"},
981+
},
982+
}
983+
serviceAccountName := "scan-completion-hook"
984+
r.ensureServiceAccountExists(
985+
hook.Namespace,
986+
serviceAccountName,
987+
"ScanCompletionHooks need to access the current scan to view where its results are stored",
988+
rules,
989+
)
990+
991+
standardEnvVars := []corev1.EnvVar{
992+
{
993+
Name: "NAMESPACE",
994+
ValueFrom: &corev1.EnvVarSource{
995+
FieldRef: &corev1.ObjectFieldSelector{
996+
FieldPath: "metadata.namespace",
997+
},
998+
},
999+
},
1000+
{
1001+
Name: "SCAN_NAME",
1002+
Value: scan.Name,
1003+
},
1004+
}
1005+
1006+
// Starting a new job based on the current ReadAndWrite Hook
1007+
labels := scan.ObjectMeta.DeepCopy().Labels
1008+
if labels == nil {
1009+
labels = make(map[string]string)
1010+
}
1011+
labels["experimental.securecodebox.io/job-type"] = "read-and-write-hook"
1012+
var backOffLimit int32 = 3
1013+
job := &batch.Job{
1014+
ObjectMeta: metav1.ObjectMeta{
1015+
Annotations: make(map[string]string),
1016+
Name: fmt.Sprintf("%s-%s", hook.Name, scan.Name),
1017+
Namespace: scan.Namespace,
1018+
Labels: labels,
1019+
},
1020+
Spec: batch.JobSpec{
1021+
BackoffLimit: &backOffLimit,
1022+
Template: corev1.PodTemplateSpec{
1023+
ObjectMeta: metav1.ObjectMeta{
1024+
Annotations: map[string]string{
1025+
"auto-discovery.experimental.securecodebox.io/ignore": "true",
1026+
},
1027+
},
1028+
Spec: corev1.PodSpec{
1029+
ServiceAccountName: serviceAccountName,
1030+
RestartPolicy: corev1.RestartPolicyNever,
1031+
Containers: []corev1.Container{
1032+
{
1033+
Name: "hook",
1034+
Image: hook.Spec.Image,
1035+
Args: cliArgs,
1036+
Env: append(hook.Spec.Env, standardEnvVars...),
1037+
ImagePullPolicy: "IfNotPresent",
1038+
},
1039+
},
1040+
},
1041+
},
1042+
TTLSecondsAfterFinished: nil,
1043+
},
1044+
}
1045+
if err := ctrl.SetControllerReference(scan, job, r.Scheme); err != nil {
1046+
r.Log.Error(err, "Unable to set controllerReference on job", "job", job)
1047+
return "", err
1048+
}
1049+
1050+
if err := r.Create(ctx, job); err != nil {
1051+
return "", err
1052+
}
1053+
return job.Name, nil
1054+
}
1055+
10181056
func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error {
10191057
// First Array entry which is not Completed.
10201058
ctx := context.Background()
@@ -1036,22 +1074,8 @@ func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error
10361074
return nil
10371075
}
10381076

1039-
if nonCompletedHook.State == executionv1.Pending {
1040-
rules := []rbacv1.PolicyRule{
1041-
{
1042-
APIGroups: []string{"execution.experimental.securecodebox.io"},
1043-
Resources: []string{"scans"},
1044-
Verbs: []string{"get"},
1045-
},
1046-
}
1047-
serviceAccountName := "scan-completion-hook"
1048-
r.ensureServiceAccountExists(
1049-
scan.Namespace,
1050-
serviceAccountName,
1051-
"ScanCompletionHooks need to access the current scan to view where its results are stored",
1052-
rules,
1053-
)
1054-
1077+
switch nonCompletedHook.State {
1078+
case executionv1.Pending:
10551079
rawFileURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile)
10561080
if err != nil {
10571081
return err
@@ -1061,97 +1085,35 @@ func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error
10611085
return err
10621086
}
10631087

1064-
bucketName := os.Getenv("S3_BUCKET")
1065-
rawFileUploadURL, err := r.MinioClient.PresignedPutObject(bucketName, fmt.Sprintf("scan-%s/%s", scan.UID, scan.Status.RawResultFile), 12*time.Hour)
1088+
rawFileUploadURL, err := r.PresignedPutURL(scan.UID, scan.Status.RawResultFile)
10661089
if err != nil {
1067-
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
10681090
return err
10691091
}
1070-
findingsUploadURL, err := r.MinioClient.PresignedPutObject(bucketName, fmt.Sprintf("scan-%s/findings.json", scan.UID), 12*time.Hour)
1092+
findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json")
10711093
if err != nil {
1072-
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
10731094
return err
10741095
}
10751096

10761097
var hook executionv1.ScanCompletionHook
1077-
err = r.Get(ctx, types.NamespacedName{Name: nonCompletedHook.HookName, Namespace: scan.Namespace}, &hook)
1078-
if err != nil {
1098+
if err := r.Get(ctx, types.NamespacedName{Name: nonCompletedHook.HookName, Namespace: scan.Namespace}, &hook); err != nil {
10791099
r.Log.Error(err, "Failed to get ReadAndWrite Hook for HookStatus")
10801100
return err
10811101
}
10821102

1083-
standardEnvVars := []corev1.EnvVar{
1084-
{
1085-
Name: "NAMESPACE",
1086-
ValueFrom: &corev1.EnvVarSource{
1087-
FieldRef: &corev1.ObjectFieldSelector{
1088-
FieldPath: "metadata.namespace",
1089-
},
1090-
},
1091-
},
1092-
{
1093-
Name: "SCAN_NAME",
1094-
Value: scan.Name,
1095-
},
1096-
}
1097-
1098-
// Starting a new job based on the current ReadAndWrite Hook
1099-
labels := scan.ObjectMeta.DeepCopy().Labels
1100-
if labels == nil {
1101-
labels = make(map[string]string)
1102-
}
1103-
labels["experimental.securecodebox.io/job-type"] = "read-and-write-hook"
1104-
var backOffLimit int32 = 3
1105-
job := &batch.Job{
1106-
ObjectMeta: metav1.ObjectMeta{
1107-
Annotations: make(map[string]string),
1108-
Name: fmt.Sprintf("%s-%s", hook.Name, scan.Name),
1109-
Namespace: scan.Namespace,
1110-
Labels: labels,
1103+
jobName, err := r.createJobForHook(
1104+
&hook,
1105+
scan,
1106+
[]string{
1107+
rawFileURL,
1108+
findingsFileURL,
1109+
rawFileUploadURL,
1110+
findingsUploadURL,
11111111
},
1112-
Spec: batch.JobSpec{
1113-
BackoffLimit: &backOffLimit,
1114-
Template: corev1.PodTemplateSpec{
1115-
ObjectMeta: metav1.ObjectMeta{
1116-
Annotations: map[string]string{
1117-
"auto-discovery.experimental.securecodebox.io/ignore": "true",
1118-
},
1119-
},
1120-
Spec: corev1.PodSpec{
1121-
ServiceAccountName: serviceAccountName,
1122-
RestartPolicy: corev1.RestartPolicyNever,
1123-
Containers: []corev1.Container{
1124-
{
1125-
Name: "hook",
1126-
Image: hook.Spec.Image,
1127-
Args: []string{
1128-
rawFileURL,
1129-
findingsFileURL,
1130-
rawFileUploadURL.String(),
1131-
findingsUploadURL.String(),
1132-
},
1133-
Env: append(hook.Spec.Env, standardEnvVars...),
1134-
ImagePullPolicy: "IfNotPresent",
1135-
},
1136-
},
1137-
},
1138-
},
1139-
TTLSecondsAfterFinished: nil,
1140-
},
1141-
}
1142-
if err := ctrl.SetControllerReference(scan, job, r.Scheme); err != nil {
1143-
r.Log.Error(err, "Unable to set controllerReference on job", "job", job)
1144-
return err
1145-
}
1146-
1147-
if err := r.Create(ctx, job); err != nil {
1148-
r.Log.Error(err, "Unable to create Job for ReadAndWriteHook", "job", job)
1149-
return err
1150-
}
1112+
)
11511113

11521114
for i, hookStatus := range scan.Status.ReadAndWriteHookStatus {
11531115
if hookStatus.HookName == nonCompletedHook.HookName {
1154-
scan.Status.ReadAndWriteHookStatus[i].JobName = job.Name
1116+
scan.Status.ReadAndWriteHookStatus[i].JobName = jobName
11551117
scan.Status.ReadAndWriteHookStatus[i].State = executionv1.InProgress
11561118
}
11571119
}
@@ -1161,9 +1123,7 @@ func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error
11611123
return err
11621124
}
11631125
return nil
1164-
}
1165-
1166-
if nonCompletedHook.State == executionv1.InProgress {
1126+
case executionv1.InProgress:
11671127
jobStatus, err := r.checkIfJobIsCompleted(nonCompletedHook.JobName, scan.Namespace)
11681128
if err != nil {
11691129
r.Log.Error(err, "Failed to check job status for ReadAndWrite Hook")
@@ -1201,5 +1161,6 @@ func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error
12011161
}
12021162
}
12031163
}
1164+
12041165
return nil
12051166
}

0 commit comments

Comments
 (0)