Ensure helm release build consistently handles deployOLM toggle (#2066)

Summary: Ensure helm release build consistently handles deployOLM toggle

Splitting the csv-deleter Job into its own file in #2063 introduced a
bug where the `px-operator` namespace becomes stuck on `helm uninstall`
again. This is because helm's `deployOLM` configuration key is patched
for the [00_olm.yaml
file](https://github.com/pixie-io/pixie/blob/5c5e9dcb259c6548c9ab2c4b31c80d31f5f90124/ci/operator_helm_build_release.sh#L60)
during the release build. When I initially tested in
#2059, the csv-deleter Job existed
in the file and received this special treatment, but after #2063 it no
longer did.

In order to address this, this change introduces a Go template comment
placeholder that `sed` will look to make it clear that helm operates
with a different conditional check. It also runs this sed command across
all template files and not just the `00_olm.yaml` file. This comment
will be stripped out during the `px` cli's yaml templating and will
leave it's `deployOLM` conditional check intact.

Relevant Issues: #1917

Type of change: /kind bug

Test Plan: Ran the sed command and verified that it works
```
# Verify bash variables are set so sed command can be copy and pasted from this PR
$ echo $repo_path
./
$ echo $helm_tmpl_checks
{{- $olmCRDFound := false }} \ {{- $nsLookup := len (lookup "v1" "Namespace" "" "") }} \ {{- range $index, $crdLookup := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "").items -}}{{ if eq $crdLookup.metadata.name "operators.operators.coreos.com"}}{{ $olmCRDFound = true }}{{ end }}{{end}} \ {{ if and (not $olmCRDFound) (not (eq $nsLookup 0))}}{{ fail "CRDs missing! Please deploy CRDs from https://github.com/pixie-io/pixie/tree/main/k8s/operator/helm/crds to continue with deploy." }}{{end}} \ {{- $lookupLen := 0 -}}{{- $opLookup := (lookup "operators.coreos.com/v1" "OperatorGroup" "" "").items -}}{{if $opLookup }}{{ $lookupLen = len $opLookup }}{{ end }}\n{{ if (or (eq (.Values.deployOLM | toString) "true") (and (not (eq (.Values.deployOLM | toString) "false")) (eq $lookupLen 0))) }}

# Run copy and pasted sed command
$ find "${repo_path}/k8s/operator/helm/templates" -type f -exec sed -i "/HELM_DEPLOY_OLM_PLACEHOLDER/c\\${helm_tmpl_checks}" {} \;

$ git diff
diff --git a/k8s/operator/helm/templates/00_olm.yaml b/k8s/operator/helm/templates/00_olm.yaml
index 381c856..cd4ccafb2 100644
--- a/k8s/operator/helm/templates/00_olm.yaml
+++ b/k8s/operator/helm/templates/00_olm.yaml
@@ -1,4 +1,9 @@
-{{if .Values.deployOLM}}{{- /* HELM_DEPLOY_OLM_PLACEHOLDER */ -}}
+{{- $olmCRDFound := false }}
+{{- $nsLookup := len (lookup "v1" "Namespace" "" "") }}
+{{- range $index, $crdLookup := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "").items -}}{{ if eq $crdLookup.metadata.name "operators.operators.coreos.com"}}{{ $olmCRDFound = true }}{{ end }}{{end}}
+{{ if and (not $olmCRDFound) (not (eq $nsLookup 0))}}{{ fail "CRDs missing! Please deploy CRDs from https://github.com/pixie-io/pixie/tree/main/k8s/operator/helm/crds to continue with deploy." }}{{end}}
+{{- $lookupLen := 0 -}}{{- $opLookup := (lookup "operators.coreos.com/v1" "OperatorGroup" "" "").items -}}{{if $opLookup }}{{ $lookupLen = len $opLookup }}{{ end }}
+{{ if (or (eq (.Values.deployOLM | toString) "true") (and (not (eq (.Values.deployOLM | toString) "false")) (eq $lookupLen 0))) }}
 {{ if not (eq .Values.olmNamespace .Release.Namespace) }}
 ---
 apiVersion: v1
diff --git a/k8s/operator/helm/templates/csv-deleter.yaml b/k8s/operator/helm/templates/csv-deleter.yaml
index f785a63..5c0efb20e 100644
--- a/k8s/operator/helm/templates/csv-deleter.yaml
+++ b/k8s/operator/helm/templates/csv-deleter.yaml
@@ -1,4 +1,9 @@
-{{if .Values.deployOLM}}{{- /* HELM_DEPLOY_OLM_PLACEHOLDER */ -}}
+{{- $olmCRDFound := false }}
+{{- $nsLookup := len (lookup "v1" "Namespace" "" "") }}
+{{- range $index, $crdLookup := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "").items -}}{{ if eq $crdLookup.metadata.name "operators.operators.coreos.com"}}{{ $olmCRDFound = true }}{{ end }}{{end}}
+{{ if and (not $olmCRDFound) (not (eq $nsLookup 0))}}{{ fail "CRDs missing! Please deploy CRDs from https://github.com/pixie-io/pixie/tree/main/k8s/operator/helm/crds to continue with deploy." }}{{end}}
+{{- $lookupLen := 0 -}}{{- $opLookup := (lookup "operators.coreos.com/v1" "OperatorGroup" "" "").items -}}{{if $opLookup }}{{ $lookupLen = len $opLookup }}{{ end }}
+{{ if (or (eq (.Values.deployOLM | toString) "true") (and (not (eq (.Values.deployOLM | toString) "false")) (eq $lookupLen 0))) }}
 ---
 apiVersion: batch/v1
 kind: Job

```

Signed-off-by: Dom Del Nano <ddelnano@gmail.com>

Author: ddelnano

ci/operator_helm_build_release.sh

@@ -57,7 +57,7 @@ cp "${repo_path}/k8s/operator/crd/base/px.dev_viziers.yaml" "${helm_path}/crds/v
 
 # Updates templates with Helm-specific template functions.
 helm_tmpl_checks="$(cat "${repo_path}/k8s/operator/helm/olm_template_checks.tmpl")"
-sed -i "1c${helm_tmpl_checks}" "${repo_path}/k8s/operator/helm/templates/00_olm.yaml"
+find "${repo_path}/k8s/operator/helm/templates" -type f -exec sed -i "/HELM_DEPLOY_OLM_PLACEHOLDER/c\\${helm_tmpl_checks}" {} \;
 rm "${repo_path}/k8s/operator/helm/olm_template_checks.tmpl"
 
 # Fetch all of the current charts in GCS, because generating the index needs all pre-existing tar versions present.

k8s/operator/helm/templates/00_olm.yaml

@@ -1,4 +1,4 @@
-{{if .Values.deployOLM}}
+{{if .Values.deployOLM}}{{- /* HELM_DEPLOY_OLM_PLACEHOLDER */ -}}
 {{ if not (eq .Values.olmNamespace .Release.Namespace) }}
 ---
 apiVersion: v1

k8s/operator/helm/templates/csv-deleter.yaml

@@ -1,4 +1,4 @@
-{{if .Values.deployOLM}}
+{{if .Values.deployOLM}}{{- /* HELM_DEPLOY_OLM_PLACEHOLDER */ -}}
 ---
 apiVersion: batch/v1
 kind: Job