Merge pull request #748 from smarterclayton/report_level_status

clusteroperator: Report when OLM reaches "level" and check syncs

Author: openshift-merge-robot

cmd/catalog/main.go

@@ -96,6 +96,6 @@ func main() {
 	http.Handle("/metrics", promhttp.Handler())
 	go http.ListenAndServe(":8081", nil)
 
-	_, done := catalogOperator.Run(stopCh)
+	_, done, _ := catalogOperator.Run(stopCh)
 	<-done
 }

cmd/olm/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"reflect"
 	"strings"
 	"time"
 
@@ -13,21 +14,20 @@ import (
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/tools/clientcmd"
 
 	configv1 "github.com/openshift/api/config/v1"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	clusteroperatorv1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
-	operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -128,98 +128,198 @@ func main() {
 	http.Handle("/metrics", promhttp.Handler())
 	go http.ListenAndServe(":8081", nil)
 
-	ready, done := operator.Run(stopCh)
+	ready, done, sync := operator.Run(stopCh)
 	<-ready
 
 	if *writeStatusName != "" {
-		opStatusGV := schema.GroupVersion{
-			Group:   "config.openshift.io",
-			Version: "v1",
+		monitorClusterStatus(sync, stopCh, opClient, configClient)
+	}
+
+	<-done
+}
+
+func monitorClusterStatus(syncCh chan error, stopCh <-chan struct{}, opClient operatorclient.ClientInterface, configClient configv1client.ConfigV1Interface) {
+	var (
+		syncs              int
+		successfulSyncs    int
+		hasClusterOperator bool
+	)
+	go wait.Until(func() {
+		// slow poll until we see a cluster operator API, which could be never
+		if !hasClusterOperator {
+			opStatusGV := schema.GroupVersion{
+				Group:   "config.openshift.io",
+				Version: "v1",
+			}
+			err := discovery.ServerSupportsVersion(opClient.KubernetesInterface().Discovery(), opStatusGV)
+			if err != nil {
+				log.Infof("ClusterOperator api not present, skipping update (%v)", err)
+				time.Sleep(time.Minute)
+				return
+			}
+			hasClusterOperator = true
 		}
-		err := discovery.ServerSupportsVersion(opClient.KubernetesInterface().Discovery(), opStatusGV)
-		if err != nil {
-			log.Infof("ClusterOperator api not present, skipping update (%v)", err)
-		} else {
-			existing, err := configClient.ClusterOperators().Get(*writeStatusName, metav1.GetOptions{})
-			if k8serrors.IsNotFound(err) {
-				log.Info("Existing operator status not found, creating")
-				created, err := configClient.ClusterOperators().Create(&configv1.ClusterOperator{
-					ObjectMeta: metav1.ObjectMeta{
-						Name: *writeStatusName,
-					},
-				})
-				if err != nil {
-					log.Fatalf("ClusterOperator create failed: %v\n", err)
+
+		// Sample the sync channel and see whether we're successfully retiring syncs as a
+		// proxy for "working" (we can't know when we hit level, but we can at least verify
+		// we are seeing some syncs succeeding). Once we observe at least one successful
+		// sync we can begin reporting available and level.
+		select {
+		case err, ok := <-syncCh:
+			if !ok {
+				// syncCh should only close if the Run() loop exits
+				time.Sleep(5 * time.Second)
+				log.Fatalf("Status sync channel closed but process did not exit in time")
+			}
+			syncs++
+			if err == nil {
+				successfulSyncs++
+			}
+			// grab any other sync events that have accumulated
+			for len(syncCh) > 0 {
+				if err := <-syncCh; err == nil {
+					successfulSyncs++
 				}
+				syncs++
+			}
+			// if we haven't yet accumulated enough syncs, wait longer
+			// TODO: replace these magic numbers with a better measure of syncs across all queueInformers
+			if successfulSyncs < 5 || syncs < 10 {
+				log.Printf("Waiting to observe more successful syncs")
+				return
+			}
+		}
 
-				created.Status = configv1.ClusterOperatorStatus{
+		// create the cluster operator in an initial state if it does not exist
+		existing, err := configClient.ClusterOperators().Get(*writeStatusName, metav1.GetOptions{})
+		if k8serrors.IsNotFound(err) {
+			log.Info("Existing operator status not found, creating")
+			created, createErr := configClient.ClusterOperators().Create(&configv1.ClusterOperator{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: *writeStatusName,
+				},
+				Status: configv1.ClusterOperatorStatus{
 					Conditions: []configv1.ClusterOperatorStatusCondition{
 						configv1.ClusterOperatorStatusCondition{
 							Type:               configv1.OperatorProgressing,
-							Status:             configv1.ConditionFalse,
-							Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
+							Status:             configv1.ConditionTrue,
+							Message:            fmt.Sprintf("Installing %s", olmversion.OLMVersion),
 							LastTransitionTime: metav1.Now(),
 						},
 						configv1.ClusterOperatorStatusCondition{
 							Type:               configv1.OperatorFailing,
 							Status:             configv1.ConditionFalse,
-							Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
 							LastTransitionTime: metav1.Now(),
 						},
 						configv1.ClusterOperatorStatusCondition{
 							Type:               configv1.OperatorAvailable,
-							Status:             configv1.ConditionTrue,
-							Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
+							Status:             configv1.ConditionFalse,
 							LastTransitionTime: metav1.Now(),
 						},
 					},
-					Versions: []configv1.OperandVersion{{
+				},
+			})
+			if createErr != nil {
+				log.Errorf("Failed to create cluster operator: %v\n", createErr)
+				return
+			}
+			existing = created
+			err = nil
+		}
+		if err != nil {
+			log.Errorf("Unable to retrieve cluster operator: %v", err)
+			return
+		}
+
+		// update the status with the appropriate state
+		previousStatus := existing.Status.DeepCopy()
+		switch {
+		case successfulSyncs > 0:
+			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
+				Type:   configv1.OperatorFailing,
+				Status: configv1.ConditionFalse,
+			})
+			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
+				Type:    configv1.OperatorProgressing,
+				Status:  configv1.ConditionFalse,
+				Message: fmt.Sprintf("Deployed %s", olmversion.OLMVersion),
+			})
+			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
+				Type:   configv1.OperatorAvailable,
+				Status: configv1.ConditionTrue,
+			})
+			// we set the versions array when all the latest code is deployed and running - in this case,
+			// the sync method is responsible for guaranteeing that happens before it returns nil
+			if version := os.Getenv("RELEASE_VERSION"); len(version) > 0 {
+				existing.Status.Versions = []configv1.OperandVersion{
+					{
 						Name:    "operator",
-						Version: olmversion.Full(),
-					}},
-				}
-				_, err = configClient.ClusterOperators().UpdateStatus(created)
-				if err != nil {
-					log.Fatalf("ClusterOperator update status failed: %v", err)
+						Version: version,
+					},
+					{
+						Name:    "operator-lifecycle-manager",
+						Version: olmversion.OLMVersion,
+					},
 				}
-			} else if err != nil {
-				log.Fatalf("ClusterOperators get failed: %v", err)
 			} else {
-				clusteroperatorv1helpers.SetStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
-					Type:               configv1.OperatorProgressing,
-					Status:             configv1.ConditionFalse,
-					Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
-					LastTransitionTime: metav1.Now(),
-				})
-				clusteroperatorv1helpers.SetStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
-					Type:               configv1.OperatorFailing,
-					Status:             configv1.ConditionFalse,
-					Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
-					LastTransitionTime: metav1.Now(),
-				})
-				clusteroperatorv1helpers.SetStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
-					Type:               configv1.OperatorAvailable,
-					Status:             configv1.ConditionTrue,
-					Message:            fmt.Sprintf("Done deploying %s.", olmversion.OLMVersion),
-					LastTransitionTime: metav1.Now(),
-				})
-
-				olmOperandVersion := configv1.OperandVersion{Name: "operator", Version: olmversion.Full()}
-				// look for operator version, even though in OLM's case should only be one
-				for _, item := range existing.Status.Versions {
-					if item.Name == "operator" && item != olmOperandVersion {
-						// if a cluster wide upgrade has occurred, hopefully any existing operator statuses have been deleted
-						log.Infof("Updating version from %v to %v\n", item.Version, olmversion.Full())
-					}
-				}
-				operatorv1helpers.SetOperandVersion(&existing.Status.Versions, olmOperandVersion)
-				_, err = configClient.ClusterOperators().UpdateStatus(existing)
-				if err != nil {
-					log.Fatalf("ClusterOperator update status failed: %v", err)
-				}
+				existing.Status.Versions = nil
+			}
+		default:
+			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
+				Type:    configv1.OperatorFailing,
+				Status:  configv1.ConditionTrue,
+				Message: "Waiting for updates to take effect",
+			})
+			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
+				Type:    configv1.OperatorProgressing,
+				Status:  configv1.ConditionFalse,
+				Message: fmt.Sprintf("Waiting to see update %s succeed", olmversion.OLMVersion),
+			})
+			// TODO: use % errors within a window to report available
+		}
+
+		// update the status
+		if !reflect.DeepEqual(previousStatus, &existing.Status) {
+			if _, err := configClient.ClusterOperators().UpdateStatus(existing); err != nil {
+				log.Errorf("Unable to update cluster operator status: %v", err)
 			}
 		}
+
+		// if we've reported success, we can sleep longer, otherwise we want to keep watching for
+		// successful
+		if successfulSyncs > 0 {
+			time.Sleep(5 * time.Minute)
+		}
+
+	}, 5*time.Second, stopCh)
+}
+
+func setOperatorStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, newCondition configv1.ClusterOperatorStatusCondition) {
+	if conditions == nil {
+		conditions = &[]configv1.ClusterOperatorStatusCondition{}
+	}
+	existingCondition := findOperatorStatusCondition(*conditions, newCondition.Type)
+	if existingCondition == nil {
+		newCondition.LastTransitionTime = metav1.NewTime(time.Now())
+		*conditions = append(*conditions, newCondition)
+		return
 	}
 
-	<-done
+	if existingCondition.Status != newCondition.Status {
+		existingCondition.Status = newCondition.Status
+		existingCondition.LastTransitionTime = newCondition.LastTransitionTime
+	}
+
+	existingCondition.Reason = newCondition.Reason
+	existingCondition.Message = newCondition.Message
+}
+
+func findOperatorStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition {
+	for i := range conditions {
+		if conditions[i].Type == conditionType {
+			return &conditions[i]
+		}
+	}
+
+	return nil
 }

cmd/package-server/apiserver.local.config/certificates/apiserver.crt

@@ -0,0 +1,37 @@
+-----BEGIN CERTIFICATE-----
+MIIDGDCCAgCgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdsb2Nh
+bGhvc3QtY2FAMTU1MjQyMzAyMDAeFw0xOTAzMTIyMDM3MDFaFw0yMDAzMTEyMDM3
+MDFaMB8xHTAbBgNVBAMMFGxvY2FsaG9zdEAxNTUyNDIzMDIxMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAubkXRqN2xYxJiVhMjHnOtPCkU44QcLosVpIj
+tbUgzjJt0BDv/XNCMhbpD3dfKjMKZiKXt1dKDK2Tl52AceWqipVQlCf7kiX+CjuO
+gTAIEbVC7FWdu/sDI8BWbhs5knT+8Y7a5uGVexclZifvcbASuVtedLH47XI25Ak4
+s103Usy5Z2WXOLd79w/tsAr1kvQzveIdbn+upMu4to2wmfXhiLaU2qMhGoz+2hzm
+z+SXkB7uCgFbGuLIUj99/faSZ3CAH6EwPIerAKtY+1hdVmsjqpIrSs4jD7YyfmVN
+3+/MLTSMyHrghHYKt/SiRdCuVrbMhCylU8NFry+iuBIsOA202QIDAQABo1wwWjAO
+BgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw
+ADAlBgNVHREEHjAcgglsb2NhbGhvc3SCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG
+9w0BAQsFAAOCAQEAacr9G8nNsHQpLCW+0meGmDz9deTfLYldFCbCjsPiUDWs9tUn
+O+04ykac2tEqZt2Ovkp6gntRPBCOKpgwHYvo0CJtCaL4yh6wYMvlbjHmHR/y+Ioy
+HymMmaQ06iVIhb2KoKFJvFtFUVNg6QE9w7dm9/C73eHcv3JhqYhGw3qBfUI6lmIc
+lWGj6WGVNfslofTYMkshbRGNZ3gFGkvcQvPOhKb/K4A3X9ZTGy9XyydVAOpdk/5n
+FBD4gOJJVSq2jJ5SOTJd5Z/YrY2tbCfZeuuPuxBK4XG3hnLN2fk9URwfCDc9EUQg
+aYagxskTB6jaDkFD5lfXxEc3W+/mP62i7mH/fQ==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIC4jCCAcqgAwIBAgIBATANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBdsb2Nh
+bGhvc3QtY2FAMTU1MjQyMzAyMDAeFw0xOTAzMTIyMDM3MDBaFw0yMDAzMTEyMDM3
+MDBaMCIxIDAeBgNVBAMMF2xvY2FsaG9zdC1jYUAxNTUyNDIzMDIwMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQb0E1iZ/R1J8bdzDP/EFx73JpU6fw6T
+aTY9QTWWgt4EcamLpJK5Z+dOLhj/i6rQbe/vKpI6BbBo+S6MuBemyUbc4VpoTde6
+Hn26uWSlkQA72GLHYWvD+ahdRpLxOFddog9xcfEoYN/rlpwMp030y6clQhrb4WML
+x1uQzqyOvzRHAN4NqxmLXbepTyWqiM3tLe2f4mPfcg/vhwQ5TSqR/Rm3FPh3rDdA
+zvk9bGkvyX8iAUoLw/0aHe2dzTfnvBvkTJFEaLq61FLQ/zfMVRhPI2Fwljxq+jSq
+FoYju/vr1sWxKc+AFxDdAZdRey2Afi1bVf8JHiDU8FSe9UcfqBUoyQIDAQABoyMw
+ITAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF
+AAOCAQEAmrIS4kJNVjKj4vSj0lNWzOjk31CI26rKwPo+cFhvnPh+eg6wI+3I/gLC
+yf9X5KIPaNS5MGzNEmpr7Ml7IviqUn8rSoVryoQwKtqnMhsGr3/Y/Rrd27OIYEW+
+6/phRyI2rM8Vzo0RVdqcQT+6qvknbZ4fr/3Or3YbjycyfqNeL0SzXff+c8s9skDw
+r9OV5uMvmVJv3VNBhAEX83I4zJsfrH9XtAmz255aw24vBGMUHYEdH15K/IBxh4LZ
+Y5AXZhVazjlzwWwnUpu8k88vesCUay8c4VtXfXHQTk/oS/ZDn7eQ7hTvzqYfEH2k
+znJYRthnuUZo6M/rtMWzXK6QuunRtg==
+-----END CERTIFICATE-----

cmd/package-server/apiserver.local.config/certificates/apiserver.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAubkXRqN2xYxJiVhMjHnOtPCkU44QcLosVpIjtbUgzjJt0BDv
+/XNCMhbpD3dfKjMKZiKXt1dKDK2Tl52AceWqipVQlCf7kiX+CjuOgTAIEbVC7FWd
+u/sDI8BWbhs5knT+8Y7a5uGVexclZifvcbASuVtedLH47XI25Ak4s103Usy5Z2WX
+OLd79w/tsAr1kvQzveIdbn+upMu4to2wmfXhiLaU2qMhGoz+2hzmz+SXkB7uCgFb
+GuLIUj99/faSZ3CAH6EwPIerAKtY+1hdVmsjqpIrSs4jD7YyfmVN3+/MLTSMyHrg
+hHYKt/SiRdCuVrbMhCylU8NFry+iuBIsOA202QIDAQABAoIBAEqc4o39c+TvdEea
+Ur6I3RNyLgJna5FuKgvpkDEbAH/2YImblF7VZD2tWJpfEbtpX/8iXKNKjTREs6vQ
+md6oLviX/hRXb8kKPGIuBRU/j65VjPpXdxQjRuKhDdgUVe/R0u6GvsjMzfnylZLR
+7m9VFmCjJXJqYaA7J3Q7hC0DAQvhBiWk0lZHR7cjGeG37fIT2yzH7gf8M4VeYjCn
+asatNUuAOORVfGudtKLCgFk/bmO1Nb5UwCYcz4OXVEpDBWrcg1SsvYwKxyUDxO8a
+8A7TAWWEXjWK+sPmaJkUzRfnd/1chvlzcaawXfgfXRHcAaLWRaBu4fdYS7fwMYy6
++/0Pa5ECgYEA0GCkaAl7qicfHTY6xTkBvJwkXu/rDIfzJCRVtdlXOhPmJ+F3+0Rj
+0d+O6LMNSyJpYdOYeWOJbjHMJ92XIRJVxqF+K2O6dToEMTG2XbqMm2gtyn16BoTt
+ngzcWqeo+zqwvHxLcM6L/tjivnbsI7mVDpdcBJZwVd6VwrR2NgRh0tUCgYEA5CsF
+rJUlOR3JJ1CUTrT1G4smBES00lL3QFlhkiF4zWOW6NwhswZlYPkzqe6tgxmtGAuQ
+mJINMcqWUkU18BWLh8RRTH+oKcUbmZkTqP9k/bqe6foIm8UyxVsSF80S4tRtMcWm
+87Nd2h+FbYY2MP9RFscdDDd5FHf+weSCbnn0s/UCgYEAz05WQeqtTSp+meFJtsxw
+HeR5irnFbkIScvJzEueXEACcCTEW3LO9Wx6+XmND5mvly51nI90S7L4+Das2n4BO
+Nb6UdzZQWi/N2+NJOxZMrI+Ifts2eyXkAElrMAV85/QLwHkn1KKoRHIhortNUn1e
+/ZU3xpikScmX1I0UzciuScECgYAbWrEOdL8GrvR7uyRcn0M3byI6psYK5RlxZIXX
+EB48eXERL7r2jJDA5H92IwA4VG61EEXglLnyOzh0WonR47NbroSUqEVP5KqfaoO5
+4gyIgsQkhu5bRnQExxtPMS3Pdeo1al3On7Vjvh2v+MQscZ+WHH72BPyGILCxLCUa
++5IDtQKBgGE1Wl2dmdAyedzCX93oOjnVQ2xdH4s+4k7yHBYEt9AIzbuZCSZLMsf+
+hDoU/TokDRXkrHnRvZvhpljgjRJULktnmZxRWW8e/YXrp+gTvSq7/bZCob8Dgs80
+w21YuIgo6sXV2uvqGUbZ3YvJQU0GnoFB/GztGlmuVyU0jpsJKq5P
+-----END RSA PRIVATE KEY-----

manifests/0000_50_olm_06-olm-operator.deployment.yaml

@@ -38,6 +38,8 @@ spec:
               path: /healthz
               port: 8080
           env:
+          - name: RELEASE_VERSION
+            value: "0.0.1-snapshot"
           - name: OPERATOR_NAMESPACE
             valueFrom:
               fieldRef:

manifests/0000_50_olm_14-operatorstatus.yaml

@@ -3,3 +3,7 @@ apiVersion: config.openshift.io/v1
 kind: ClusterOperator
 metadata:
   name: operator-lifecycle-manager
+status:
+  versions:
+  - name: operator
+    version: "0.0.1-snapshot"