use websocket instead of ssh

sp-yduck · sp-yduck · commit 5a20c4ef57dd · 2023-07-31T20:45:47.000+09:00
diff --git a/api/v1beta1/proxmoxcluster_types.go b/api/v1beta1/proxmoxcluster_types.go
@@ -38,9 +38,6 @@ type ProxmoxClusterSpec struct {
 	// ServerRef is used for configuring Proxmox client
 	ServerRef ServerRef `json:"serverRef"`
 
-	// NodesRef contains reference of nodes used for ProxmoxCluster
-	NodeRefs []NodeRef `json:"nodeRefs,omitempty"`
-
 	// storage is for proxmox storage used by vm instances
 	// +optional
 	Storage Storage `json:"storage"`
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/cloud/interfaces.go b/cloud/interfaces.go
@@ -7,7 +7,6 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 	infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
-	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scope"
 )
 
 type Reconciler interface {
@@ -17,7 +16,6 @@ type Reconciler interface {
 
 type Client interface {
 	CloudClient() *proxmox.Service
-	RemoteClient() *scope.SSHClient
 }
 
 type Cluster interface {
diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go
@@ -29,7 +29,6 @@ import (
 
 type ProxmoxServices struct {
 	Compute *proxmox.Service
-	Remote  *SSHClient
 }
 
 func newComputeService(ctx context.Context, serverRef infrav1.ServerRef, crClient client.Client) (*proxmox.Service, error) {
@@ -53,30 +52,3 @@ func newComputeService(ctx context.Context, serverRef infrav1.ServerRef, crClien
 
 	return proxmox.NewService(serverRef.Endpoint, authConfig, true)
 }
-
-func newRemoteClient(ctx context.Context, secretRef *infrav1.ObjectReference, crClient client.Client) (*SSHClient, error) {
-	if secretRef == nil {
-		return nil, errors.New("failed to get proxmox client form nil secretRef")
-	}
-
-	var secret corev1.Secret
-	key := client.ObjectKey{Namespace: secretRef.Namespace, Name: secretRef.Name}
-	if err := crClient.Get(ctx, key, &secret); err != nil {
-		return nil, err
-	}
-
-	nodeurl, ok := secret.Data["NODE_URL"]
-	if !ok {
-		return nil, errors.Errorf("failed to fetch NODE_URL from Secret : %v", key)
-	}
-	nodeuser, ok := secret.Data["NODE_USER"]
-	if !ok {
-		return nil, errors.Errorf("failed to fetch PROXMOX_USER from Secret : %v", key)
-	}
-	nodepassword, ok := secret.Data["NODE_PASSWORD"]
-	if !ok {
-		return nil, errors.Errorf("failed to fetch PROXMOX_PASSWORD from Secret : %v", key)
-	}
-
-	return NewSSHClient(string(nodeurl), string(nodeuser), string(nodepassword))
-}
diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go
@@ -52,15 +52,6 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
 		params.ProxmoxServices.Compute = computeSvc
 	}
 
-	if params.ProxmoxServices.Remote == nil {
-		// current CAPPX is compatible with only single node proxmox cluster
-		remote, err := newRemoteClient(ctx, params.ProxmoxCluster.Spec.NodeRefs[0].SecretRef, params.Client)
-		if err != nil {
-			return nil, errors.Errorf("failed to create remote client: %v", err)
-		}
-		params.ProxmoxServices.Remote = remote
-	}
-
 	helper, err := patch.NewHelper(params.ProxmoxCluster, params.Client)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to init patch helper")
@@ -79,11 +70,6 @@ func populateNamespace(proxmoxCluster *infrav1.ProxmoxCluster) {
 	if proxmoxCluster.Spec.ServerRef.SecretRef.Namespace == "" {
 		proxmoxCluster.Spec.ServerRef.SecretRef.Namespace = proxmoxCluster.Namespace
 	}
-	for i, nodeRef := range proxmoxCluster.Spec.NodeRefs {
-		if nodeRef.SecretRef.Namespace == "" {
-			proxmoxCluster.Spec.NodeRefs[i].SecretRef.Namespace = proxmoxCluster.Namespace
-		}
-	}
 }
 
 type ClusterScope struct {
@@ -114,10 +100,6 @@ func (s *ClusterScope) CloudClient() *proxmox.Service {
 	return s.ProxmoxServices.Compute
 }
 
-func (s *ClusterScope) RemoteClient() *SSHClient {
-	return s.ProxmoxServices.Remote
-}
-
 func (s *ClusterScope) Close() error {
 	return s.PatchObject()
 }
diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go
@@ -82,10 +82,6 @@ func (m *MachineScope) CloudClient() *proxmox.Service {
 	return m.ClusterGetter.CloudClient()
 }
 
-func (m *MachineScope) RemoteClient() *SSHClient {
-	return m.ClusterGetter.Remote
-}
-
 func (m *MachineScope) GetStorage() infrav1.Storage {
 	return m.ClusterGetter.ProxmoxCluster.Spec.Storage
 }
diff --git a/cloud/scope/remote.go b/cloud/scope/remote.go
diff --git a/cloud/services/compute/instance/cloudinit.go b/cloud/services/compute/instance/cloudinit.go
@@ -16,9 +16,9 @@ const (
 )
 
 // reconcileCloudInit
-func (s *Service) reconcileCloudInit(bootstrap string) error {
+func (s *Service) reconcileCloudInit(ctx context.Context, bootstrap string) error {
 	// user
-	if err := s.reconcileCloudInitUser(bootstrap); err != nil {
+	if err := s.reconcileCloudInitUser(ctx, bootstrap); err != nil {
 		return err
 	}
 	return nil
@@ -42,9 +42,8 @@ func (s *Service) deleteCloudConfig(ctx context.Context) error {
 	return storage.DeleteVolume(ctx, volumeID)
 }
 
-func (s *Service) reconcileCloudInitUser(bootstrap string) error {
+func (s *Service) reconcileCloudInitUser(ctx context.Context, bootstrap string) error {
 	vmName := s.scope.Name()
-	storagePath := s.scope.GetStorage().Path
 	config := s.scope.GetCloudInit().User
 
 	bootstrapConfig, err := cloudinit.ParseUser(bootstrap)
@@ -66,13 +65,17 @@ func (s *Service) reconcileCloudInitUser(bootstrap string) error {
 	if err != nil {
 		return err
 	}
-
 	klog.Info(configYaml)
 
 	// to do: should be set via API
-	out, err := s.remote.RunWithStdin(fmt.Sprintf("tee %s/%s", storagePath, userSnippetPath(vmName)), configYaml)
+	vnc, err := s.vncClient(s.scope.NodeName())
 	if err != nil {
-		return errors.Errorf("ssh command error : %s : %v", out, err)
+		return err
+	}
+	defer vnc.Close()
+	filePath := fmt.Sprintf("%s/%s", s.scope.GetStorage().Path, userSnippetPath(vmName))
+	if err := vnc.WriteFile(context.TODO(), configYaml, filePath); err != nil {
+		return errors.Errorf("failed to write file error : %v", err)
 	}
 
 	return nil
diff --git a/cloud/services/compute/instance/image.go b/cloud/services/compute/instance/image.go
@@ -9,27 +9,20 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sp-yduck/proxmox-go/proxmox"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-
-	infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
-	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scope"
 )
 
 // reconcileBootDevice
 func (s *Service) reconcileBootDevice(ctx context.Context, vm *proxmox.VirtualMachine) error {
-	vmid := s.scope.GetVMID()
-	storage := s.scope.GetStorage()
-	image := s.scope.GetImage()
-	hardware := s.scope.GetHardware()
 	log := log.FromContext(ctx)
-	log.Info(fmt.Sprintf("%v", hardware))
+	log.Info("reconcile boot device")
 
 	// os image
-	if err := setCloudImage(ctx, *vmid, storage, image, s.remote); err != nil {
+	if err := s.setCloudImage(ctx); err != nil {
 		return err
 	}
 
 	// volume
-	if err := vm.ResizeVolume(ctx, "scsi0", hardware.Disk); err != nil {
+	if err := vm.ResizeVolume(ctx, bootDvice, s.scope.GetHardware().Disk); err != nil {
 		return err
 	}
 
@@ -38,18 +31,21 @@ func (s *Service) reconcileBootDevice(ctx context.Context, vm *proxmox.VirtualMa
 
 // setCloudImage downloads OS image into Proxmox node
 // and then sets it to specified storage
-func setCloudImage(ctx context.Context, vmid int, storage infrav1.Storage, image infrav1.Image, ssh scope.SSHClient) error {
+func (s *Service) setCloudImage(ctx context.Context) error {
 	log := log.FromContext(ctx)
 	log.Info("setting cloud image")
 
+	image := s.scope.GetImage()
 	url := image.URL
 	fileName := path.Base(url)
 	rawImageDirPath := fmt.Sprintf("%s/images", etcCAPPX)
 	rawImageFilePath := fmt.Sprintf("%s/%s", rawImageDirPath, fileName)
 
 	// workaround
 	// API does not support something equivalent of "qm importdisk"
-	out, err := ssh.RunCommand(fmt.Sprintf("wget %s --directory-prefix %s -nc", url, rawImageDirPath))
+	vnc, err := s.vncClient(s.scope.NodeName())
+	defer vnc.Close()
+	out, _, err := vnc.Exec(ctx, fmt.Sprintf("wget %s --directory-prefix %s -nc", url, rawImageDirPath))
 	if err != nil {
 		return errors.Errorf("failed to download image: %s : %v", out, err)
 	}
@@ -60,16 +56,16 @@ func setCloudImage(ctx context.Context, vmid int, storage infrav1.Storage, image
 		if err != nil {
 			return err
 		}
-		cmd := fmt.Sprintf("%s --check -", cscmd)
-		stdin := fmt.Sprintf("%s %s", image.Checksum, rawImageFilePath)
-		out, err = ssh.RunWithStdin(cmd, stdin)
+		cmd := fmt.Sprintf("echo '%s %s' | %s --check -", image.Checksum, rawImageFilePath, cscmd)
+		out, _, err = vnc.Exec(context.TODO(), cmd)
 		if err != nil {
 			return errors.Errorf("failed to confirm checksum: %s : %v", out, err)
 		}
 	}
 
-	destPath := fmt.Sprintf("%s/images/%d/vm-%d-disk-0.raw", storage.Path, vmid, vmid)
-	out, err = ssh.RunCommand(fmt.Sprintf("/usr/bin/qemu-img convert -O raw %s %s", rawImageFilePath, destPath))
+	vmid := s.scope.GetVMID()
+	destPath := fmt.Sprintf("%s/images/%d/vm-%d-disk-0.raw", s.scope.GetStorage().Path, *vmid, *vmid)
+	out, _, err = vnc.Exec(context.TODO(), fmt.Sprintf("/usr/bin/qemu-img convert -O raw %s %s", rawImageFilePath, destPath))
 	if err != nil {
 		return errors.Errorf("failed to convert iamge : %s : %v", out, err)
 	}
diff --git a/cloud/services/compute/instance/qemu.go b/cloud/services/compute/instance/qemu.go
@@ -15,6 +15,10 @@ import (
 	infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
 )
 
+const (
+	bootDvice = "scsi0"
+)
+
 func (s *Service) reconcileQEMU(ctx context.Context) (*proxmox.VirtualMachine, error) {
 	log := log.FromContext(ctx)
 	log.Info("Reconciling QEMU")
@@ -109,7 +113,7 @@ func generateVMOptions(vmName, storageName string, network infrav1.Network, hard
 		Memory:       hardware.Memory,
 		Name:         vmName,
 		NameServer:   network.NameServer,
-		Boot:         "order=scsi0",
+		Boot:         fmt.Sprintf("order=%s", bootDvice),
 		Ide:          api.Ide{Ide2: fmt.Sprintf("file=%s:cloudinit,media=cdrom", storageName)},
 		CiCustom:     fmt.Sprintf("user=%s:%s", storageName, userSnippetPath(vmName)),
 		IPConfig:     api.IPConfig{IPConfig0: network.IPConfig.String()},
diff --git a/cloud/services/compute/instance/reconcile.go b/cloud/services/compute/instance/reconcile.go
@@ -145,7 +145,7 @@ func (s *Service) createInstance(ctx context.Context, bootstrap string) (*proxmo
 	log.Info(fmt.Sprintf("reconciled qemu: node=%s,vmid=%d", instance.Node, vmid))
 
 	// cloud init
-	if err := s.reconcileCloudInit(bootstrap); err != nil {
+	if err := s.reconcileCloudInit(ctx, bootstrap); err != nil {
 		return nil, err
 	}
 
diff --git a/cloud/services/compute/instance/service.go b/cloud/services/compute/instance/service.go
@@ -1,10 +1,11 @@
 package instance
 
 import (
+	"context"
+
 	"github.com/sp-yduck/proxmox-go/proxmox"
 
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud"
-	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scope"
 )
 
 type Scope interface {
@@ -14,13 +15,19 @@ type Scope interface {
 type Service struct {
 	scope  Scope
 	client proxmox.Service
-	remote scope.SSHClient
 }
 
 func NewService(s Scope) *Service {
 	return &Service{
 		scope:  s,
 		client: *s.CloudClient(),
-		remote: *s.RemoteClient(),
 	}
 }
+
+func (s *Service) vncClient(nodeName string) (*proxmox.VNCWebSocketClient, error) {
+	client, err := s.client.NewNodeVNCWebSocketConnection(context.TODO(), nodeName)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml
diff --git a/go.mod b/go.mod
diff --git a/go.sum b/go.sum

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,6 @@ import (`
`7`	`7`	`clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"`
`8`	`8`
`9`	`9`	`infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"`
`10`		`- "github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scope"`
`11`	`10`	`)`
`12`	`11`
`13`	`12`	`type Reconciler interface {`
`@@ -17,7 +16,6 @@ type Reconciler interface {`
`17`	`16`
`18`	`17`	`type Client interface {`
`19`	`18`	`CloudClient() *proxmox.Service`
`20`		`- RemoteClient() *scope.SSHClient`
`21`	`19`	`}`
`22`	`20`
`23`	`21`	`type Cluster interface {`
Original file line number	Diff line number	Diff line change
`@@ -82,10 +82,6 @@ func (m MachineScope) CloudClient() proxmox.Service {`
`82`	`82`	`return m.ClusterGetter.CloudClient()`
`83`	`83`	`}`
`84`	`84`
`85`		`-func (m MachineScope) RemoteClient() SSHClient {`
`86`		`- return m.ClusterGetter.Remote`
`87`		`-}`
`88`		`-`
`89`	`85`	`func (m *MachineScope) GetStorage() infrav1.Storage {`
`90`	`86`	`return m.ClusterGetter.ProxmoxCluster.Spec.Storage`
`91`	`87`	`}`