feat: provider id changes & add microvm proxy

Changed the machine reconciliation so that it uses the standard format
for the provider id with uses a prefix. The prefix for this provider is
`microvm://`. The provider id is parsed when looking up the status of
the microvm.

Additionally, the ability to explicitly set a proxy server for the
microvm service has been added to the APIs. This is for scenarios where
you don't want to use the HTTP[S]_PROXY env vars.

Signed-off-by: Richard Case <[email protected]>

api/v1alpha1/microvmcluster_types.go

@@ -25,6 +25,9 @@ type MicrovmClusterSpec struct {
 	// Placement specifies how machines for the cluster should be placed onto hosts (i.e. where the microvms are created).
 	// +kubebuilder:validation:Required
 	Placement Placement `json:"placement"`
+	// MicrovmProxy is the proxy server details to use when calling the microvm service. This is an
+	// alteranative to using the http proxy environment variables and applied purely to the grpc service.
+	MicrovmProxy *Proxy `json:"microvmProxy,omitempty"`
 }
 
 // MicrovmClusterStatus defines the observed state of MicrovmCluster.

api/v1alpha1/types.go

@@ -163,3 +163,9 @@ type MicrovmHost struct {
 	// +kubebuilder:default=true
 	ControlPlaneAllowed bool `json:"controlplaneAllowed"`
 }
+
+// Proxy represents a proxy server.
+type Proxy struct {
+	// Endpoint is the address of the proxy.
+	Endpoint string `json:"endpoint"`
+}

config/crd/bases/infrastructure.cluster.x-k8s.io_microvmclusters.yaml

@@ -69,6 +69,18 @@ spec:
                 - host
                 - port
                 type: object
+              microvmProxy:
+                description: MicrovmProxy is the proxy server details to use when
+                  calling the microvm service. This is an alteranative to using the
+                  http proxy environment variables and applied purely to the grpc
+                  service.
+                properties:
+                  endpoint:
+                    description: Endpoint is the address of the proxy.
+                    type: string
+                required:
+                - endpoint
+                type: object
               placement:
                 description: Placement specifies how machines for the cluster should
                   be placed onto hosts (i.e. where the microvms are created).

controllers/helpers_test.go

@@ -6,6 +6,7 @@ package controllers_test
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 
 	. "github.com/onsi/gomega"
 	"gopkg.in/yaml.v2"
@@ -29,7 +30,7 @@ import (
 	"github.com/weaveworks/cluster-api-provider-microvm/internal/services/microvm/mock_client"
 	flintlockv1 "github.com/weaveworks/flintlock/api/services/microvm/v1alpha1"
 	flintlocktypes "github.com/weaveworks/flintlock/api/types"
-	"github.com/weaveworks/flintlock/client/cloudinit"
+	"github.com/weaveworks/flintlock/client/cloudinit/userdata"
 )
 
 const (
@@ -86,7 +87,7 @@ func (co clusterObjects) AsRuntimeObjects() []runtime.Object {
 func reconcileMachine(client client.Client, mockAPIClient microvm.Client) (ctrl.Result, error) {
 	machineController := &controllers.MicrovmMachineReconciler{
 		Client: client,
-		MvmClientFunc: func(address string) (microvm.Client, error) {
+		MvmClientFunc: func(address string, proxy *infrav1.Proxy) (microvm.Client, error) {
 			return mockAPIClient, nil
 		},
 	}
@@ -363,7 +364,8 @@ func assertMachineReconciled(g *WithT, reconciled *infrav1.MicrovmMachine) {
 	assertMachineVMState(g, reconciled, infrav1.VMStateRunning)
 	assertMachineFinalizer(g, reconciled)
 	g.Expect(reconciled.Spec.ProviderID).ToNot(BeNil())
-	g.Expect(*reconciled.Spec.ProviderID).To(Equal(testMachineUID))
+	expectedProviderID := fmt.Sprintf("microvm://%s", testMachineUID)
+	g.Expect(*reconciled.Spec.ProviderID).To(Equal(expectedProviderID))
 	g.Expect(reconciled.Status.Ready).To(BeTrue(), "The Ready property must be true when the machine has been reconciled")
 }
 
@@ -401,7 +403,7 @@ func assertVendorData(g *WithT, vendorDataRaw string, expectedSSHKey string) {
 	g.Expect(err).NotTo(HaveOccurred(), "expect vendor data to be base64 encoded")
 
 	if expectedSSHKey != "" {
-		vendorData := &cloudinit.UserData{}
+		vendorData := &userdata.UserData{}
 
 		unmarshallErr := yaml.Unmarshal(data, vendorData)
 		g.Expect(unmarshallErr).NotTo(HaveOccurred(), "expect vendor data to unmarshall to cloud-init userdata")

controllers/microvmmachine_controller.go

@@ -243,7 +243,9 @@ func (r *MicrovmMachineReconciler) reconcileNormal(
 
 	var microvm *flintlocktypes.MicroVM
 
-	if machineScope.MvmMachine.Spec.ProviderID != nil {
+	providerID := machineScope.GetProviderID()
+
+	if providerID != "" {
 		var err error
 
 		microvm, err = mvmSvc.Get(ctx)
@@ -302,7 +304,7 @@ func (r *MicrovmMachineReconciler) getMicrovmService(machineScope *scope.Machine
 		return nil, err
 	}
 
-	client, err := r.MvmClientFunc(addr)
+	client, err := r.MvmClientFunc(addr, machineScope.MvmCluster.Spec.MicrovmProxy)
 	if err != nil {
 		return nil, fmt.Errorf("creating microvm client: %w", err)
 	}

controllers/microvmmachine_controller_test.go

@@ -6,6 +6,7 @@ package controllers_test
 import (
 	"encoding/base64"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -274,7 +275,8 @@ func TestMachineReconcileNoVmCreateSucceeds(t *testing.T) {
 	reconciled, err := getMicrovmMachine(client, testMachineName, testClusterNamespace)
 	g.Expect(err).NotTo(HaveOccurred(), "Getting microvm machine should not fail")
 
-	g.Expect(reconciled.Spec.ProviderID).To(Equal(pointer.String(testMachineUID)))
+	expectedProviderID := fmt.Sprintf("microvm://%s", testMachineUID)
+	g.Expect(reconciled.Spec.ProviderID).To(Equal(pointer.String(expectedProviderID)))
 	g.Expect(reconciled.Spec.FailureDomain).To(Equal(pointer.String("127.0.0.1:9090")))
 
 	assertConditionFalse(g, reconciled, infrav1.MicrovmReadyCondition, infrav1.MicrovmPendingReason)

docs/compatibility.md

@@ -2,8 +2,9 @@
 
 This table shows the compatibility between CAPMVM and Flintlock versions.
 
-| CAPMVM      | Flintlock |
-| ----------- | ----------- |
+| CAPMVM      | Flintlock              |
+| ----------- | ---------------------- |
+| `v0.4.0`    | `v0.1.0-alpha.8`       |
 | `v0.3.0`    | `v0.1.0-alpha.6`       |
 | `v0.2.2`    | `v0.1.0-alpha.5`       |
 

go.mod

@@ -4,12 +4,12 @@ go 1.17
 
 require (
 	github.com/go-logr/logr v0.4.0
-	github.com/onsi/gomega v1.17.0
+	github.com/onsi/gomega v1.18.1
 	github.com/spf13/pflag v1.0.5
 	github.com/weaveworks/flintlock/api v0.0.0-20220126090930-054c6c3be154
-	github.com/weaveworks/flintlock/client v0.0.0-20220126090930-054c6c3be154
+	github.com/weaveworks/flintlock/client v0.0.0-20220221140503-8032d3ff7d2c
 	github.com/yitsushi/macpot v1.0.2
-	google.golang.org/grpc v1.42.0
+	google.golang.org/grpc v1.44.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.22.4
@@ -63,7 +63,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect

go.sum

@@ -283,6 +283,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -438,13 +439,16 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
 github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -546,6 +550,8 @@ github.com/weaveworks/flintlock/api v0.0.0-20220126090930-054c6c3be154 h1:ReCeLg
 github.com/weaveworks/flintlock/api v0.0.0-20220126090930-054c6c3be154/go.mod h1:RFgQ7RSa7zGNxxR+dS6NRDCQ/IAN23WCfXg4+L6fclI=
 github.com/weaveworks/flintlock/client v0.0.0-20220126090930-054c6c3be154 h1:LbMhm60d/6/MkhwXSJaf960JcNK986r5YzC75vNqylc=
 github.com/weaveworks/flintlock/client v0.0.0-20220126090930-054c6c3be154/go.mod h1:1jiepUOR2kxAdoxXXyNQ0LnOeT2gOUSnWGuXh9akjDw=
+github.com/weaveworks/flintlock/client v0.0.0-20220221140503-8032d3ff7d2c h1:997um3xki94Qmho3WiuuzfFGsxCn96X7T4PAfLR33kE=
+github.com/weaveworks/flintlock/client v0.0.0-20220221140503-8032d3ff7d2c/go.mod h1:4G/IetnXREmEo1agCWLwYScjw4+ElzKcDLjJ4seZJ6k=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -798,6 +804,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -1009,6 +1017,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
+google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

internal/scope/machine.go

@@ -6,12 +6,14 @@ package scope
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/klogr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/noderefutil"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -25,6 +27,8 @@ import (
 
 var _ Scoper = &MachineScope{}
 
+const ProviderPrefix = "microvm://"
+
 type MachineScopeParams struct {
 	Cluster        *clusterv1.Cluster
 	MicroVMCluster *infrav1.MicrovmCluster
@@ -113,7 +117,7 @@ type MachineScope struct {
 // UID returns the MicrovmMachine UID/ProviderID.
 func (m *MachineScope) UID() string {
 	if m.MvmMachine.Spec.ProviderID != nil {
-		return *m.MvmMachine.Spec.ProviderID
+		return strings.ReplaceAll(*m.MvmMachine.Spec.ProviderID, ProviderPrefix, "")
 	}
 
 	return ""
@@ -131,7 +135,7 @@ func (m *MachineScope) Namespace() string {
 
 // ClusterName returns the name of the cluster.
 func (m *MachineScope) ClusterName() string {
-	return m.Cluster.ClusterName
+	return m.Cluster.Name
 }
 
 // ControllerName returns the name of the controller that created the scope.
@@ -244,7 +248,28 @@ func (m *MachineScope) SetNotReady(
 
 // SetProviderID saves the unique microvm and object ID to the MvmMachine spec.
 func (m *MachineScope) SetProviderID(mvmUID *string) {
-	m.MvmMachine.Spec.ProviderID = mvmUID
+	providerID := fmt.Sprintf("%s%s", ProviderPrefix, *mvmUID)
+	m.MvmMachine.Spec.ProviderID = &providerID
+}
+
+// GetProviderID returns the provider if for the machine. If there is no provider id
+// then an empty string will be returned.
+func (m *MachineScope) GetProviderID() string {
+	if m.MvmMachine.Spec.ProviderID != nil {
+		return *m.MvmMachine.Spec.ProviderID
+	}
+
+	return ""
+}
+
+// GetInstanceID gets the instance ID (i.e. UID) of the machine.
+func (m *MachineScope) GetInstanceID() string {
+	parsed, err := noderefutil.NewProviderID(m.GetProviderID())
+	if err != nil {
+		return ""
+	}
+
+	return parsed.ID()
 }
 
 // SetFailureDomain saves the microvm host address to the MvmMachine spec.

internal/services/microvm/service.go

@@ -10,12 +10,14 @@ import (
 
 	flintlockv1 "github.com/weaveworks/flintlock/api/services/microvm/v1alpha1"
 	flintlocktypes "github.com/weaveworks/flintlock/api/types"
-	"github.com/weaveworks/flintlock/client/cloudinit"
+	"github.com/weaveworks/flintlock/client/cloudinit/instance"
+	"github.com/weaveworks/flintlock/client/cloudinit/userdata"
 	"github.com/yitsushi/macpot"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"gopkg.in/yaml.v2"
 	"k8s.io/utils/pointer"
 
+	infrav1 "github.com/weaveworks/cluster-api-provider-microvm/api/v1alpha1"
 	"github.com/weaveworks/cluster-api-provider-microvm/internal/defaults"
 	"github.com/weaveworks/cluster-api-provider-microvm/internal/scope"
 )
@@ -24,7 +26,7 @@ const (
 	cloudInitHeader = "#cloud-config\n"
 )
 
-type ClientFactoryFunc func(address string) (Client, error)
+type ClientFactoryFunc func(address string, proxy *infrav1.Proxy) (Client, error)
 
 type Client interface {
 	flintlockv1.MicroVMClient
@@ -89,7 +91,7 @@ func (s *Service) Get(ctx context.Context) (*flintlocktypes.MicroVM, error) {
 		Info("Getting microvm for machine", "machine-name", s.scope.Name(), "cluster-name", s.scope.ClusterName())
 
 	input := &flintlockv1.GetMicroVMRequest{
-		Uid: s.scope.UID(),
+		Uid: s.scope.GetInstanceID(),
 	}
 
 	resp, err := s.client.GetMicroVM(ctx, input)
@@ -139,7 +141,7 @@ func (s *Service) addMetadata(apiMicroVM *flintlocktypes.MicroVMSpec) error {
 
 func (s *Service) createVendorData() (string, error) {
 	// TODO: remove the boot command temporary fix after image-builder change #89
-	vendorUserdata := &cloudinit.UserData{
+	vendorUserdata := &userdata.UserData{
 		HostName:     s.scope.MvmMachine.Name,
 		FinalMessage: "The Liquid Metal booted system is good to go after $UPTIME seconds",
 		BootCommands: []string{
@@ -150,10 +152,10 @@ func (s *Service) createVendorData() (string, error) {
 	// TODO:  allow setting multiple keys #88
 	machineSSHKey := s.scope.GetSSHPublicKey()
 	if machineSSHKey != "" {
-		defaultUser := cloudinit.User{
+		defaultUser := userdata.User{
 			Name: "ubuntu",
 		}
-		rootUser := cloudinit.User{
+		rootUser := userdata.User{
 			Name: "root",
 		}
 
@@ -164,7 +166,7 @@ func (s *Service) createVendorData() (string, error) {
 			machineSSHKey,
 		}
 
-		vendorUserdata.Users = []cloudinit.User{defaultUser, rootUser}
+		vendorUserdata.Users = []userdata.User{defaultUser, rootUser}
 	}
 
 	data, err := yaml.Marshal(vendorUserdata)
@@ -178,12 +180,11 @@ func (s *Service) createVendorData() (string, error) {
 }
 
 func (s *Service) createInstanceData() (string, error) {
-	userMetadata := cloudinit.Metadata{
-		InstanceID:    fmt.Sprintf("%s/%s", s.scope.Namespace(), s.scope.Name()),
-		LocalHostname: s.scope.Name(),
-		Platform:      platformLiquidMetal,
-		ClusterName:   s.scope.ClusterName(),
-	}
+	userMetadata := instance.New(
+		instance.WithLocalHostname(s.scope.Name()),
+		instance.WithPlatform(platformLiquidMetal),
+		instance.WithClusterName(s.scope.ClusterName()),
+	)
 
 	userMeta, err := yaml.Marshal(userMetadata)
 	if err != nil {