diff --git a/.gitignore b/.gitignore index 41e3565d..a0fc4191 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,13 @@ __debug_* test/e2e/inventory* test/e2e/sa-key* test/e2e/kubeconfig* + +# kubetest2 testing +_rundir +_artifacts* +credentials.json + +# binaries +/cloud-controller-manager +/stackit-csi-plugin +/kubetest2-stackit diff --git a/.ko.yaml b/.ko.yaml index 17989a4e..4a5bf283 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -4,4 +4,5 @@ builds: ldflags: - -s -w - -X github.com/stackitcloud/cloud-provider-stackit/pkg/version.Version={{.Env.VERSION}} + - -X github.com/stackitcloud/cloud-provider-stackit/pkg/csi.DriverName={{.Env.DRIVER_NAME}} diff --git a/Makefile b/Makefile index 0466e1e5..f866772f 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ REGISTRY ?= ghcr.io REPO ?= stackitcloud/cloud-provider-stackit PLATFORMS ?= amd64 arm64 IS_DEV ?= true +DRIVER_NAME ?= .PHONY: all all: verify @@ -19,10 +20,12 @@ include ./hack/tools.mk build: $(BUILD_IMAGES) +stackit-csi-plugin: LDFLAGS = $(if $(DRIVER_NAME),-X github.com/stackitcloud/cloud-provider-stackit/pkg/csi.DriverName=$(DRIVER_NAME)) + $(BUILD_IMAGES): $(SOURCES) CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=${GOPROXY} go build \ -trimpath \ - -ldflags $(LDFLAGS) \ + $(if $(LDFLAGS),-ldflags '$(LDFLAGS)') \ -o $@ \ cmd/$@/main.go @@ -40,6 +43,7 @@ image-%: $(APKO) $(KO) APKO_EXTRA_PACKAGES=$(APKO_EXTRA_PACKAGES) \ LOCAL=$(LOCAL) \ VERSION=$(VERSION) \ + DRIVER_NAME=$(DRIVER_NAME) \ PLATFORMS="$(PLATFORMS)" \ REGISTRY=$(REGISTRY) \ REPO=$(REPO)/$* \ @@ -61,7 +65,7 @@ modules: ## Runs go mod to ensure modules are up to date. .PHONY: test test: ## Run tests. - ./hack/test.sh ./cmd/... ./pkg/... + ./hack/test.sh ./cmd/... ./pkg/... ./test/... .PHONY: test-cover test-cover: ## Run tests with coverage. @@ -132,6 +136,7 @@ mocks: $(MOCKGEN) # API mocks @$(MOCKGEN) -destination ./pkg/mock/loadbalancer/loadbalancer.go -package loadbalancer github.com/stackitcloud/stackit-sdk-go/services/loadbalancer/v2api DefaultAPI @$(MOCKGEN) -destination ./pkg/mock/iaas/iaas.go -package iaas github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api DefaultAPI + @$(MOCKGEN) -destination ./pkg/mock/ske/ske.go -package ske github.com/stackitcloud/stackit-sdk-go/services/ske/v2api DefaultAPI # client mocks @$(MOCKGEN) -destination ./pkg/stackit/metadata/metadata_mock.go -package metadata ./pkg/stackit/metadata IMetadata @@ -139,6 +144,7 @@ mocks: $(MOCKGEN) @$(MOCKGEN) -destination ./pkg/stackit/client/mock/iaas_mock.go -typed -package client ./pkg/stackit/client IaaSClient @$(MOCKGEN) -destination ./pkg/stackit/client/mock/loadbalancer_mock.go -typed -package client ./pkg/stackit/client LoadBalancingClient + @$(MOCKGEN) -destination ./pkg/stackit/client/mock/ske_mock.go -typed -package client ./pkg/stackit/client SKEClient @$(MOCKGEN) -destination ./pkg/stackit/client/mock/mock.go -package client ./pkg/stackit/client Factory .PHONY: generate diff --git a/cmd/kubetest2-stackit/main.go b/cmd/kubetest2-stackit/main.go new file mode 100644 index 00000000..26906755 --- /dev/null +++ b/cmd/kubetest2-stackit/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/stackitcloud/cloud-provider-stackit/test/e2e/kubetest2-stackit/deployer" + "sigs.k8s.io/kubetest2/pkg/app" +) + +func main() { + app.Main(deployer.Name, deployer.New) +} diff --git a/cmd/stackit-csi-plugin/apko-base-image.yaml b/cmd/stackit-csi-plugin/apko-base-image.yaml index 9db9604e..f61a6b7d 100644 --- a/cmd/stackit-csi-plugin/apko-base-image.yaml +++ b/cmd/stackit-csi-plugin/apko-base-image.yaml @@ -6,20 +6,20 @@ contents: packages: - busybox # required by fsck.xfs as it is a sh script - ca-certificates - - blkid=~2.41.2 - - blockdev=~2.41.2 - - e2fsprogs=~1.47.3 - - e2fsprogs-extra=~1.47.3 - - lsblk=~2.41.2 - - mount=~2.41.2 - - umount=~2.41.2 - - btrfs-progs=~6.17 - - udev=~258.1 - - util-linux-misc=~2.41.2 # contains fsck - - xfsprogs=~6.17.0 - - xfsprogs-core=~6.17.0 - - xfsprogs-extra=~6.17.0 # contains xfs_io - - findmnt=~2.41.2 + - blkid=~2.42.2 + - blockdev=~2.42.2 + - btrfs-progs=~7.1 + - e2fsprogs=~1.47.4 + - e2fsprogs-extra=~1.47.4 + - lsblk=~2.42.2 + - mount=~2.42.2 + - umount=~2.42.2 + - udev=~261.2 + - util-linux-misc=~2.42.2 # contains fsck + - xfsprogs=~7.1.1 + - xfsprogs-core=~7.1.1 + - xfsprogs-extra=~7.1.1 # contains xfs_io + - findmnt=~2.42.2 accounts: run-as: root diff --git a/cmd/stackit-csi-plugin/main.go b/cmd/stackit-csi-plugin/main.go index 302aee97..12658ee5 100644 --- a/cmd/stackit-csi-plugin/main.go +++ b/cmd/stackit-csi-plugin/main.go @@ -104,9 +104,10 @@ func handle(ctx context.Context) { } // Initialize cloud driverOpts := &blockstorage.DriverOpts{ - Endpoint: endpoint, - ClusterID: cluster, - PVCLister: csi.GetPVCLister(), + Endpoint: endpoint, + ClusterID: cluster, + DriverName: csi.DriverName, + PVCLister: csi.GetPVCLister(), } if legacyStorageMode { diff --git a/deploy/csi-plugin/controllerplugin.yaml b/deploy/csi-plugin/controllerplugin.yaml index 0b2dab96..60af067f 100644 --- a/deploy/csi-plugin/controllerplugin.yaml +++ b/deploy/csi-plugin/controllerplugin.yaml @@ -22,6 +22,8 @@ spec: labels: app: csi-stackit-controllerplugin spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet serviceAccount: csi-stackit-controller-sa containers: - name: csi-attacher @@ -64,7 +66,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: Always + imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir diff --git a/docs/testing.md b/docs/testing.md index 97000ba2..314cf3d0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -9,6 +9,7 @@ - [Accessing the Cluster](#accessing-the-cluster) - [Destroying the Cluster](#destroying-the-cluster) - [Testing Custom Branches or Images](#testing-custom-branches-or-images) +- [Building the kubetest2 STACKIT Deployer](#building-the-kubetest2-stackit-deployer) - [Running End-to-End (E2E) Tests for the CSI Driver](#running-end-to-end-e2e-tests-for-the-csi-driver) - [Parallel E2E Test Suite](#parallel-e2e-test-suite) - [Sequential E2E Test Suite (Snapshots & Backups)](#sequential-e2e-test-suite-snapshots--backups) @@ -38,7 +39,6 @@ Before running the script, you must have the following installed and configured - You must be authenticated. Run `stackit auth login` if you haven't already. 2. **jq:** The `jq` command-line JSON processor. 3. **SSH Key Pair:** The script needs an SSH key pair to access the VM. - - By default, it looks for `$HOME/.ssh/stackit-ccm-test.pub` and `$HOME/.ssh/stackit-ccm-test`. - You can generate a new key pair with: @@ -171,6 +171,18 @@ E2E_DEPLOY_CCM_IMAGE="ghcr.io/stackitcloud/cloud-provider-stackit/cloud-controll --kubernetes-version ``` +## Building the kubetest2 STACKIT Deployer + +The repository now includes a local `kubetest2-stackit` deployer binary for provisioning a single STACKIT SKE cluster during a `kubetest2` run. + +Build it with: + +```bash +make build +``` + +The binary is written to the repository root as `./kubetest2-stackit` alongside the existing controller-manager and CSI binaries. + ## Running End-to-End (E2E) Tests for the CSI Driver The CSI E2E test suite validates the full functionality of the CSI driver. Tests are divided into parallel and sequential execution sets to accommodate different operational requirements, particularly for stateful operations like snapshots. diff --git a/go.mod b/go.mod index bb8e9dee..a4ef770f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes-csi/csi-lib-utils v0.24.0 github.com/kubernetes-csi/csi-test/v5 v5.5.0 + github.com/octago/sflags v0.3.1 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 github.com/prometheus/client_golang v1.24.0 @@ -17,6 +18,7 @@ require ( github.com/stackitcloud/stackit-sdk-go/core v0.26.0 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.13.0 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.0 + github.com/stackitcloud/stackit-sdk-go/services/ske v1.17.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -31,6 +33,8 @@ require ( k8s.io/klog/v2 v2.140.0 k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 + sigs.k8s.io/yaml v1.6.0 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -81,6 +85,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/term v0.5.2 // indirect @@ -136,5 +141,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 14a10650..77b113e2 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= @@ -153,6 +155,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/octago/sflags v0.3.1 h1:LW65z20iAQKteEyjsnnc+/lyoCUnIoRuAocggr6RB6A= +github.com/octago/sflags v0.3.1/go.mod h1:hVUkbnYwMU9kZiZJyOAIVN56YiVMMPxgJ46kRZ19jh0= github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= @@ -190,6 +194,8 @@ github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.0 h1:YWpy3VDK github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.0/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.0 h1:JPP6a0ME1tZXr4iB69d/LtJsCAr58ENBadFaK9f48/c= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.0/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.17.0 h1:fQFPbm5cNsTxFy86EDiVBXcxM+7TzXtN44arGYxQWgI= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.17.0/go.mod h1:TbqmZhLMofmfl+HhVl6oHYcI3zvXTm1vRjN3A/fOkM4= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -369,6 +375,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/parallel.sh b/parallel.sh new file mode 100644 index 00000000..0dc7ce6c --- /dev/null +++ b/parallel.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export STACKIT_SERVICE_ACCOUNT="$(cat ./credentials.json)" + +run_id="stackit-parallel" +cluster_name="kt2-par" +kubernetes_version="1.35.6" +rundir_root="${PWD}/_rundir" +artifacts_root="${PWD}/_artifacts-parallel" +testdriver="${rundir_root}/${run_id}/csi-testdriver.yaml" +parallel_nodes="$(getconf _NPROCESSORS_ONLN 2>/dev/null || true)" + +if [[ ! "${parallel_nodes}" =~ ^[0-9]+$ ]] || (( parallel_nodes < 1 )); then + parallel_nodes=4 +fi + +go run ./cmd/kubetest2-stackit \ + --run-id "${run_id}" \ + --cluster-name "${cluster_name}" \ + --rundir "${rundir_root}" \ + --artifacts "${artifacts_root}" \ + --up \ + --test=ginkgo \ + --project-id e928aade-ce15-4188-8230-c162c8fb3bd4 \ + --region eu01 \ + --kubernetes-version "${kubernetes_version}" \ + --availability-zone eu01-1 \ + --machine-type c2i.2 \ + --node-image-name flatcar \ + --node-image-version 4593.2.3-containerd2.1.9 \ + --csi-image-name ttl.sh/csi-plugin/stackit-csi-plugin \ + --csi-image-tag 5h \ + -- \ + --test-package-version="v${kubernetes_version}" \ + --focus-regex="External.Storage" \ + --skip-regex="\[Feature:|\[Disruptive\]|\[Serial\]" \ + --ginkgo-args="-v" \ + --parallel="${parallel_nodes}" \ + --test-args="--storage.testdriver=${testdriver}" diff --git a/pkg/csi/blockstorage/controllerserver.go b/pkg/csi/blockstorage/controllerserver.go index 367d8340..9191c4c2 100644 --- a/pkg/csi/blockstorage/controllerserver.go +++ b/pkg/csi/blockstorage/controllerserver.go @@ -58,9 +58,8 @@ type stackitParameterConfig struct { } const ( - blockStorageCSIClusterIDKey = "block-storage.csi.stackit.cloud/cluster" - snapshotTypeSnapshot = "snapshot" - snapshotTypeBackup = "backup" + snapshotTypeSnapshot = "snapshot" + snapshotTypeBackup = "backup" ) func (cs *controllerServer) validateVolumeCapabilities(req []*csi.VolumeCapability) error { @@ -79,7 +78,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol cloud := cs.Instance if cs.Driver.blockVolumeCreation { - return nil, status.Errorf(codes.Unimplemented, "The %s driver is update/read-only mode please migrate to the new driver", legacyDriverName) + return nil, status.Errorf(codes.Unimplemented, "The %s driver is update/read-only mode please migrate to the new driver", LegacyDriverName) } // Volume Name @@ -120,7 +119,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if cs.Driver.legacyDriver { volAvailability = sharedcsi.GetAZFromTopology(legacyTopologyKey, accessibleTopologyReq) } else { - volAvailability = sharedcsi.GetAZFromTopology(topologyKey, accessibleTopologyReq) + volAvailability = sharedcsi.GetAZFromTopology(cs.Driver.topologyKey(), accessibleTopologyReq) } } } @@ -148,7 +147,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // Volume Create // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{cs.Driver.clusterMetadataKey(): cs.Driver.clusterID} properties := map[string]string{} // Tag volume with metadata if present: https://github.com/kubernetes-csi/external-provisioner/pull/399 for _, mKey := range sharedcsi.RecognizedCSIProvisionerParams { @@ -493,7 +492,7 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS cloud := cs.Instance if cs.Driver.blockVolumeCreation { - return nil, status.Errorf(codes.Unimplemented, "The %s driver is update/read-only mode please migrate to the new driver", legacyDriverName) + return nil, status.Errorf(codes.Unimplemented, "The %s driver is update/read-only mode please migrate to the new driver", LegacyDriverName) } name := req.Name @@ -678,7 +677,7 @@ func (cs *controllerServer) createSnapshot(ctx context.Context, name, volumeID s // Add cluster ID to the snapshot metadata // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{cs.Driver.clusterMetadataKey(): cs.Driver.clusterID} properties := map[string]string{} // see https://github.com/kubernetes-csi/external-snapshotter/pull/375/ @@ -711,7 +710,7 @@ func (cs *controllerServer) createSnapshot(ctx context.Context, name, volumeID s func (cs *controllerServer) createBackup(ctx context.Context, cloud stackitclient.IaaSClient, name, volumeID string, snap *iaas.Snapshot, parameters map[string]string) (*iaas.Backup, error) { //nolint:lll // looks weird when shortened // Add cluster ID to the snapshot metadata // TODO: Use once IaaS has extended the label regex to allow for forward slashes and dots - // properties := map[string]string{blockStorageCSIClusterIDKey: cs.Driver.clusterID} + // properties := map[string]string{cs.Driver.clusterMetadataKey(): cs.Driver.clusterID} properties := map[string]string{} // see https://github.com/kubernetes-csi/external-snapshotter/pull/375/ @@ -1023,7 +1022,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat volumeSourceType = stackitclient.VolumeSourceTypes(vol.Source.Type) switch volumeSourceType { case stackitclient.VolumeSource: - volCnx[ResizeRequired] = "true" + volCnx[cs.Driver.resizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Volume{ @@ -1033,7 +1032,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat }, } case stackitclient.BackupSource: - volCnx[ResizeRequired] = "true" + volCnx[cs.Driver.resizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Snapshot{ @@ -1043,7 +1042,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat }, } case stackitclient.SnapshotSource: - volCnx[ResizeRequired] = "true" + volCnx[cs.Driver.resizeRequiredKey()] = "true" volsrc = &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Snapshot{ @@ -1055,7 +1054,7 @@ func (cs *controllerServer) getCreateVolumeResponse(vol *iaas.Volume) *csi.Creat } } - topoKey := topologyKey + topoKey := cs.Driver.topologyKey() if cs.Driver.legacyDriver { topoKey = legacyTopologyKey } diff --git a/pkg/csi/blockstorage/driver.go b/pkg/csi/blockstorage/driver.go index 60e3b6f9..45fbc17d 100644 --- a/pkg/csi/blockstorage/driver.go +++ b/pkg/csi/blockstorage/driver.go @@ -15,16 +15,16 @@ import ( ) const ( - driverName = "block-storage.csi.stackit.cloud" - legacyDriverName = "cinder.csi.openstack.org" - topologyKey = "topology." + driverName + "/zone" - legacyTopologyKey = "topology." + legacyDriverName + "/zone" + DefaultDriverName = "block-storage.csi.stackit.cloud" + LegacyDriverName = "cinder.csi.openstack.org" // ResizeRequired parameter, if set to true, will trigger a resize on mount operation - ResizeRequired = driverName + "/resizeRequired" + ResizeRequired = DefaultDriverName + "/resizeRequired" ) var ( + topologyKey = topologyKeyForDriver(DefaultDriverName) + legacyTopologyKey = topologyKeyForDriver(LegacyDriverName) // CSI spec version specVersion = "1.12.0" Version = "1.0.0" @@ -53,15 +53,31 @@ type Driver struct { type DriverOpts struct { ClusterID string Endpoint string + DriverName string LegacyDriverName bool BlockVolumeCreation bool PVCLister corev1.PersistentVolumeClaimLister } +func ValidateDriverOpts(o *DriverOpts) error { + if o == nil { + return fmt.Errorf("driver options must not be nil") + } + return nil +} + func NewDriver(o *DriverOpts) *Driver { + if err := ValidateDriverOpts(o); err != nil { + klog.Fatal(err) + } + + name := DefaultDriverName + if o.DriverName != "" { + name = o.DriverName + } d := &Driver{ - name: driverName, + name: name, fqVersion: fmt.Sprintf("%s@%s", Version, version.Version), endpoint: o.Endpoint, clusterID: o.ClusterID, @@ -69,7 +85,7 @@ func NewDriver(o *DriverOpts) *Driver { } if o.LegacyDriverName { - d.name = legacyDriverName + d.name = LegacyDriverName d.legacyDriver = true } @@ -112,6 +128,30 @@ func NewDriver(o *DriverOpts) *Driver { return d } +func (d *Driver) topologyKey() string { + return topologyKeyForDriver(d.name) +} + +func (d *Driver) resizeRequiredKey() string { + return resizeRequiredKeyForDriver(d.name) +} + +func (d *Driver) clusterMetadataKey() string { + return clusterMetadataKeyForDriver(d.name) +} + +func topologyKeyForDriver(name string) string { + return "topology." + name + "/zone" +} + +func resizeRequiredKeyForDriver(name string) string { + return name + "/resizeRequired" +} + +func clusterMetadataKeyForDriver(name string) string { + return name + "/cluster" +} + func (d *Driver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) { csc := make([]*csi.ControllerServiceCapability, 0, len(cl)) diff --git a/pkg/csi/blockstorage/driver_custom_test.go b/pkg/csi/blockstorage/driver_custom_test.go new file mode 100644 index 00000000..e130bb03 --- /dev/null +++ b/pkg/csi/blockstorage/driver_custom_test.go @@ -0,0 +1,44 @@ +package blockstorage + +import ( + "context" + "strings" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +func TestValidateDriverOptsRejectsLegacyWithCustomDriverName(t *testing.T) { + err := ValidateDriverOpts(&DriverOpts{ + DriverName: "custom.csi.stackit.cloud", + LegacyDriverName: true, + }) + if err == nil || !strings.Contains(err.Error(), "--legacy-storage-mode") { + t.Fatalf("expected legacy/custom conflict error, got %v", err) + } +} + +func TestNewDriverUsesConfiguredDriverName(t *testing.T) { + driver := NewDriver(&DriverOpts{ + Endpoint: "unix://tmp/csi.sock", + ClusterID: "cluster-id", + DriverName: "custom.csi.stackit.cloud", + }) + + info, err := driver.ids.GetPluginInfo(context.Background(), &csi.GetPluginInfoRequest{}) + if err != nil { + t.Fatalf("GetPluginInfo() returned error: %v", err) + } + if info.Name != "custom.csi.stackit.cloud" { + t.Fatalf("unexpected driver name: %q", info.Name) + } + if driver.topologyKey() != "topology.custom.csi.stackit.cloud/zone" { + t.Fatalf("unexpected topology key: %q", driver.topologyKey()) + } + if driver.resizeRequiredKey() != "custom.csi.stackit.cloud/resizeRequired" { + t.Fatalf("unexpected resize annotation key: %q", driver.resizeRequiredKey()) + } + if driver.clusterMetadataKey() != "custom.csi.stackit.cloud/cluster" { + t.Fatalf("unexpected cluster metadata key: %q", driver.clusterMetadataKey()) + } +} diff --git a/pkg/csi/blockstorage/nodeserver.go b/pkg/csi/blockstorage/nodeserver.go index 17501b5a..3f426c35 100644 --- a/pkg/csi/blockstorage/nodeserver.go +++ b/pkg/csi/blockstorage/nodeserver.go @@ -215,7 +215,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } } - if required, ok := volumeContext[ResizeRequired]; ok && strings.EqualFold(required, "true") { + if required, ok := volumeContext[ns.Driver.resizeRequiredKey()]; ok && strings.EqualFold(required, "true") { r := mountutil.NewResizeFs(ns.Mount.Mounter().Exec) needResize, err := r.NeedResize(devicePath, stagingTarget) @@ -312,7 +312,7 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest return nil, status.Errorf(codes.Internal, "[NodeGetInfo] Unable to retrieve availability zone of node %v", err) } - topoKey := topologyKey + topoKey := ns.Driver.topologyKey() if ns.Driver.legacyDriver { topoKey = legacyTopologyKey } @@ -334,10 +334,10 @@ func (ns *nodeServer) calculateMaxVolumesPerNode() int64 { freePCIeRootPorts = 0 } - csiDriverName := driverName + csiDriverName := ns.Driver.name if ns.Driver.legacyDriver { // If driver launched in legacy-mode use "cinder.csi.openstack.org" - csiDriverName = legacyDriverName + csiDriverName = LegacyDriverName } mountedCSIVolumes, err := mount.CountLocalCSIVolumes(csiDriverName) diff --git a/pkg/csi/csi.go b/pkg/csi/csi.go index 488ed1bc..1dee22a1 100644 --- a/pkg/csi/csi.go +++ b/pkg/csi/csi.go @@ -67,6 +67,8 @@ var ( ) var ( + // DriverName is set by the linker flags in the Makefile + DriverName string // CSI controller options pvcAnnotations bool // k8s client options diff --git a/pkg/mock/ske/ske.go b/pkg/mock/ske/ske.go new file mode 100644 index 00000000..f731e192 --- /dev/null +++ b/pkg/mock/ske/ske.go @@ -0,0 +1,448 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/stackitcloud/stackit-sdk-go/services/ske/v2api (interfaces: DefaultAPI) +// +// Generated by this command: +// +// mockgen -destination ./pkg/mock/ske/ske.go -package ske github.com/stackitcloud/stackit-sdk-go/services/ske/v2api DefaultAPI +// + +// Package ske is a generated GoMock package. +package ske + +import ( + context "context" + reflect "reflect" + + v2api "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" + gomock "go.uber.org/mock/gomock" +) + +// MockDefaultAPI is a mock of DefaultAPI interface. +type MockDefaultAPI struct { + ctrl *gomock.Controller + recorder *MockDefaultAPIMockRecorder + isgomock struct{} +} + +// MockDefaultAPIMockRecorder is the mock recorder for MockDefaultAPI. +type MockDefaultAPIMockRecorder struct { + mock *MockDefaultAPI +} + +// NewMockDefaultAPI creates a new mock instance. +func NewMockDefaultAPI(ctrl *gomock.Controller) *MockDefaultAPI { + mock := &MockDefaultAPI{ctrl: ctrl} + mock.recorder = &MockDefaultAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDefaultAPI) EXPECT() *MockDefaultAPIMockRecorder { + return m.recorder +} + +// CompleteCredentialsRotation mocks base method. +func (m *MockDefaultAPI) CompleteCredentialsRotation(ctx context.Context, projectId, region, clusterName string) v2api.ApiCompleteCredentialsRotationRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteCredentialsRotation", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiCompleteCredentialsRotationRequest) + return ret0 +} + +// CompleteCredentialsRotation indicates an expected call of CompleteCredentialsRotation. +func (mr *MockDefaultAPIMockRecorder) CompleteCredentialsRotation(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteCredentialsRotation", reflect.TypeOf((*MockDefaultAPI)(nil).CompleteCredentialsRotation), ctx, projectId, region, clusterName) +} + +// CompleteCredentialsRotationExecute mocks base method. +func (m *MockDefaultAPI) CompleteCredentialsRotationExecute(r v2api.ApiCompleteCredentialsRotationRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteCredentialsRotationExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteCredentialsRotationExecute indicates an expected call of CompleteCredentialsRotationExecute. +func (mr *MockDefaultAPIMockRecorder) CompleteCredentialsRotationExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteCredentialsRotationExecute", reflect.TypeOf((*MockDefaultAPI)(nil).CompleteCredentialsRotationExecute), r) +} + +// CreateKubeconfig mocks base method. +func (m *MockDefaultAPI) CreateKubeconfig(ctx context.Context, projectId, region, clusterName string) v2api.ApiCreateKubeconfigRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateKubeconfig", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiCreateKubeconfigRequest) + return ret0 +} + +// CreateKubeconfig indicates an expected call of CreateKubeconfig. +func (mr *MockDefaultAPIMockRecorder) CreateKubeconfig(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKubeconfig", reflect.TypeOf((*MockDefaultAPI)(nil).CreateKubeconfig), ctx, projectId, region, clusterName) +} + +// CreateKubeconfigExecute mocks base method. +func (m *MockDefaultAPI) CreateKubeconfigExecute(r v2api.ApiCreateKubeconfigRequest) (*v2api.Kubeconfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateKubeconfigExecute", r) + ret0, _ := ret[0].(*v2api.Kubeconfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateKubeconfigExecute indicates an expected call of CreateKubeconfigExecute. +func (mr *MockDefaultAPIMockRecorder) CreateKubeconfigExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKubeconfigExecute", reflect.TypeOf((*MockDefaultAPI)(nil).CreateKubeconfigExecute), r) +} + +// CreateOrUpdateCluster mocks base method. +func (m *MockDefaultAPI) CreateOrUpdateCluster(ctx context.Context, projectId, region, clusterName string) v2api.ApiCreateOrUpdateClusterRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdateCluster", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiCreateOrUpdateClusterRequest) + return ret0 +} + +// CreateOrUpdateCluster indicates an expected call of CreateOrUpdateCluster. +func (mr *MockDefaultAPIMockRecorder) CreateOrUpdateCluster(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCluster", reflect.TypeOf((*MockDefaultAPI)(nil).CreateOrUpdateCluster), ctx, projectId, region, clusterName) +} + +// CreateOrUpdateClusterExecute mocks base method. +func (m *MockDefaultAPI) CreateOrUpdateClusterExecute(r v2api.ApiCreateOrUpdateClusterRequest) (*v2api.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdateClusterExecute", r) + ret0, _ := ret[0].(*v2api.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateOrUpdateClusterExecute indicates an expected call of CreateOrUpdateClusterExecute. +func (mr *MockDefaultAPIMockRecorder) CreateOrUpdateClusterExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateClusterExecute", reflect.TypeOf((*MockDefaultAPI)(nil).CreateOrUpdateClusterExecute), r) +} + +// DeleteCluster mocks base method. +func (m *MockDefaultAPI) DeleteCluster(ctx context.Context, projectId, region, clusterName string) v2api.ApiDeleteClusterRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCluster", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiDeleteClusterRequest) + return ret0 +} + +// DeleteCluster indicates an expected call of DeleteCluster. +func (mr *MockDefaultAPIMockRecorder) DeleteCluster(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockDefaultAPI)(nil).DeleteCluster), ctx, projectId, region, clusterName) +} + +// DeleteClusterExecute mocks base method. +func (m *MockDefaultAPI) DeleteClusterExecute(r v2api.ApiDeleteClusterRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteClusterExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteClusterExecute indicates an expected call of DeleteClusterExecute. +func (mr *MockDefaultAPIMockRecorder) DeleteClusterExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteClusterExecute", reflect.TypeOf((*MockDefaultAPI)(nil).DeleteClusterExecute), r) +} + +// GetCluster mocks base method. +func (m *MockDefaultAPI) GetCluster(ctx context.Context, projectId, region, clusterName string) v2api.ApiGetClusterRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCluster", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiGetClusterRequest) + return ret0 +} + +// GetCluster indicates an expected call of GetCluster. +func (mr *MockDefaultAPIMockRecorder) GetCluster(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCluster", reflect.TypeOf((*MockDefaultAPI)(nil).GetCluster), ctx, projectId, region, clusterName) +} + +// GetClusterExecute mocks base method. +func (m *MockDefaultAPI) GetClusterExecute(r v2api.ApiGetClusterRequest) (*v2api.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterExecute", r) + ret0, _ := ret[0].(*v2api.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetClusterExecute indicates an expected call of GetClusterExecute. +func (mr *MockDefaultAPIMockRecorder) GetClusterExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterExecute", reflect.TypeOf((*MockDefaultAPI)(nil).GetClusterExecute), r) +} + +// GetIDPKubeconfig mocks base method. +func (m *MockDefaultAPI) GetIDPKubeconfig(ctx context.Context, projectId, region, clusterName string) v2api.ApiGetIDPKubeconfigRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDPKubeconfig", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiGetIDPKubeconfigRequest) + return ret0 +} + +// GetIDPKubeconfig indicates an expected call of GetIDPKubeconfig. +func (mr *MockDefaultAPIMockRecorder) GetIDPKubeconfig(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDPKubeconfig", reflect.TypeOf((*MockDefaultAPI)(nil).GetIDPKubeconfig), ctx, projectId, region, clusterName) +} + +// GetIDPKubeconfigExecute mocks base method. +func (m *MockDefaultAPI) GetIDPKubeconfigExecute(r v2api.ApiGetIDPKubeconfigRequest) (*v2api.IDPKubeconfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDPKubeconfigExecute", r) + ret0, _ := ret[0].(*v2api.IDPKubeconfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIDPKubeconfigExecute indicates an expected call of GetIDPKubeconfigExecute. +func (mr *MockDefaultAPIMockRecorder) GetIDPKubeconfigExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDPKubeconfigExecute", reflect.TypeOf((*MockDefaultAPI)(nil).GetIDPKubeconfigExecute), r) +} + +// GetLoginKubeconfig mocks base method. +func (m *MockDefaultAPI) GetLoginKubeconfig(ctx context.Context, projectId, region, clusterName string) v2api.ApiGetLoginKubeconfigRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoginKubeconfig", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiGetLoginKubeconfigRequest) + return ret0 +} + +// GetLoginKubeconfig indicates an expected call of GetLoginKubeconfig. +func (mr *MockDefaultAPIMockRecorder) GetLoginKubeconfig(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoginKubeconfig", reflect.TypeOf((*MockDefaultAPI)(nil).GetLoginKubeconfig), ctx, projectId, region, clusterName) +} + +// GetLoginKubeconfigExecute mocks base method. +func (m *MockDefaultAPI) GetLoginKubeconfigExecute(r v2api.ApiGetLoginKubeconfigRequest) (*v2api.LoginKubeconfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoginKubeconfigExecute", r) + ret0, _ := ret[0].(*v2api.LoginKubeconfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLoginKubeconfigExecute indicates an expected call of GetLoginKubeconfigExecute. +func (mr *MockDefaultAPIMockRecorder) GetLoginKubeconfigExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoginKubeconfigExecute", reflect.TypeOf((*MockDefaultAPI)(nil).GetLoginKubeconfigExecute), r) +} + +// ListClusters mocks base method. +func (m *MockDefaultAPI) ListClusters(ctx context.Context, projectId, region string) v2api.ApiListClustersRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListClusters", ctx, projectId, region) + ret0, _ := ret[0].(v2api.ApiListClustersRequest) + return ret0 +} + +// ListClusters indicates an expected call of ListClusters. +func (mr *MockDefaultAPIMockRecorder) ListClusters(ctx, projectId, region any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListClusters", reflect.TypeOf((*MockDefaultAPI)(nil).ListClusters), ctx, projectId, region) +} + +// ListClustersExecute mocks base method. +func (m *MockDefaultAPI) ListClustersExecute(r v2api.ApiListClustersRequest) (*v2api.ListClustersResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListClustersExecute", r) + ret0, _ := ret[0].(*v2api.ListClustersResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListClustersExecute indicates an expected call of ListClustersExecute. +func (mr *MockDefaultAPIMockRecorder) ListClustersExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListClustersExecute", reflect.TypeOf((*MockDefaultAPI)(nil).ListClustersExecute), r) +} + +// ListProviderOptions mocks base method. +func (m *MockDefaultAPI) ListProviderOptions(ctx context.Context, region string) v2api.ApiListProviderOptionsRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListProviderOptions", ctx, region) + ret0, _ := ret[0].(v2api.ApiListProviderOptionsRequest) + return ret0 +} + +// ListProviderOptions indicates an expected call of ListProviderOptions. +func (mr *MockDefaultAPIMockRecorder) ListProviderOptions(ctx, region any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListProviderOptions", reflect.TypeOf((*MockDefaultAPI)(nil).ListProviderOptions), ctx, region) +} + +// ListProviderOptionsExecute mocks base method. +func (m *MockDefaultAPI) ListProviderOptionsExecute(r v2api.ApiListProviderOptionsRequest) (*v2api.ProviderOptions, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListProviderOptionsExecute", r) + ret0, _ := ret[0].(*v2api.ProviderOptions) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListProviderOptionsExecute indicates an expected call of ListProviderOptionsExecute. +func (mr *MockDefaultAPIMockRecorder) ListProviderOptionsExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListProviderOptionsExecute", reflect.TypeOf((*MockDefaultAPI)(nil).ListProviderOptionsExecute), r) +} + +// StartCredentialsRotation mocks base method. +func (m *MockDefaultAPI) StartCredentialsRotation(ctx context.Context, projectId, region, clusterName string) v2api.ApiStartCredentialsRotationRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartCredentialsRotation", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiStartCredentialsRotationRequest) + return ret0 +} + +// StartCredentialsRotation indicates an expected call of StartCredentialsRotation. +func (mr *MockDefaultAPIMockRecorder) StartCredentialsRotation(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartCredentialsRotation", reflect.TypeOf((*MockDefaultAPI)(nil).StartCredentialsRotation), ctx, projectId, region, clusterName) +} + +// StartCredentialsRotationExecute mocks base method. +func (m *MockDefaultAPI) StartCredentialsRotationExecute(r v2api.ApiStartCredentialsRotationRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartCredentialsRotationExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StartCredentialsRotationExecute indicates an expected call of StartCredentialsRotationExecute. +func (mr *MockDefaultAPIMockRecorder) StartCredentialsRotationExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartCredentialsRotationExecute", reflect.TypeOf((*MockDefaultAPI)(nil).StartCredentialsRotationExecute), r) +} + +// TriggerHibernate mocks base method. +func (m *MockDefaultAPI) TriggerHibernate(ctx context.Context, projectId, region, clusterName string) v2api.ApiTriggerHibernateRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerHibernate", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiTriggerHibernateRequest) + return ret0 +} + +// TriggerHibernate indicates an expected call of TriggerHibernate. +func (mr *MockDefaultAPIMockRecorder) TriggerHibernate(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerHibernate", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerHibernate), ctx, projectId, region, clusterName) +} + +// TriggerHibernateExecute mocks base method. +func (m *MockDefaultAPI) TriggerHibernateExecute(r v2api.ApiTriggerHibernateRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerHibernateExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TriggerHibernateExecute indicates an expected call of TriggerHibernateExecute. +func (mr *MockDefaultAPIMockRecorder) TriggerHibernateExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerHibernateExecute", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerHibernateExecute), r) +} + +// TriggerMaintenance mocks base method. +func (m *MockDefaultAPI) TriggerMaintenance(ctx context.Context, projectId, region, clusterName string) v2api.ApiTriggerMaintenanceRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerMaintenance", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiTriggerMaintenanceRequest) + return ret0 +} + +// TriggerMaintenance indicates an expected call of TriggerMaintenance. +func (mr *MockDefaultAPIMockRecorder) TriggerMaintenance(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerMaintenance", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerMaintenance), ctx, projectId, region, clusterName) +} + +// TriggerMaintenanceExecute mocks base method. +func (m *MockDefaultAPI) TriggerMaintenanceExecute(r v2api.ApiTriggerMaintenanceRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerMaintenanceExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TriggerMaintenanceExecute indicates an expected call of TriggerMaintenanceExecute. +func (mr *MockDefaultAPIMockRecorder) TriggerMaintenanceExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerMaintenanceExecute", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerMaintenanceExecute), r) +} + +// TriggerReconcile mocks base method. +func (m *MockDefaultAPI) TriggerReconcile(ctx context.Context, projectId, region, clusterName string) v2api.ApiTriggerReconcileRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerReconcile", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiTriggerReconcileRequest) + return ret0 +} + +// TriggerReconcile indicates an expected call of TriggerReconcile. +func (mr *MockDefaultAPIMockRecorder) TriggerReconcile(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerReconcile", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerReconcile), ctx, projectId, region, clusterName) +} + +// TriggerReconcileExecute mocks base method. +func (m *MockDefaultAPI) TriggerReconcileExecute(r v2api.ApiTriggerReconcileRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerReconcileExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TriggerReconcileExecute indicates an expected call of TriggerReconcileExecute. +func (mr *MockDefaultAPIMockRecorder) TriggerReconcileExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerReconcileExecute", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerReconcileExecute), r) +} + +// TriggerWakeup mocks base method. +func (m *MockDefaultAPI) TriggerWakeup(ctx context.Context, projectId, region, clusterName string) v2api.ApiTriggerWakeupRequest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerWakeup", ctx, projectId, region, clusterName) + ret0, _ := ret[0].(v2api.ApiTriggerWakeupRequest) + return ret0 +} + +// TriggerWakeup indicates an expected call of TriggerWakeup. +func (mr *MockDefaultAPIMockRecorder) TriggerWakeup(ctx, projectId, region, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerWakeup", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerWakeup), ctx, projectId, region, clusterName) +} + +// TriggerWakeupExecute mocks base method. +func (m *MockDefaultAPI) TriggerWakeupExecute(r v2api.ApiTriggerWakeupRequest) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TriggerWakeupExecute", r) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TriggerWakeupExecute indicates an expected call of TriggerWakeupExecute. +func (mr *MockDefaultAPIMockRecorder) TriggerWakeupExecute(r any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TriggerWakeupExecute", reflect.TypeOf((*MockDefaultAPI)(nil).TriggerWakeupExecute), r) +} diff --git a/pkg/stackit/client/factory.go b/pkg/stackit/client/factory.go index 66b089b1..337a8576 100644 --- a/pkg/stackit/client/factory.go +++ b/pkg/stackit/client/factory.go @@ -40,6 +40,9 @@ type Factory interface { // IaaS returns a STACKIT IaaS service client. IaaS(options []sdkconfig.ConfigurationOption) (IaaSClient, error) + + // SKE returns a STACKIT Kubernetes Engine service client. + SKE(options []sdkconfig.ConfigurationOption) (SKEClient, error) } type factory struct { @@ -62,6 +65,10 @@ func (f factory) IaaS(options []sdkconfig.ConfigurationOption) (IaaSClient, erro return NewIaaSClient(f.StackitRegion, f.StackitProjectID, withDefaultOptions(options)) } +func (f factory) SKE(options []sdkconfig.ConfigurationOption) (SKEClient, error) { + return NewSKEClient(f.StackitRegion, f.StackitProjectID, withDefaultOptions(options)) +} + func withDefaultOptions(options []sdkconfig.ConfigurationOption) []sdkconfig.ConfigurationOption { return append(options, sdkconfig.WithUserAgent(BuildUserAgent(defaultUserAgentComponent, version.Version))) diff --git a/pkg/stackit/client/mock/mock.go b/pkg/stackit/client/mock/mock.go index 25af6cc2..3b8f0714 100644 --- a/pkg/stackit/client/mock/mock.go +++ b/pkg/stackit/client/mock/mock.go @@ -70,3 +70,18 @@ func (mr *MockFactoryMockRecorder) LoadBalancing(options any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancing", reflect.TypeOf((*MockFactory)(nil).LoadBalancing), options) } + +// SKE mocks base method. +func (m *MockFactory) SKE(options []config.ConfigurationOption) (client.SKEClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SKE", options) + ret0, _ := ret[0].(client.SKEClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SKE indicates an expected call of SKE. +func (mr *MockFactoryMockRecorder) SKE(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SKE", reflect.TypeOf((*MockFactory)(nil).SKE), options) +} diff --git a/pkg/stackit/client/mock/ske_mock.go b/pkg/stackit/client/mock/ske_mock.go new file mode 100644 index 00000000..027a05a2 --- /dev/null +++ b/pkg/stackit/client/mock/ske_mock.go @@ -0,0 +1,313 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/stackit/client (interfaces: SKEClient) +// +// Generated by this command: +// +// mockgen -destination ./pkg/stackit/client/mock/ske_mock.go -typed -package client ./pkg/stackit/client SKEClient +// + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + v2api "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" + gomock "go.uber.org/mock/gomock" +) + +// MockSKEClient is a mock of SKEClient interface. +type MockSKEClient struct { + ctrl *gomock.Controller + recorder *MockSKEClientMockRecorder + isgomock struct{} +} + +// MockSKEClientMockRecorder is the mock recorder for MockSKEClient. +type MockSKEClientMockRecorder struct { + mock *MockSKEClient +} + +// NewMockSKEClient creates a new mock instance. +func NewMockSKEClient(ctrl *gomock.Controller) *MockSKEClient { + mock := &MockSKEClient{ctrl: ctrl} + mock.recorder = &MockSKEClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSKEClient) EXPECT() *MockSKEClientMockRecorder { + return m.recorder +} + +// CreateKubeconfig mocks base method. +func (m *MockSKEClient) CreateKubeconfig(ctx context.Context, clusterName string, payload v2api.CreateKubeconfigPayload) (*v2api.Kubeconfig, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateKubeconfig", ctx, clusterName, payload) + ret0, _ := ret[0].(*v2api.Kubeconfig) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateKubeconfig indicates an expected call of CreateKubeconfig. +func (mr *MockSKEClientMockRecorder) CreateKubeconfig(ctx, clusterName, payload any) *MockSKEClientCreateKubeconfigCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateKubeconfig", reflect.TypeOf((*MockSKEClient)(nil).CreateKubeconfig), ctx, clusterName, payload) + return &MockSKEClientCreateKubeconfigCall{Call: call} +} + +// MockSKEClientCreateKubeconfigCall wrap *gomock.Call +type MockSKEClientCreateKubeconfigCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientCreateKubeconfigCall) Return(arg0 *v2api.Kubeconfig, arg1 error) *MockSKEClientCreateKubeconfigCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientCreateKubeconfigCall) Do(f func(context.Context, string, v2api.CreateKubeconfigPayload) (*v2api.Kubeconfig, error)) *MockSKEClientCreateKubeconfigCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientCreateKubeconfigCall) DoAndReturn(f func(context.Context, string, v2api.CreateKubeconfigPayload) (*v2api.Kubeconfig, error)) *MockSKEClientCreateKubeconfigCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// CreateOrUpdateCluster mocks base method. +func (m *MockSKEClient) CreateOrUpdateCluster(ctx context.Context, clusterName string, payload v2api.CreateOrUpdateClusterPayload) (*v2api.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdateCluster", ctx, clusterName, payload) + ret0, _ := ret[0].(*v2api.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateOrUpdateCluster indicates an expected call of CreateOrUpdateCluster. +func (mr *MockSKEClientMockRecorder) CreateOrUpdateCluster(ctx, clusterName, payload any) *MockSKEClientCreateOrUpdateClusterCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateCluster", reflect.TypeOf((*MockSKEClient)(nil).CreateOrUpdateCluster), ctx, clusterName, payload) + return &MockSKEClientCreateOrUpdateClusterCall{Call: call} +} + +// MockSKEClientCreateOrUpdateClusterCall wrap *gomock.Call +type MockSKEClientCreateOrUpdateClusterCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientCreateOrUpdateClusterCall) Return(arg0 *v2api.Cluster, arg1 error) *MockSKEClientCreateOrUpdateClusterCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientCreateOrUpdateClusterCall) Do(f func(context.Context, string, v2api.CreateOrUpdateClusterPayload) (*v2api.Cluster, error)) *MockSKEClientCreateOrUpdateClusterCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientCreateOrUpdateClusterCall) DoAndReturn(f func(context.Context, string, v2api.CreateOrUpdateClusterPayload) (*v2api.Cluster, error)) *MockSKEClientCreateOrUpdateClusterCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// DeleteCluster mocks base method. +func (m *MockSKEClient) DeleteCluster(ctx context.Context, clusterName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCluster", ctx, clusterName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCluster indicates an expected call of DeleteCluster. +func (mr *MockSKEClientMockRecorder) DeleteCluster(ctx, clusterName any) *MockSKEClientDeleteClusterCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockSKEClient)(nil).DeleteCluster), ctx, clusterName) + return &MockSKEClientDeleteClusterCall{Call: call} +} + +// MockSKEClientDeleteClusterCall wrap *gomock.Call +type MockSKEClientDeleteClusterCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientDeleteClusterCall) Return(arg0 error) *MockSKEClientDeleteClusterCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientDeleteClusterCall) Do(f func(context.Context, string) error) *MockSKEClientDeleteClusterCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientDeleteClusterCall) DoAndReturn(f func(context.Context, string) error) *MockSKEClientDeleteClusterCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// GetCluster mocks base method. +func (m *MockSKEClient) GetCluster(ctx context.Context, clusterName string) (*v2api.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCluster", ctx, clusterName) + ret0, _ := ret[0].(*v2api.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCluster indicates an expected call of GetCluster. +func (mr *MockSKEClientMockRecorder) GetCluster(ctx, clusterName any) *MockSKEClientGetClusterCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCluster", reflect.TypeOf((*MockSKEClient)(nil).GetCluster), ctx, clusterName) + return &MockSKEClientGetClusterCall{Call: call} +} + +// MockSKEClientGetClusterCall wrap *gomock.Call +type MockSKEClientGetClusterCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientGetClusterCall) Return(arg0 *v2api.Cluster, arg1 error) *MockSKEClientGetClusterCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientGetClusterCall) Do(f func(context.Context, string) (*v2api.Cluster, error)) *MockSKEClientGetClusterCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientGetClusterCall) DoAndReturn(f func(context.Context, string) (*v2api.Cluster, error)) *MockSKEClientGetClusterCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// ListProviderOptions mocks base method. +func (m *MockSKEClient) ListProviderOptions(ctx context.Context) (*v2api.ProviderOptions, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListProviderOptions", ctx) + ret0, _ := ret[0].(*v2api.ProviderOptions) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListProviderOptions indicates an expected call of ListProviderOptions. +func (mr *MockSKEClientMockRecorder) ListProviderOptions(ctx any) *MockSKEClientListProviderOptionsCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListProviderOptions", reflect.TypeOf((*MockSKEClient)(nil).ListProviderOptions), ctx) + return &MockSKEClientListProviderOptionsCall{Call: call} +} + +// MockSKEClientListProviderOptionsCall wrap *gomock.Call +type MockSKEClientListProviderOptionsCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientListProviderOptionsCall) Return(arg0 *v2api.ProviderOptions, arg1 error) *MockSKEClientListProviderOptionsCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientListProviderOptionsCall) Do(f func(context.Context) (*v2api.ProviderOptions, error)) *MockSKEClientListProviderOptionsCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientListProviderOptionsCall) DoAndReturn(f func(context.Context) (*v2api.ProviderOptions, error)) *MockSKEClientListProviderOptionsCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// WaitClusterDeleted mocks base method. +func (m *MockSKEClient) WaitClusterDeleted(ctx context.Context, clusterName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitClusterDeleted", ctx, clusterName) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitClusterDeleted indicates an expected call of WaitClusterDeleted. +func (mr *MockSKEClientMockRecorder) WaitClusterDeleted(ctx, clusterName any) *MockSKEClientWaitClusterDeletedCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitClusterDeleted", reflect.TypeOf((*MockSKEClient)(nil).WaitClusterDeleted), ctx, clusterName) + return &MockSKEClientWaitClusterDeletedCall{Call: call} +} + +// MockSKEClientWaitClusterDeletedCall wrap *gomock.Call +type MockSKEClientWaitClusterDeletedCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientWaitClusterDeletedCall) Return(arg0 error) *MockSKEClientWaitClusterDeletedCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientWaitClusterDeletedCall) Do(f func(context.Context, string) error) *MockSKEClientWaitClusterDeletedCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientWaitClusterDeletedCall) DoAndReturn(f func(context.Context, string) error) *MockSKEClientWaitClusterDeletedCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// WaitClusterReady mocks base method. +func (m *MockSKEClient) WaitClusterReady(ctx context.Context, clusterName string) (*v2api.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitClusterReady", ctx, clusterName) + ret0, _ := ret[0].(*v2api.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WaitClusterReady indicates an expected call of WaitClusterReady. +func (mr *MockSKEClientMockRecorder) WaitClusterReady(ctx, clusterName any) *MockSKEClientWaitClusterReadyCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitClusterReady", reflect.TypeOf((*MockSKEClient)(nil).WaitClusterReady), ctx, clusterName) + return &MockSKEClientWaitClusterReadyCall{Call: call} +} + +// MockSKEClientWaitClusterReadyCall wrap *gomock.Call +type MockSKEClientWaitClusterReadyCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSKEClientWaitClusterReadyCall) Return(arg0 *v2api.Cluster, arg1 error) *MockSKEClientWaitClusterReadyCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSKEClientWaitClusterReadyCall) Do(f func(context.Context, string) (*v2api.Cluster, error)) *MockSKEClientWaitClusterReadyCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSKEClientWaitClusterReadyCall) DoAndReturn(f func(context.Context, string) (*v2api.Cluster, error)) *MockSKEClientWaitClusterReadyCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/pkg/stackit/client/ske.go b/pkg/stackit/client/ske.go new file mode 100644 index 00000000..1822c01a --- /dev/null +++ b/pkg/stackit/client/ske.go @@ -0,0 +1,86 @@ +package client + +import ( + "context" + + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + ske "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" + skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api/wait" +) + +type skeClient struct { + Client ske.DefaultAPI + projectID string + region string +} + +type SKEClient interface { + ListProviderOptions(ctx context.Context) (*ske.ProviderOptions, error) + CreateOrUpdateCluster(ctx context.Context, clusterName string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) + GetCluster(ctx context.Context, clusterName string) (*ske.Cluster, error) + DeleteCluster(ctx context.Context, clusterName string) error + CreateKubeconfig(ctx context.Context, clusterName string, payload ske.CreateKubeconfigPayload) (*ske.Kubeconfig, error) + WaitClusterReady(ctx context.Context, clusterName string) (*ske.Cluster, error) + WaitClusterDeleted(ctx context.Context, clusterName string) error +} + +func NewSKEClient(region, projectID string, options []sdkconfig.ConfigurationOption) (SKEClient, error) { + apiClient, err := ske.NewAPIClient(options...) + if err != nil { + return nil, err + } + + return &skeClient{ + Client: apiClient.DefaultAPI, + projectID: projectID, + region: region, + }, nil +} + +func (s *skeClient) ListProviderOptions(ctx context.Context) (*ske.ProviderOptions, error) { + return withResponseID(ctx, func(ctx context.Context) (*ske.ProviderOptions, error) { + return s.Client.ListProviderOptions(ctx, s.region).Execute() + }) +} + +//nolint:gocritic // Payload is passed by value to match the shared SKEClient interface. +func (s *skeClient) CreateOrUpdateCluster(ctx context.Context, clusterName string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + return withResponseID(ctx, func(ctx context.Context) (*ske.Cluster, error) { + return s.Client. + CreateOrUpdateCluster(ctx, s.projectID, s.region, clusterName). + CreateOrUpdateClusterPayload(payload). + Execute() + }) +} + +func (s *skeClient) GetCluster(ctx context.Context, clusterName string) (*ske.Cluster, error) { + return withResponseID(ctx, func(ctx context.Context) (*ske.Cluster, error) { + return s.Client.GetCluster(ctx, s.projectID, s.region, clusterName).Execute() + }) +} + +func (s *skeClient) DeleteCluster(ctx context.Context, clusterName string) error { + _, err := withResponseID(ctx, func(ctx context.Context) (map[string]interface{}, error) { + return s.Client.DeleteCluster(ctx, s.projectID, s.region, clusterName).Execute() + }) + return err +} + +//nolint:gocritic // Payload is passed by value to match the shared SKEClient interface. +func (s *skeClient) CreateKubeconfig(ctx context.Context, clusterName string, payload ske.CreateKubeconfigPayload) (*ske.Kubeconfig, error) { + return withResponseID(ctx, func(ctx context.Context) (*ske.Kubeconfig, error) { + return s.Client. + CreateKubeconfig(ctx, s.projectID, s.region, clusterName). + CreateKubeconfigPayload(payload). + Execute() + }) +} + +func (s *skeClient) WaitClusterReady(ctx context.Context, clusterName string) (*ske.Cluster, error) { + return skewait.CreateOrUpdateClusterWaitHandler(ctx, s.Client, s.projectID, s.region, clusterName).WaitWithContext(ctx) +} + +func (s *skeClient) WaitClusterDeleted(ctx context.Context, clusterName string) error { + _, err := skewait.DeleteClusterWaitHandler(ctx, s.Client, s.projectID, s.region, clusterName).WaitWithContext(ctx) + return err +} diff --git a/pkg/stackit/client/ske_test.go b/pkg/stackit/client/ske_test.go new file mode 100644 index 00000000..0133dc5c --- /dev/null +++ b/pkg/stackit/client/ske_test.go @@ -0,0 +1,135 @@ +package client + +import ( + "context" + "net/http" + "testing" + + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + ske "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" +) + +func TestSKEClientListProviderOptions(t *testing.T) { + expectedVersion := "1.32.1" + api := ske.DefaultAPIServiceMock{ + ListProviderOptionsExecuteMock: testPtr(func(_ ske.ApiListProviderOptionsRequest) (*ske.ProviderOptions, error) { + return &ske.ProviderOptions{ + KubernetesVersions: []ske.KubernetesVersion{ + {Version: &expectedVersion}, + }, + }, nil + }), + } + + c := &skeClient{ + Client: api, + region: "eu01", + } + + options, err := c.ListProviderOptions(context.Background()) + if err != nil { + t.Fatalf("ListProviderOptions() returned error: %v", err) + } + if len(options.KubernetesVersions) != 1 || options.KubernetesVersions[0].GetVersion() != expectedVersion { + t.Fatalf("unexpected provider options: %#v", options) + } +} + +func TestSKEClientCreateOrUpdateClusterPropagatesError(t *testing.T) { + api := ske.DefaultAPIServiceMock{ + CreateOrUpdateClusterExecuteMock: testPtr(func(_ ske.ApiCreateOrUpdateClusterRequest) (*ske.Cluster, error) { + return nil, &oapierror.GenericOpenAPIError{StatusCode: http.StatusBadRequest} + }), + } + + c := &skeClient{ + Client: api, + projectID: "project-id", + region: "eu01", + } + + _, err := c.CreateOrUpdateCluster(context.Background(), "cluster", *ske.NewCreateOrUpdateClusterPayload( + *ske.NewKubernetes("1.32.1"), + []ske.Nodepool{}, + )) + if err == nil { + t.Fatal("expected error") + } +} + +func TestSKEClientCreateKubeconfig(t *testing.T) { + content := "apiVersion: v1\n" + api := ske.DefaultAPIServiceMock{ + CreateKubeconfigExecuteMock: testPtr(func(_ ske.ApiCreateKubeconfigRequest) (*ske.Kubeconfig, error) { + return &ske.Kubeconfig{Kubeconfig: &content}, nil + }), + } + + c := &skeClient{ + Client: api, + projectID: "project-id", + region: "eu01", + } + + kubeconfig, err := c.CreateKubeconfig(context.Background(), "cluster", ske.CreateKubeconfigPayload{}) + if err != nil { + t.Fatalf("CreateKubeconfig() returned error: %v", err) + } + if kubeconfig.GetKubeconfig() != content { + t.Fatalf("unexpected kubeconfig: %#v", kubeconfig) + } +} + +func TestSKEClientWaitClusterReady(t *testing.T) { + clusterName := "cluster" + status := ske.CLUSTERSTATUSSTATE_STATE_HEALTHY + api := ske.DefaultAPIServiceMock{ + GetClusterExecuteMock: testPtr(func(_ ske.ApiGetClusterRequest) (*ske.Cluster, error) { + return &ske.Cluster{ + Name: &clusterName, + Status: &ske.ClusterStatus{ + Aggregated: &status, + }, + Kubernetes: *ske.NewKubernetes("1.32.1"), + Nodepools: []ske.Nodepool{}, + }, nil + }), + } + + c := &skeClient{ + Client: api, + projectID: "project-id", + region: "eu01", + } + + cluster, err := c.WaitClusterReady(context.Background(), clusterName) + if err != nil { + t.Fatalf("WaitClusterReady() returned error: %v", err) + } + if cluster.GetName() != clusterName { + t.Fatalf("unexpected cluster: %#v", cluster) + } +} + +func TestSKEClientWaitClusterDeletedPropagatesError(t *testing.T) { + api := ske.DefaultAPIServiceMock{ + ListClustersExecuteMock: testPtr(func(_ ske.ApiListClustersRequest) (*ske.ListClustersResponse, error) { + return nil, &oapierror.GenericOpenAPIError{StatusCode: http.StatusInternalServerError} + }), + } + + c := &skeClient{ + Client: api, + projectID: "project-id", + region: "eu01", + } + + err := c.WaitClusterDeleted(context.Background(), "cluster") + if err == nil { + t.Fatal("expected error") + } +} + +func testPtr[T any](v T) *T { + return &v +} diff --git a/sequential-backup.sh b/sequential-backup.sh new file mode 100644 index 00000000..f32e58ad --- /dev/null +++ b/sequential-backup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export STACKIT_SERVICE_ACCOUNT="$(cat ./credentials.json)" + +run_id="stackit-seq-backup" +cluster_name="kt2-back" +kubernetes_version="1.35.6" +rundir_root="${PWD}/_rundir" +artifacts_root="${PWD}/_artifacts-sequential-backup" +testdriver="${rundir_root}/${run_id}/csi-testdriver.yaml" + +go run ./cmd/kubetest2-stackit \ + --run-id "${run_id}" \ + --cluster-name "${cluster_name}" \ + --rundir "${rundir_root}" \ + --artifacts "${artifacts_root}" \ + --up \ + --test=ginkgo \ + --project-id e928aade-ce15-4188-8230-c162c8fb3bd4 \ + --region eu01 \ + --kubernetes-version "${kubernetes_version}" \ + --availability-zone eu01-1 \ + --machine-type c2i.2 \ + --node-image-name flatcar \ + --node-image-version 4593.2.3-containerd2.1.9 \ + --csi-snapshot-type backup \ + --csi-image-name ttl.sh/csi-plugin/stackit-csi-plugin \ + --csi-image-tag 5h \ + -- \ + --test-package-version="v${kubernetes_version}" \ + --focus-regex="External.Storage.*(\[Feature:|\[Disruptive\]|\[Serial\])" \ + --ginkgo-args="-v" \ + --parallel=1 \ + --test-args="--storage.testdriver=${testdriver}" diff --git a/sequential-snapshot.sh b/sequential-snapshot.sh new file mode 100644 index 00000000..d702caa4 --- /dev/null +++ b/sequential-snapshot.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export STACKIT_SERVICE_ACCOUNT="$(cat ./credentials.json)" + +run_id="stackit-seq-snapshot" +cluster_name="kt2-snap" +kubernetes_version="1.35.6" +rundir_root="${PWD}/_rundir" +artifacts_root="${PWD}/_artifacts-sequential-snapshot" +testdriver="${rundir_root}/${run_id}/csi-testdriver.yaml" + +go run ./cmd/kubetest2-stackit \ + --run-id "${run_id}" \ + --cluster-name "${cluster_name}" \ + --rundir "${rundir_root}" \ + --artifacts "${artifacts_root}" \ + --up \ + --test=ginkgo \ + --project-id e928aade-ce15-4188-8230-c162c8fb3bd4 \ + --region eu01 \ + --kubernetes-version "${kubernetes_version}" \ + --availability-zone eu01-1 \ + --machine-type c2i.2 \ + --node-image-name flatcar \ + --node-image-version 4593.2.3-containerd2.1.9 \ + --csi-image-name ttl.sh/csi-plugin/stackit-csi-plugin \ + --csi-image-tag 5h \ + -- \ + --test-package-version="v${kubernetes_version}" \ + --focus-regex="External.Storage.*(\[Feature:|\[Disruptive\]|\[Serial\])" \ + --ginkgo-args="-v" \ + --parallel=1 \ + --test-args="--storage.testdriver=${testdriver}" diff --git a/test/e2e/assets.go b/test/e2e/assets.go new file mode 100644 index 00000000..51a34e86 --- /dev/null +++ b/test/e2e/assets.go @@ -0,0 +1,8 @@ +package e2e + +import "embed" + +// FS contains embedded e2e test assets. +// +//go:embed kustomize +var FS embed.FS diff --git a/test/e2e/kubetest2-stackit/deployer/csi_reconciler.go b/test/e2e/kubetest2-stackit/deployer/csi_reconciler.go new file mode 100644 index 00000000..2cfe00ae --- /dev/null +++ b/test/e2e/kubetest2-stackit/deployer/csi_reconciler.go @@ -0,0 +1,404 @@ +package deployer + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + e2eassets "github.com/stackitcloud/cloud-provider-stackit/test/e2e" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +const ( + csiTestDriverFileName = "csi-testdriver.yaml" + + kubeSystemNamespace = "kube-system" + stackitCloudConfigMapName = "stackit-cloud-config" + stackitCloudConfigMapKey = "cloud.yaml" + stackitCloudSecretName = "stackit-cloud-secret" + stackitCloudSecretKey = "sa_key.json" + snapshotControllerDeploymentName = "snapshot-controller" + csiControllerDeploymentName = "csi-stackit-controllerplugin" + csiNodeDaemonSetName = "csi-stackit-nodeplugin" + defaultBlockStorageRescanOnResize = true + kustomizePath = "kustomize" + + kustomizeTestDriverName = "kubetest2.csi.stackit.cloud" + kustomizeTestClassName = "kubetest2-stackit" +) + +type csiInstallConfig struct { + KubeconfigPath string + ProjectID string + Region string + ServiceAccountJSON string + StorageClassType string + SnapshotType string + ImageName string + ImageTag string + RescanOnResize bool + TestDriverOutputPath string +} + +type csiReconciler interface { + Reconcile(context.Context, *csiInstallConfig) error +} + +type kustomizeCSIReconciler struct{} + +func newKustomizeCSIReconciler() *kustomizeCSIReconciler { + return &kustomizeCSIReconciler{} +} + +func (r *kustomizeCSIReconciler) Reconcile(ctx context.Context, cfg *csiInstallConfig) error { + if err := ensureNamespace(ctx, cfg.KubeconfigPath, kubeSystemNamespace); err != nil { + return err + } + if err := upsertCloudConfig(ctx, cfg.KubeconfigPath, cfg); err != nil { + return err + } + if err := upsertCloudSecret(ctx, cfg.KubeconfigPath, cfg); err != nil { + return err + } + if err := r.applyKustomize(ctx, cfg); err != nil { + return err + } + if err := waitForCRDs(ctx, cfg.KubeconfigPath); err != nil { + return err + } + if err := waitForWorkloads(ctx, cfg.KubeconfigPath); err != nil { + return err + } + return writeCSITestDriverConfig(cfg.TestDriverOutputPath) +} + +func (r *kustomizeCSIReconciler) applyKustomize(ctx context.Context, cfg *csiInstallConfig) error { + tmpDir, err := os.MkdirTemp("", "stackit-csi-*") + if err != nil { + return fmt.Errorf("create temp directory: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := writeKustomizeOverlay(tmpDir, cfg); err != nil { + return fmt.Errorf("write kustomize overlay: %w", err) + } + + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-k", tmpDir, "--kubeconfig", cfg.KubeconfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + klog.Infof("Applying kustomize overlay from %s", tmpDir) + if err := cmd.Run(); err != nil { + return fmt.Errorf("kubectl apply -k: %w", err) + } + return nil +} + +func writeKustomizeOverlay(tmpDir string, cfg *csiInstallConfig) error { + if err := copyEmbeddedFS(tmpDir); err != nil { + return fmt.Errorf("copy embedded fs: %w", err) + } + + kustomizationPath := filepath.Join(tmpDir, "kustomization.yaml") + content, err := os.ReadFile(kustomizationPath) + if err != nil { + return fmt.Errorf("read kustomization.yaml: %w", err) + } + + csiPluginDir, err := resolveCSIPluginDir() + if err != nil { + return fmt.Errorf("resolve csi-plugin directory: %w", err) + } + + kustomization := string(content) + kustomization = strings.ReplaceAll(kustomization, "../../../deploy/csi-plugin", csiPluginDir) + kustomization = strings.ReplaceAll(kustomization, "IMAGE_NAME_PLACEHOLDER", cfg.ImageName) + kustomization = strings.ReplaceAll(kustomization, "IMAGE_TAG_PLACEHOLDER", cfg.ImageTag) + kustomization = strings.ReplaceAll(kustomization, "CSI_DRIVER_NAME_PLACEHOLDER", kustomizeTestDriverName) + kustomization = strings.ReplaceAll(kustomization, "CSI_CLASS_NAME_PLACEHOLDER", kustomizeTestClassName) + + if err := os.WriteFile(kustomizationPath, []byte(kustomization), 0o644); err != nil { + return fmt.Errorf("write kustomization.yaml: %w", err) + } + + return nil +} + +func copyEmbeddedFS(tmpDir string) error { + return copyFSDir(e2eassets.FS, kustomizePath, tmpDir) +} + +func resolveCSIPluginDir() (string, error) { + _, filename, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("runtime.Caller failed") + } + repoRoot := filepath.Join(filepath.Dir(filename), "..", "..", "..", "..", "..") + return filepath.Abs(filepath.Join(repoRoot, "deploy", "csi-plugin")) +} + +func copyFSDir(fsys fs.FS, srcDir, dstDir string) error { + entries, err := fs.ReadDir(fsys, srcDir) + if err != nil { + return fmt.Errorf("read dir %s: %w", srcDir, err) + } + + for _, entry := range entries { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + if entry.IsDir() { + if err := os.MkdirAll(dstPath, 0o755); err != nil { + return err + } + if err := copyFSDir(fsys, srcPath, dstPath); err != nil { + return err + } + } else { + content, err := fs.ReadFile(fsys, srcPath) + if err != nil { + return fmt.Errorf("read file %s: %w", srcPath, err) + } + if err := os.WriteFile(dstPath, content, 0o644); err != nil { + return fmt.Errorf("write file %s: %w", dstPath, err) + } + } + } + return nil +} + +func ensureNamespace(ctx context.Context, kubeconfigPath, name string) error { + cmd := exec.CommandContext(ctx, "kubectl", "create", "namespace", name, + "--kubeconfig", kubeconfigPath, "--dry-run=client", "-o", "yaml") + var stdout bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("generate namespace manifest: %w", err) + } + + applyCmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", "-", "--kubeconfig", kubeconfigPath) + applyCmd.Stdin = &stdout + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("apply namespace %q: %w", name, err) + } + return nil +} + +func upsertCloudConfig(ctx context.Context, kubeconfigPath string, cfg *csiInstallConfig) error { + content, err := buildCSICloudConfig(cfg) + if err != nil { + return err + } + + tmpFile, err := os.CreateTemp("", "stackit-cloud-config-*.yaml") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + defer os.Remove(tmpFile.Name()) + + manifest := fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s +data: + %s: | +%s`, stackitCloudConfigMapName, kubeSystemNamespace, stackitCloudConfigMapKey, indentYAML(content, 4)) + + if err := os.WriteFile(tmpFile.Name(), []byte(manifest), 0o600); err != nil { + return fmt.Errorf("write configmap manifest: %w", err) + } + + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", tmpFile.Name(), "--kubeconfig", kubeconfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("apply configmap %q: %w", stackitCloudConfigMapName, err) + } + return nil +} + +func upsertCloudSecret(ctx context.Context, kubeconfigPath string, cfg *csiInstallConfig) error { + tmpFile, err := os.CreateTemp("", "stackit-cloud-secret-*.yaml") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + defer os.Remove(tmpFile.Name()) + + manifest := fmt.Sprintf(`apiVersion: v1 +kind: Secret +metadata: + name: %s + namespace: %s +type: Opaque +stringData: + %s: | +%s`, stackitCloudSecretName, kubeSystemNamespace, stackitCloudSecretKey, indentYAML(cfg.ServiceAccountJSON, 4)) + + if err := os.WriteFile(tmpFile.Name(), []byte(manifest), 0o600); err != nil { + return fmt.Errorf("write secret manifest: %w", err) + } + + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", tmpFile.Name(), "--kubeconfig", kubeconfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("apply secret %q: %w", stackitCloudSecretName, err) + } + return nil +} + +func indentYAML(content string, spaces int) string { + indent := strings.Repeat(" ", spaces) + lines := strings.Split(content, "\n") + var result strings.Builder + for i, line := range lines { + if i > 0 { + result.WriteByte('\n') + } + if line != "" { + result.WriteString(indent) + result.WriteString(line) + } + } + return result.String() +} + +func waitForCRDs(ctx context.Context, kubeconfigPath string) error { + crds := []string{ + "volumesnapshots.snapshot.storage.k8s.io", + "volumesnapshotclasses.snapshot.storage.k8s.io", + "volumesnapshotcontents.snapshot.storage.k8s.io", + } + for _, crd := range crds { + klog.Infof("Waiting for CRD %q to become established", crd) + cmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=Established", + fmt.Sprintf("crd/%s", crd), "--timeout=5m", "--kubeconfig", kubeconfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("wait for CRD %q: %w", crd, err) + } + } + return nil +} + +func waitForWorkloads(ctx context.Context, kubeconfigPath string) error { + workloads := []struct { + kind string + name string + namespace string + }{ + {"deployment", snapshotControllerDeploymentName, kubeSystemNamespace}, + {"deployment", csiControllerDeploymentName, kubeSystemNamespace}, + {"daemonset", csiNodeDaemonSetName, kubeSystemNamespace}, + } + for _, w := range workloads { + klog.Infof("Waiting for %s %q in namespace %q to roll out", w.kind, w.name, w.namespace) + cmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", + fmt.Sprintf("%s/%s", w.kind, w.name), "-n", w.namespace, "--timeout=5m", + "--kubeconfig", kubeconfigPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("wait for %s %q rollout: %w", w.kind, w.name, err) + } + } + return nil +} + +func buildCSICloudConfig(cfg *csiInstallConfig) (string, error) { + config := map[string]any{ + "global": map[string]any{ + "projectId": cfg.ProjectID, + "region": cfg.Region, + }, + "blockStorage": map[string]any{ + "rescanOnResize": cfg.RescanOnResize, + }, + } + + content, err := yaml.Marshal(config) + if err != nil { + return "", fmt.Errorf("marshal CSI cloud config: %w", err) + } + + return string(content), nil +} + +type csiTestDriverConfig struct { + StorageClass csiTestClassConfig `yaml:"StorageClass"` + SnapshotClass csiTestClassConfig `yaml:"SnapshotClass"` + DriverInfo csiTestDriverInfo `yaml:"DriverInfo"` +} + +type csiTestClassConfig struct { + FromExistingClassName string `yaml:"FromExistingClassName"` +} + +type csiTestDriverInfo struct { + Name string `yaml:"Name"` + Capabilities csiTestDriverCapabilities `yaml:"Capabilities"` +} + +type csiTestDriverCapabilities struct { + Block bool `yaml:"block" json:"block"` + ControllerExpansion bool `yaml:"controllerExpansion" json:"controllerExpansion"` + FSGroup bool `yaml:"fsGroup" json:"fsGroup"` + Exec bool `yaml:"exec" json:"exec"` + RWX bool `yaml:"rwx" json:"rwx"` + Multipods bool `yaml:"multipods" json:"multipods"` + Persistence bool `yaml:"persistence" json:"persistence"` + PVCDataSource bool `yaml:"pvcDataSource" json:"pvcDataSource"` + SnapshotDataSource bool `yaml:"snapshotDataSource" json:"snapshotDataSource"` + Topology bool `yaml:"topology" json:"topology"` + Capacity bool `yaml:"capacity" json:"capacity"` + ReadWriteOncePod bool `yaml:"readWriteOncePod" json:"readWriteOncePod"` + MultiplePVsSameID bool `yaml:"multiplePVsSameID" json:"multiplePVsSameID"` + CapReadOnlyMany bool `yaml:"capReadOnlyMany" json:"capReadOnlyMany"` +} + +func writeCSITestDriverConfig(path string) error { + content, err := yaml.Marshal(csiTestDriverConfig{ + StorageClass: csiTestClassConfig{ + FromExistingClassName: kustomizeTestClassName, + }, + SnapshotClass: csiTestClassConfig{ + FromExistingClassName: kustomizeTestClassName, + }, + DriverInfo: csiTestDriverInfo{ + Name: kustomizeTestDriverName, + Capabilities: csiTestDriverCapabilities{ + Block: true, + ControllerExpansion: true, + FSGroup: true, + Exec: true, + RWX: false, + Multipods: false, + Persistence: true, + PVCDataSource: true, + SnapshotDataSource: true, + Topology: true, + Capacity: false, + ReadWriteOncePod: true, + MultiplePVsSameID: false, + CapReadOnlyMany: false, + }, + }, + }) + if err != nil { + return fmt.Errorf("marshal CSI testdriver config: %w", err) + } + + return os.WriteFile(path, content, 0o600) +} diff --git a/test/e2e/kubetest2-stackit/deployer/csi_reconciler_test.go b/test/e2e/kubetest2-stackit/deployer/csi_reconciler_test.go new file mode 100644 index 00000000..5c5c911c --- /dev/null +++ b/test/e2e/kubetest2-stackit/deployer/csi_reconciler_test.go @@ -0,0 +1,140 @@ +package deployer + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestBuildCSICloudConfigUsesNestedShape(t *testing.T) { + content, err := buildCSICloudConfig(&csiInstallConfig{ + ProjectID: "project-id", + Region: "eu01", + RescanOnResize: true, + }) + if err != nil { + t.Fatalf("buildCSICloudConfig() returned error: %v", err) + } + if !strings.Contains(content, "global:\n") || !strings.Contains(content, "projectId: project-id") { + t.Fatalf("expected nested global projectId in cloud config, got:\n%s", content) + } + if !strings.Contains(content, "blockStorage:\n") || !strings.Contains(content, "rescanOnResize: true") { + t.Fatalf("expected blockStorage.rescanOnResize in cloud config, got:\n%s", content) + } +} + +func TestWriteCSITestDriverConfigUsesLowercaseCapabilityKeys(t *testing.T) { + path := t.TempDir() + "/csi-testdriver.yaml" + + err := writeCSITestDriverConfig(path) + if err != nil { + t.Fatalf("writeCSITestDriverConfig() returned error: %v", err) + } + + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed reading generated CSI testdriver config: %v", err) + } + + text := string(content) + if !strings.Contains(text, "block: true") { + t.Fatalf("expected lowercase block capability key in generated testdriver config, got:\n%s", text) + } + if strings.Contains(text, "Block: true") { + t.Fatalf("unexpected uppercase block capability key in generated testdriver config, got:\n%s", text) + } +} + +func TestWriteKustomizeOverlayAppliesPlaceholders(t *testing.T) { + tmpDir := t.TempDir() + cfg := &csiInstallConfig{ + ImageName: "test-image", + ImageTag: "v1.0.0", + } + + if err := writeKustomizeOverlay(tmpDir, cfg); err != nil { + t.Fatalf("writeKustomizeOverlay() returned error: %v", err) + } + + kustomizationPath := filepath.Join(tmpDir, "kustomization.yaml") + content, err := os.ReadFile(kustomizationPath) + if err != nil { + t.Fatalf("failed reading kustomization.yaml: %v", err) + } + + kustomizationContent := string(content) + if strings.Contains(kustomizationContent, "IMAGE_NAME_PLACEHOLDER") { + t.Fatalf("kustomization.yaml still contains IMAGE_NAME_PLACEHOLDER") + } + if strings.Contains(kustomizationContent, "IMAGE_TAG_PLACEHOLDER") { + t.Fatalf("kustomization.yaml still contains IMAGE_TAG_PLACEHOLDER") + } + if strings.Contains(kustomizationContent, "CSI_DRIVER_NAME_PLACEHOLDER") { + t.Fatalf("kustomization.yaml still contains CSI_DRIVER_NAME_PLACEHOLDER") + } + if strings.Contains(kustomizationContent, "CSI_CLASS_NAME_PLACEHOLDER") { + t.Fatalf("kustomization.yaml still contains CSI_CLASS_NAME_PLACEHOLDER") + } + if !strings.Contains(kustomizationContent, "value: "+kustomizeTestDriverName) { + t.Fatalf("expected driver name %q in kustomization, got:\n%s", kustomizeTestDriverName, kustomizationContent) + } + if !strings.Contains(kustomizationContent, "value: "+kustomizeTestClassName) { + t.Fatalf("expected class name %q in kustomization, got:\n%s", kustomizeTestClassName, kustomizationContent) + } + if !strings.Contains(kustomizationContent, "newName: test-image") { + t.Fatalf("expected newName: test-image in kustomization, got:\n%s", kustomizationContent) + } + if !strings.Contains(kustomizationContent, "newTag: v1.0.0") { + t.Fatalf("expected newTag: v1.0.0 in kustomization, got:\n%s", kustomizationContent) + } + if !strings.Contains(kustomizationContent, "github.com/kubernetes-csi/external-snapshotter") { + t.Fatalf("expected remote snapshot-controller reference, got:\n%s", kustomizationContent) + } + if !strings.Contains(kustomizationContent, "deploy/csi-plugin") { + t.Fatalf("expected csi-plugin reference, got:\n%s", kustomizationContent) + } +} + +func TestIndentYAML(t *testing.T) { + tests := []struct { + name string + input string + spaces int + expected string + }{ + { + name: "empty string", + input: "", + spaces: 2, + expected: "", + }, + { + name: "single line", + input: "foo", + spaces: 2, + expected: " foo", + }, + { + name: "multiple lines", + input: "foo\nbar\nbaz", + spaces: 4, + expected: " foo\n bar\n baz", + }, + { + name: "empty lines preserved", + input: "foo\n\nbar", + spaces: 2, + expected: " foo\n\n bar", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := indentYAML(tt.input, tt.spaces) + if result != tt.expected { + t.Fatalf("indentYAML(%q, %d) = %q, want %q", tt.input, tt.spaces, result, tt.expected) + } + }) + } +} diff --git a/test/e2e/kubetest2-stackit/deployer/deployer.go b/test/e2e/kubetest2-stackit/deployer/deployer.go new file mode 100644 index 00000000..09043985 --- /dev/null +++ b/test/e2e/kubetest2-stackit/deployer/deployer.go @@ -0,0 +1,719 @@ +package deployer + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/octago/sflags/gen/gpflag" + "github.com/spf13/pflag" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + "github.com/stackitcloud/cloud-provider-stackit/pkg/version" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + ske "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +const ( + Name = "stackit" + stackitServiceAccountEnvVar = "STACKIT_SERVICE_ACCOUNT" + maxClusterNameLength = 11 + defaultNodepoolName = "default" + defaultNodes = 1 + defaultVolumeSize = 100 + defaultKubeconfigExpiration = 6 * 60 * 60 // 6 hours + defaultCSIStorageClassType = "storage_premium_perf4" + defaultCSISnapshotType = "snapshot" + defaultCSIImageName = "ghcr.io/stackitcloud/cloud-provider-stackit/stackit-csi-plugin" + defaultCSIImageTag = "1.35.5" + clusterStateFileName = "cluster-state.json" + kubeconfigFileName = "kubeconfig" + kubeconfigExpirationMin = 600 + kubeconfigExpirationMax = 15552000 +) + +type factoryBuilder func(region, projectID string) client.Factory + +type Deployer struct { + Options types.Options `flag:"-"` + + ProjectID string `flag:"~project-id" desc:"STACKIT project ID for the SKE cluster."` + Region string `flag:"~region" desc:"STACKIT region for the SKE cluster."` + KubernetesVersion string `flag:"~kubernetes-version" desc:"Kubernetes version to provision."` + AvailabilityZone string `flag:"~availability-zone" desc:"Availability zone for the nodepool."` + MachineType string `flag:"~machine-type" desc:"Machine type for the nodepool."` + NodeImageName string `flag:"~node-image-name" desc:"Node image name for the nodepool."` + NodeImageVersion string `flag:"~node-image-version" desc:"Node image version for the nodepool."` + ClusterName string `flag:"~cluster-name" desc:"Cluster name. Defaults to kt2-."` + NodepoolName string `flag:"~nodepool-name" desc:"Nodepool name."` + Nodes int `flag:"~nodes" desc:"Fixed node count for the single v1 nodepool."` + VolumeSize int `flag:"~volume-size" desc:"Root volume size in GiB for the nodepool."` + VolumeType string `flag:"~volume-type" desc:"Optional root volume type for the nodepool."` + KubeconfigExpirationSeconds int `flag:"~kubeconfig-expiration-seconds" desc:"Expiration for the admin kubeconfig in seconds."` + CSIStorageClassType string `flag:"~csi-storage-class-type" desc:"STACKIT block storage performance class for the reconciled StorageClass."` + CSISnapshotType string `flag:"~csi-snapshot-type" desc:"STACKIT snapshot type to configure for the reconciled VolumeSnapshotClass."` + CSIImageName string `flag:"~csi-image-name" desc:"Container image repository for the STACKIT CSI plugin workload containers."` + CSIImageTag string `flag:"~csi-image-tag" desc:"Container image tag for the STACKIT CSI plugin workload containers."` + + newFactory factoryBuilder + lookupEnv func(string) (string, bool) + csiReconciler csiReconciler + clientFactory client.Factory + skeClient client.SKEClient + runDir string + kubeconfigPath string + csiTestDriverPath string + clusterStatePath string + currentClusterName string + lastObservedStatus string + lastRequestPayload *ske.CreateOrUpdateClusterPayload +} + +type clusterState struct { + ClusterName string `json:"clusterName"` + ProjectID string `json:"projectId"` + Region string `json:"region"` + RequestPayload *ske.CreateOrUpdateClusterPayload `json:"requestPayload,omitempty"` + LastObservedStatus string `json:"lastObservedStatus,omitempty"` +} + +var errClusterStateNotFound = errors.New("persisted cluster state not found") + +var _ types.NewDeployer = New +var _ types.Deployer = &Deployer{} +var _ types.DeployerWithInit = &Deployer{} +var _ types.DeployerWithKubeconfig = &Deployer{} +var _ types.DeployerWithProvider = &Deployer{} +var _ types.DeployerWithVersion = &Deployer{} + +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + d := NewDeployer(opts) + + klog.InitFlags(nil) + fs := bindFlags(d) + fs.AddGoFlagSet(flag.CommandLine) + return d, fs +} + +func NewDeployer(opts types.Options) *Deployer { + return &Deployer{ + Options: opts, + ClusterName: defaultClusterName(opts.RunID()), + NodepoolName: defaultNodepoolName, + Nodes: defaultNodes, + VolumeSize: defaultVolumeSize, + KubeconfigExpirationSeconds: defaultKubeconfigExpiration, + CSIStorageClassType: defaultCSIStorageClassType, + CSISnapshotType: defaultCSISnapshotType, + CSIImageName: defaultCSIImageName, + CSIImageTag: defaultCSIImageTag, + newFactory: client.New, + lookupEnv: os.LookupEnv, + csiReconciler: newKustomizeCSIReconciler(), + } +} + +func bindFlags(d *Deployer) *pflag.FlagSet { + flags, err := gpflag.Parse(d) + if err != nil { + klog.Fatalf("unable to generate flags from deployer: %v", err) + } + + return flags +} + +func (d *Deployer) Provider() string { + return Name +} + +func (d *Deployer) Version() string { + return version.Version +} + +func (d *Deployer) Init() error { + klog.Infof("Initializing %s deployer", Name) + + if err := d.initPaths(); err != nil { + return err + } + if d.skeClient != nil { + return nil + } + + if d.clientFactory == nil { + d.clientFactory = d.newFactory(d.Region, d.ProjectID) + } + + skeOptions, err := d.skeOptions() + if err != nil { + return err + } + + klog.Infof("Creating STACKIT SKE client for project %q in region %q", d.ProjectID, d.Region) + d.skeClient, err = d.clientFactory.SKE(skeOptions) + return err +} + +func (d *Deployer) initPaths() error { + if d.runDir == "" { + d.runDir = d.Options.RunDir() + } + + var err error + d.kubeconfigPath, err = filepath.Abs(filepath.Join(d.runDir, kubeconfigFileName)) + if err != nil { + return err + } + + d.clusterStatePath, err = filepath.Abs(filepath.Join(d.runDir, clusterStateFileName)) + if err != nil { + return err + } + + d.csiTestDriverPath, err = filepath.Abs(filepath.Join(d.runDir, csiTestDriverFileName)) + if err != nil { + return err + } + + return nil +} + +func (d *Deployer) Build() error { + return nil +} + +func (d *Deployer) Up() error { + if err := d.validateUpFlags(); err != nil { + return err + } + + if _, err := d.restoreClusterStateIfPresent(); err != nil { + return err + } + if err := d.Init(); err != nil { + return err + } + + ctx := context.Background() + klog.Infof("Fetching SKE provider options for region %q", d.Region) + providerOptions, err := d.skeClient.ListProviderOptions(ctx) + if err != nil { + return err + } + if err := d.validateProviderOptions(providerOptions); err != nil { + return err + } + klog.Infof("Provider options validated for kubernetes version %q, zone %q, machine type %q", d.KubernetesVersion, d.AvailabilityZone, d.MachineType) + + payload := d.buildClusterPayload() + d.currentClusterName = d.lookupClusterName() + d.lastRequestPayload = &payload + + klog.Infof("Reconciling SKE cluster %q", d.currentClusterName) + cluster, err := d.skeClient.CreateOrUpdateCluster(ctx, d.currentClusterName, payload) + d.observeCluster(cluster) + if err != nil { + return err + } + + klog.Infof("Waiting for cluster %q to become ready", d.currentClusterName) + cluster, err = d.skeClient.WaitClusterReady(ctx, d.currentClusterName) + d.observeCluster(cluster) + if err != nil { + return err + } + klog.Infof("Cluster %q is ready with state %q", d.ClusterName, d.lastObservedStatus) + + if err := d.refreshKubeconfig(ctx, d.currentClusterName); err != nil { + return err + } + if err := d.persistClusterState(); err != nil { + return err + } + if err := d.reconcileCSI(ctx); err != nil { + return err + } + return nil +} + +func (d *Deployer) Down() error { + if d.currentClusterName == "" { + foundState, err := d.restoreClusterStateIfPresent() + if err != nil { + return err + } + if !foundState { + if err := d.initPaths(); err != nil { + return err + } + return types.NewIncorrectUsage(fmt.Sprintf( + "--down requires persisted cluster state in %q for run-id %q", + d.clusterStatePath, + d.Options.RunID(), + )) + } + } + if err := d.Init(); err != nil { + return err + } + + ctx := context.Background() + klog.Infof("Deleting SKE cluster %q", d.currentClusterName) + if err := d.skeClient.DeleteCluster(ctx, d.currentClusterName); err != nil && !stackiterrors.IsNotFound(err) { + return err + } + klog.Infof("Waiting for cluster %q to be deleted", d.currentClusterName) + if err := d.skeClient.WaitClusterDeleted(ctx, d.currentClusterName); err != nil { + return err + } + + d.lastObservedStatus = string(ske.CLUSTERSTATUSSTATE_STATE_DELETING) + d.currentClusterName = "" + if err := d.removeClusterState(); err != nil { + return err + } + klog.Infof("Cluster deletion completed") + return nil +} + +func (d *Deployer) IsUp() (bool, error) { + if _, err := d.restoreClusterStateIfPresent(); err != nil { + return false, err + } + + clusterName := d.lookupClusterName() + if clusterName == "" { + return false, nil + } + if err := d.Init(); err != nil { + return false, err + } + + cluster, err := d.skeClient.GetCluster(context.Background(), clusterName) + if err != nil { + if stackiterrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + d.observeCluster(cluster) + return clusterReady(cluster), nil +} + +func (d *Deployer) DumpClusterLogs() error { + return nil +} + +func (d *Deployer) Kubeconfig() (string, error) { + if d.kubeconfigPath == "" { + if err := d.initPaths(); err != nil { + return "", err + } + } + if d.kubeconfigPath == "" { + return "", errors.New("kubeconfig has not been created yet") + } + if _, err := os.Stat(d.kubeconfigPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", errors.New("kubeconfig has not been created yet") + } + return "", err + } + + return d.kubeconfigPath, nil +} + +func (d *Deployer) skeOptions() ([]sdkconfig.ConfigurationOption, error) { + serviceAccount, err := d.stackitServiceAccount() + if err != nil { + return nil, err + } + return []sdkconfig.ConfigurationOption{ + sdkconfig.WithServiceAccountKey(serviceAccount), + }, nil +} + +func (d *Deployer) validateUpFlags() error { + required := map[string]string{ + "project-id": d.ProjectID, + "region": d.Region, + "kubernetes-version": d.KubernetesVersion, + "availability-zone": d.AvailabilityZone, + "machine-type": d.MachineType, + "node-image-name": d.NodeImageName, + "node-image-version": d.NodeImageVersion, + } + for flagName, value := range required { + if strings.TrimSpace(value) == "" { + return types.NewIncorrectUsage(fmt.Sprintf("--%s is required", flagName)) + } + } + + if strings.TrimSpace(d.ClusterName) == "" { + return types.NewIncorrectUsage("--cluster-name must not be empty") + } + if len([]rune(d.ClusterName)) > maxClusterNameLength { + return types.NewIncorrectUsage(fmt.Sprintf("--cluster-name must be at most %d characters", maxClusterNameLength)) + } + if strings.TrimSpace(d.NodepoolName) == "" { + return types.NewIncorrectUsage("--nodepool-name must not be empty") + } + if d.Nodes < 1 { + return types.NewIncorrectUsage("--nodes must be at least 1") + } + if d.VolumeSize < 1 { + return types.NewIncorrectUsage("--volume-size must be at least 1") + } + if d.KubeconfigExpirationSeconds < kubeconfigExpirationMin || d.KubeconfigExpirationSeconds > kubeconfigExpirationMax { + return types.NewIncorrectUsage(fmt.Sprintf( + "--kubeconfig-expiration-seconds must be between %d and %d", + kubeconfigExpirationMin, + kubeconfigExpirationMax, + )) + } + csiRequired := map[string]string{ + "csi-storage-class-type": d.CSIStorageClassType, + "csi-snapshot-type": d.CSISnapshotType, + "csi-image-name": d.CSIImageName, + "csi-image-tag": d.CSIImageTag, + } + for flagName, value := range csiRequired { + if strings.TrimSpace(value) == "" { + return types.NewIncorrectUsage(fmt.Sprintf("--%s must not be empty", flagName)) + } + } + + return nil +} + +func (d *Deployer) validateProviderOptions(providerOptions *ske.ProviderOptions) error { + if providerOptions == nil { + return errors.New("received empty provider options from SKE") + } + + if !containsKubernetesVersion(providerOptions.KubernetesVersions, d.KubernetesVersion) { + return types.NewIncorrectUsage(fmt.Sprintf( + "kubernetes version %q is not available in region %q", + d.KubernetesVersion, + d.Region, + )) + } + if !containsAvailabilityZone(providerOptions.AvailabilityZones, d.AvailabilityZone) { + return types.NewIncorrectUsage(fmt.Sprintf( + "availability zone %q is not available in region %q", + d.AvailabilityZone, + d.Region, + )) + } + if !containsMachineType(providerOptions.MachineTypes, d.MachineType) { + return types.NewIncorrectUsage(fmt.Sprintf( + "machine type %q is not available in region %q", + d.MachineType, + d.Region, + )) + } + if !containsMachineImageVersion(providerOptions.MachineImages, d.NodeImageName, d.NodeImageVersion) { + return types.NewIncorrectUsage(fmt.Sprintf( + "node image %q version %q is not available in region %q", + d.NodeImageName, + d.NodeImageVersion, + d.Region, + )) + } + if d.VolumeType != "" && !containsVolumeType(providerOptions.VolumeTypes, d.VolumeType) { + return types.NewIncorrectUsage(fmt.Sprintf( + "volume type %q is not available in region %q", + d.VolumeType, + d.Region, + )) + } + + return nil +} + +func (d *Deployer) buildClusterPayload() ske.CreateOrUpdateClusterPayload { + image := ske.NewImage(d.NodeImageName, d.NodeImageVersion) + machine := ske.NewMachine(*image, d.MachineType) + volume := ske.NewVolume(int32(d.VolumeSize)) + if d.VolumeType != "" { + volume.SetType(d.VolumeType) + } + + nodepool := ske.NewNodepool( + []string{d.AvailabilityZone}, + *machine, + int32(d.Nodes), + int32(d.Nodes), + d.NodepoolName, + *volume, + ) + nodepool.SetAllowSystemComponents(true) + + return *ske.NewCreateOrUpdateClusterPayload( + *ske.NewKubernetes(d.KubernetesVersion), + []ske.Nodepool{*nodepool}, + ) +} + +func (d *Deployer) persistClusterState() error { + if err := d.initPaths(); err != nil { + return err + } + + state := clusterState{ + ClusterName: d.lookupClusterName(), + ProjectID: d.ProjectID, + Region: d.Region, + RequestPayload: d.lastRequestPayload, + LastObservedStatus: d.lastObservedStatus, + } + + content, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + + return os.WriteFile(d.clusterStatePath, content, 0o600) +} + +func (d *Deployer) loadClusterState() (*clusterState, error) { + if err := d.initPaths(); err != nil { + return nil, err + } + + content, err := os.ReadFile(d.clusterStatePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, errClusterStateNotFound + } + return nil, err + } + + var state clusterState + if err := json.Unmarshal(content, &state); err != nil { + return nil, fmt.Errorf("failed to parse persisted cluster state %q: %w", d.clusterStatePath, err) + } + if strings.TrimSpace(state.ClusterName) == "" { + return nil, fmt.Errorf("persisted cluster state %q is missing clusterName", d.clusterStatePath) + } + if strings.TrimSpace(state.ProjectID) == "" { + return nil, fmt.Errorf("persisted cluster state %q is missing projectId", d.clusterStatePath) + } + if strings.TrimSpace(state.Region) == "" { + return nil, fmt.Errorf("persisted cluster state %q is missing region", d.clusterStatePath) + } + + return &state, nil +} + +func (d *Deployer) restoreClusterStateIfPresent() (bool, error) { + if d.currentClusterName != "" { + return true, nil + } + + state, err := d.loadClusterState() + if err != nil { + if errors.Is(err, errClusterStateNotFound) { + return false, nil + } + return false, err + } + + d.ClusterName = state.ClusterName + d.ProjectID = state.ProjectID + d.Region = state.Region + d.currentClusterName = state.ClusterName + d.lastRequestPayload = state.RequestPayload + d.lastObservedStatus = state.LastObservedStatus + return true, nil +} + +func (d *Deployer) removeClusterState() error { + if err := d.initPaths(); err != nil { + return err + } + + if err := os.Remove(d.clusterStatePath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func (d *Deployer) refreshKubeconfig(ctx context.Context, clusterName string) error { + kubeconfigPayload := ske.NewCreateKubeconfigPayload() + kubeconfigPayload.SetExpirationSeconds(strconv.Itoa(d.KubeconfigExpirationSeconds)) + klog.Infof("Creating admin kubeconfig for cluster %q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, clusterName, *kubeconfigPayload) + if err != nil { + return err + } + if kubeconfig == nil || kubeconfig.Kubeconfig == nil { + return errors.New("received empty kubeconfig payload from SKE") + } + + if err := os.WriteFile(d.kubeconfigPath, []byte(*kubeconfig.Kubeconfig), 0o600); err != nil { + return err + } + klog.Infof("Wrote kubeconfig for cluster %q to %q", clusterName, d.kubeconfigPath) + return nil +} + +func (d *Deployer) stackitServiceAccount() (string, error) { + lookupEnv := d.lookupEnv + if lookupEnv == nil { + lookupEnv = os.LookupEnv + } + + serviceAccount, ok := lookupEnv(stackitServiceAccountEnvVar) + if !ok || strings.TrimSpace(serviceAccount) == "" { + klog.Errorf("Missing required environment variable %s", stackitServiceAccountEnvVar) + return "", types.NewIncorrectUsage(fmt.Sprintf("--%s requires %s to be set", Name, stackitServiceAccountEnvVar)) + } + + klog.Infof("Using STACKIT service account from %s", stackitServiceAccountEnvVar) + return serviceAccount, nil +} + +func (d *Deployer) reconcileCSI(ctx context.Context) error { + serviceAccount, err := d.stackitServiceAccount() + if err != nil { + return err + } + if d.csiReconciler == nil { + return errors.New("CSI reconciler is not configured") + } + + klog.Infof("Reconciling STACKIT CSI stack into cluster %q", d.lookupClusterName()) + return d.csiReconciler.Reconcile(ctx, &csiInstallConfig{ + KubeconfigPath: d.kubeconfigPath, + ProjectID: d.ProjectID, + Region: d.Region, + ServiceAccountJSON: serviceAccount, + StorageClassType: d.CSIStorageClassType, + SnapshotType: d.CSISnapshotType, + ImageName: d.CSIImageName, + ImageTag: d.CSIImageTag, + RescanOnResize: defaultBlockStorageRescanOnResize, + TestDriverOutputPath: d.csiTestDriverPath, + }) +} + +func (d *Deployer) lookupClusterName() string { + if d.currentClusterName != "" { + return d.currentClusterName + } + + return d.ClusterName +} + +func (d *Deployer) observeCluster(cluster *ske.Cluster) { + if cluster == nil { + return + } + + d.lastObservedStatus = clusterStatus(cluster) +} + +func clusterReady(cluster *ske.Cluster) bool { + status := clusterStatus(cluster) + return status == string(ske.CLUSTERSTATUSSTATE_STATE_HEALTHY) || status == string(ske.CLUSTERSTATUSSTATE_STATE_HIBERNATED) +} + +func clusterStatus(cluster *ske.Cluster) string { + if cluster == nil || cluster.Status == nil || cluster.Status.Aggregated == nil { + return "" + } + + return string(*cluster.Status.Aggregated) +} + +func containsKubernetesVersion(versions []ske.KubernetesVersion, expected string) bool { + for i := range versions { + if versions[i].GetVersion() == expected { + return true + } + } + + return false +} + +func containsAvailabilityZone(zones []ske.AvailabilityZone, expected string) bool { + for i := range zones { + if zones[i].GetName() == expected { + return true + } + } + + return false +} + +func containsMachineType(machineTypes []ske.MachineType, expected string) bool { + for i := range machineTypes { + if machineTypes[i].GetName() == expected { + return true + } + } + + return false +} + +func containsMachineImageVersion(images []ske.MachineImage, imageName, imageVersion string) bool { + for i := range images { + if images[i].GetName() != imageName { + continue + } + for j := range images[i].Versions { + if images[i].Versions[j].GetVersion() == imageVersion { + return true + } + } + } + + return false +} + +func containsVolumeType(volumeTypes []ske.VolumeType, expected string) bool { + for i := range volumeTypes { + if volumeTypes[i].GetName() == expected { + return true + } + } + + return false +} + +func defaultClusterName(runID string) string { + name := strings.ToLower(runID) + name = strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z': + return r + case r >= '0' && r <= '9': + return r + case r == '-': + return r + default: + return '-' + } + }, name) + name = strings.Trim(name, "-") + if name == "" { + name = "run" + } + + maxSuffixLength := maxClusterNameLength - len("kt2-") + if len(name) > maxSuffixLength { + name = name[:maxSuffixLength] + } + + return "kt2-" + name +} diff --git a/test/e2e/kubetest2-stackit/deployer/deployer_test.go b/test/e2e/kubetest2-stackit/deployer/deployer_test.go new file mode 100644 index 00000000..64f281d8 --- /dev/null +++ b/test/e2e/kubetest2-stackit/deployer/deployer_test.go @@ -0,0 +1,887 @@ +package deployer + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + client "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client" + clientmock "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client/mock" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + ske "github.com/stackitcloud/stackit-sdk-go/services/ske/v2api" + "go.uber.org/mock/gomock" + "sigs.k8s.io/kubetest2/pkg/types" +) + +type fakeOptions struct { + runID string + runDir string +} + +func (f fakeOptions) HelpRequested() bool { return false } +func (f fakeOptions) ShouldBuild() bool { return false } +func (f fakeOptions) ShouldUp() bool { return false } +func (f fakeOptions) ShouldDown() bool { return false } +func (f fakeOptions) ShouldTest() bool { return false } +func (f fakeOptions) SkipTestJUnitReport() bool { return false } +func (f fakeOptions) RunID() string { return f.runID } +func (f fakeOptions) RunDir() string { return f.runDir } +func (f fakeOptions) RundirInArtifacts() bool { return false } +func (f fakeOptions) PostTestCmd() []string { return nil } +func (f fakeOptions) PreTestCmd() []string { return nil } + +func TestNewDefaults(t *testing.T) { + opts := fakeOptions{runID: "1234567890abcdef", runDir: t.TempDir()} + + d := NewDeployer(opts) + if d.ClusterName != "kt2-1234567" { + t.Fatalf("unexpected default cluster name: %q", d.ClusterName) + } + if d.NodepoolName != defaultNodepoolName { + t.Fatalf("unexpected default nodepool name: %q", d.NodepoolName) + } + if d.Nodes != defaultNodes { + t.Fatalf("unexpected default nodes: %d", d.Nodes) + } + if d.VolumeSize != defaultVolumeSize { + t.Fatalf("unexpected default volume size: %d", d.VolumeSize) + } + if d.KubeconfigExpirationSeconds != defaultKubeconfigExpiration { + t.Fatalf("unexpected kubeconfig expiration: %d", d.KubeconfigExpirationSeconds) + } + if d.CSIImageName != defaultCSIImageName || d.CSIImageTag != defaultCSIImageTag { + t.Fatalf("unexpected default CSI image override: %s:%s", d.CSIImageName, d.CSIImageTag) + } + + _, fs := New(opts) + for _, name := range []string{ + "project-id", + "cluster-name", + "kubeconfig-expiration-seconds", + "csi-image-name", + "csi-image-tag", + } { + if fs.Lookup(name) == nil { + t.Fatalf("expected flag %q to be registered", name) + } + } +} + +func TestInitRequiresServiceAccountEnv(t *testing.T) { + ctrl := gomock.NewController(t) + mockFactory := clientmock.NewMockFactory(ctrl) + + d := newConfiguredDeployer(t) + d.clientFactory = mockFactory + d.lookupEnv = func(string) (string, bool) { + return "", false + } + + err := d.Init() + if err == nil { + t.Fatal("expected error") + } + if _, ok := err.(types.IncorrectUsage); !ok { + t.Fatalf("expected IncorrectUsage, got %T", err) + } + if !strings.Contains(err.Error(), stackitServiceAccountEnvVar) { + t.Fatalf("expected error to mention %s, got %q", stackitServiceAccountEnvVar, err.Error()) + } +} + +func TestInitBuildsSKEClientWithServiceAccount(t *testing.T) { + ctrl := gomock.NewController(t) + mockFactory := clientmock.NewMockFactory(ctrl) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.clientFactory = mockFactory + d.lookupEnv = func(string) (string, bool) { + return `{"credentials":{"private_key":"key"}}`, true + } + + mockFactory.EXPECT(). + SKE(gomock.Any()). + DoAndReturn(func(options []sdkconfig.ConfigurationOption) (client.SKEClient, error) { + if len(options) != 1 { + t.Fatalf("expected one sdk option, got %d", len(options)) + } + return mockSKE, nil + }) + + if err := d.Init(); err != nil { + t.Fatalf("Init() returned error: %v", err) + } + if d.skeClient == nil { + t.Fatal("expected SKE client to be initialized") + } +} + +func TestPersistClusterStateRoundTrip(t *testing.T) { + d := newConfiguredDeployer(t) + payload := d.buildClusterPayload() + d.currentClusterName = "kt2-test123" + d.lastRequestPayload = &payload + d.lastObservedStatus = string(ske.CLUSTERSTATUSSTATE_STATE_HEALTHY) + + if err := d.persistClusterState(); err != nil { + t.Fatalf("persistClusterState() returned error: %v", err) + } + + state, err := d.loadClusterState() + if err != nil { + t.Fatalf("loadClusterState() returned error: %v", err) + } + if state.ClusterName != "kt2-test123" { + t.Fatalf("unexpected cluster name: %q", state.ClusterName) + } + if state.ProjectID != d.ProjectID { + t.Fatalf("unexpected project ID: %q", state.ProjectID) + } + if state.Region != d.Region { + t.Fatalf("unexpected region: %q", state.Region) + } + if state.LastObservedStatus != "STATE_HEALTHY" { + t.Fatalf("unexpected status: %q", state.LastObservedStatus) + } + if state.RequestPayload == nil || state.RequestPayload.Kubernetes.Version != "1.32.1" { + t.Fatalf("unexpected request payload: %#v", state.RequestPayload) + } +} + +func TestLoadClusterStateMissing(t *testing.T) { + d := newConfiguredDeployer(t) + + _, err := d.loadClusterState() + if !errors.Is(err, errClusterStateNotFound) { + t.Fatalf("expected errClusterStateNotFound, got %v", err) + } +} + +func TestLoadClusterStateMalformed(t *testing.T) { + d := newConfiguredDeployer(t) + if err := d.initPaths(); err != nil { + t.Fatalf("initPaths() returned error: %v", err) + } + if err := os.WriteFile(d.clusterStatePath, []byte("{"), 0o600); err != nil { + t.Fatalf("failed writing malformed state: %v", err) + } + + _, err := d.loadClusterState() + if err == nil || !strings.Contains(err.Error(), "failed to parse persisted cluster state") { + t.Fatalf("expected malformed state error, got %v", err) + } +} + +func TestUpRequiresFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mutate func(*Deployer) + contains string + }{ + { + name: "project id", + mutate: func(d *Deployer) { d.ProjectID = "" }, + contains: "--project-id is required", + }, + { + name: "region", + mutate: func(d *Deployer) { d.Region = "" }, + contains: "--region is required", + }, + { + name: "kubernetes version", + mutate: func(d *Deployer) { d.KubernetesVersion = "" }, + contains: "--kubernetes-version is required", + }, + { + name: "node count", + mutate: func(d *Deployer) { d.Nodes = 0 }, + contains: "--nodes must be at least 1", + }, + { + name: "kubeconfig expiration", + mutate: func(d *Deployer) { d.KubeconfigExpirationSeconds = 42 }, + contains: "--kubeconfig-expiration-seconds must be between", + }, + { + name: "cluster name too long", + mutate: func(d *Deployer) { d.ClusterName = "kt2-name-too-long" }, + contains: "--cluster-name must be at most 11 characters", + }, + { + name: "csi image tag", + mutate: func(d *Deployer) { d.CSIImageTag = "" }, + contains: "--csi-image-tag must not be empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := newConfiguredDeployer(t) + tt.mutate(d) + + err := d.Up() + if err == nil { + t.Fatal("expected error") + } + if _, ok := err.(types.IncorrectUsage); !ok { + t.Fatalf("expected IncorrectUsage, got %T", err) + } + if !strings.Contains(err.Error(), tt.contains) { + t.Fatalf("expected error to contain %q, got %q", tt.contains, err.Error()) + } + }) + } +} + +func TestProviderOptionValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + options *ske.ProviderOptions + contains string + }{ + { + name: "kubernetes version", + options: baseProviderOptions("1.31.6", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + contains: `kubernetes version "1.32.1" is not available`, + }, + { + name: "availability zone", + options: baseProviderOptions("1.32.1", "eu01-2", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + contains: `availability zone "eu01-1" is not available`, + }, + { + name: "machine type", + options: baseProviderOptions("1.32.1", "eu01-1", "g1.4", "flatcar", "3033.2.4", "storage_premium_perf0"), + contains: `machine type "g1.2" is not available`, + }, + { + name: "image version", + options: baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "9999.0.0", "storage_premium_perf0"), + contains: `node image "flatcar" version "3033.2.4" is not available`, + }, + { + name: "volume type", + options: baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf4"), + contains: `volume type "storage_premium_perf0" is not available`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return(tt.options, nil) + + err := d.Up() + if err == nil { + t.Fatal("expected error") + } + if _, ok := err.(types.IncorrectUsage); !ok { + t.Fatalf("expected IncorrectUsage, got %T", err) + } + if !strings.Contains(err.Error(), tt.contains) { + t.Fatalf("expected error to contain %q, got %q", tt.contains, err.Error()) + } + }) + } +} + +func TestUpBuildsPayloadAndWritesArtifacts(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + reconciler := d.csiReconciler.(*recordingCSIReconciler) + d.skeClient = mockSKE + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + DoAndReturn(func(_ context.Context, _ string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + if payload.Kubernetes.Version != "1.32.1" { + t.Fatalf("unexpected kubernetes version: %q", payload.Kubernetes.Version) + } + if len(payload.Nodepools) != 1 { + t.Fatalf("expected one nodepool, got %d", len(payload.Nodepools)) + } + + nodepool := payload.Nodepools[0] + if nodepool.Name != "np-default" { + t.Fatalf("unexpected nodepool name: %q", nodepool.Name) + } + if nodepool.Minimum != 2 || nodepool.Maximum != 2 { + t.Fatalf("unexpected node bounds: min=%d max=%d", nodepool.Minimum, nodepool.Maximum) + } + if !nodepool.GetAllowSystemComponents() { + t.Fatal("expected allowSystemComponents to be true") + } + if nodepool.Machine.Type != "g1.2" { + t.Fatalf("unexpected machine type: %q", nodepool.Machine.Type) + } + if nodepool.Machine.Image.Name != "flatcar" || nodepool.Machine.Image.Version != "3033.2.4" { + t.Fatalf("unexpected image: %#v", nodepool.Machine.Image) + } + if len(nodepool.AvailabilityZones) != 1 || nodepool.AvailabilityZones[0] != "eu01-1" { + t.Fatalf("unexpected availability zones: %#v", nodepool.AvailabilityZones) + } + if nodepool.Volume.Size != 150 { + t.Fatalf("unexpected volume size: %d", nodepool.Volume.Size) + } + if nodepool.Volume.GetType() != "storage_premium_perf0" { + t.Fatalf("unexpected volume type: %q", nodepool.Volume.GetType()) + } + + return newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_CREATING), nil + }) + + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + DoAndReturn(func(_ context.Context, _ string, payload ske.CreateKubeconfigPayload) (*ske.Kubeconfig, error) { + if payload.GetExpirationSeconds() != "7200" { + t.Fatalf("unexpected kubeconfig expiration: %q", payload.GetExpirationSeconds()) + } + content := "apiVersion: v1\nclusters: []\n" + return &ske.Kubeconfig{Kubeconfig: &content}, nil + }) + + if err := d.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } + + kubeconfigPath, err := d.Kubeconfig() + if err != nil { + t.Fatalf("Kubeconfig() returned error: %v", err) + } + if !filepath.IsAbs(kubeconfigPath) { + t.Fatalf("expected absolute kubeconfig path, got %q", kubeconfigPath) + } + + kubeconfigContent, err := os.ReadFile(kubeconfigPath) + if err != nil { + t.Fatalf("failed reading kubeconfig: %v", err) + } + if !strings.Contains(string(kubeconfigContent), "apiVersion: v1") { + t.Fatalf("unexpected kubeconfig content: %q", string(kubeconfigContent)) + } + + clusterStateContent, err := os.ReadFile(d.clusterStatePath) + if err != nil { + t.Fatalf("failed reading persisted cluster state: %v", err) + } + if !strings.Contains(string(clusterStateContent), `"projectId": "project-id"`) { + t.Fatalf("persisted state missing project ID: %s", clusterStateContent) + } + if !strings.Contains(string(clusterStateContent), `"region": "eu01"`) { + t.Fatalf("persisted state missing region: %s", clusterStateContent) + } + if len(reconciler.calls) != 1 { + t.Fatalf("expected one CSI reconcile call, got %d", len(reconciler.calls)) + } + testDriverContent, err := os.ReadFile(d.csiTestDriverPath) + if err != nil { + t.Fatalf("failed reading generated CSI testdriver config: %v", err) + } + if !strings.Contains(string(testDriverContent), "FromExistingClassName: "+kustomizeTestClassName) { + t.Fatalf("generated testdriver config missing storage class: %s", testDriverContent) + } + if !strings.Contains(string(testDriverContent), "Name: "+kustomizeTestDriverName) { + t.Fatalf("generated testdriver config missing driver name: %s", testDriverContent) + } +} + +func TestUpReusesHealthyClusterFromPersistedState(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + reconciler := d.csiReconciler.(*recordingCSIReconciler) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + LastObservedStatus: "STATE_HEALTHY", + }) + if err := os.WriteFile(filepath.Join(d.Options.RunDir(), kubeconfigFileName), []byte("stale"), 0o600); err != nil { + t.Fatalf("failed writing stale kubeconfig: %v", err) + } + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + DoAndReturn(func(_ context.Context, _ string, payload ske.CreateKubeconfigPayload) (*ske.Kubeconfig, error) { + if payload.GetExpirationSeconds() != "7200" { + t.Fatalf("unexpected kubeconfig expiration: %q", payload.GetExpirationSeconds()) + } + content := "apiVersion: v1\nclusters:\n- reused\n" + return &ske.Kubeconfig{Kubeconfig: &content}, nil + }) + + if err := d.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } + + content, err := os.ReadFile(d.kubeconfigPath) + if err != nil { + t.Fatalf("failed reading kubeconfig: %v", err) + } + if !strings.Contains(string(content), "reused") { + t.Fatalf("expected fresh kubeconfig content, got %q", string(content)) + } + if len(reconciler.calls) != 1 { + t.Fatalf("expected one CSI reconcile call on reuse, got %d", len(reconciler.calls)) + } +} + +func TestUpReusesHibernatedClusterFromPersistedState(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + }) + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HIBERNATED), nil) + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HIBERNATED), nil) + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + Return(&ske.Kubeconfig{Kubeconfig: stringPtr("apiVersion: v1\n")}, nil) + + if err := d.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } +} + +func TestUpIgnoresStalePersistedStateAndCreatesCluster(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + }) + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_CREATING), nil) + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + Return(&ske.Kubeconfig{Kubeconfig: stringPtr("apiVersion: v1\nclusters:\n- recreated\n")}, nil) + + if err := d.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } +} + +func TestUpIsIdempotentWithPersistedState(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + }) + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_CREATING), nil) + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + Return(&ske.Kubeconfig{Kubeconfig: stringPtr("apiVersion: v1\nclusters:\n- converged\n")}, nil) + + if err := d.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } +} + +func TestUpFailsWhenCSIReconciliationFailsAfterClusterReady(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + reconciler := d.csiReconciler.(*recordingCSIReconciler) + reconciler.err = errors.New("csi install failed") + reconciler.writeTestDriver = false + d.skeClient = mockSKE + + mockSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + mockSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_CREATING), nil) + mockSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + mockSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + Return(&ske.Kubeconfig{Kubeconfig: stringPtr("apiVersion: v1\nclusters:\n- ready\n")}, nil) + + err := d.Up() + if err == nil || !strings.Contains(err.Error(), "csi install failed") { + t.Fatalf("expected CSI reconciliation error, got %v", err) + } + if len(reconciler.calls) != 1 { + t.Fatalf("expected one CSI reconcile call, got %d", len(reconciler.calls)) + } + if _, statErr := os.Stat(d.clusterStatePath); statErr != nil { + t.Fatalf("expected cluster state to remain after CSI failure, got %v", statErr) + } +} + +func TestIsUp(t *testing.T) { + tests := []struct { + name string + status ske.ClusterStatusState + up bool + }{ + {name: "healthy", status: ske.CLUSTERSTATUSSTATE_STATE_HEALTHY, up: true}, + {name: "hibernated", status: ske.CLUSTERSTATUSSTATE_STATE_HIBERNATED, up: true}, + {name: "creating", status: ske.CLUSTERSTATUSSTATE_STATE_CREATING, up: false}, + {name: "deleting", status: ske.CLUSTERSTATUSSTATE_STATE_DELETING, up: false}, + {name: "unhealthy", status: ske.CLUSTERSTATUSSTATE_STATE_UNHEALTHY, up: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + + mockSKE.EXPECT(). + GetCluster(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", tt.status), nil) + + up, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() returned error: %v", err) + } + if up != tt.up { + t.Fatalf("expected up=%t, got %t", tt.up, up) + } + }) + } +} + +func TestDownRequiresPersistedState(t *testing.T) { + d := newConfiguredDeployer(t) + + err := d.Down() + if err == nil { + t.Fatal("expected error") + } + if _, ok := err.(types.IncorrectUsage); !ok { + t.Fatalf("expected IncorrectUsage, got %T", err) + } + if !strings.Contains(err.Error(), clusterStateFileName) { + t.Fatalf("expected error to mention persisted state file, got %q", err.Error()) + } +} + +func TestDownDeleteAndWait(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + d.currentClusterName = "kt2-test123" + + mockSKE.EXPECT().DeleteCluster(gomock.Any(), "kt2-test123").Return(nil) + mockSKE.EXPECT().WaitClusterDeleted(gomock.Any(), "kt2-test123").Return(nil) + + if err := d.Down(); err != nil { + t.Fatalf("Down() returned error: %v", err) + } + if d.currentClusterName != "" { + t.Fatalf("expected current cluster name to be cleared, got %q", d.currentClusterName) + } +} + +func TestDownDeleteAndWaitWithPersistedState(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + }) + + mockSKE.EXPECT().DeleteCluster(gomock.Any(), "kt2-test123").Return(nil) + mockSKE.EXPECT().WaitClusterDeleted(gomock.Any(), "kt2-test123").Return(nil) + + if err := d.Down(); err != nil { + t.Fatalf("Down() returned error: %v", err) + } + if _, err := os.Stat(d.clusterStatePath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected persisted state to be removed, got %v", err) + } +} + +func TestDownAlreadyGoneWithPersistedState(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + writeClusterState(t, d, clusterState{ + ClusterName: "kt2-test123", + ProjectID: "project-id", + Region: "eu01", + }) + + mockSKE.EXPECT(). + DeleteCluster(gomock.Any(), "kt2-test123"). + Return(notFoundError()) + mockSKE.EXPECT().WaitClusterDeleted(gomock.Any(), "kt2-test123").Return(nil) + + if err := d.Down(); err != nil { + t.Fatalf("Down() returned error: %v", err) + } +} + +func TestDownWaitError(t *testing.T) { + ctrl := gomock.NewController(t) + mockSKE := clientmock.NewMockSKEClient(ctrl) + + d := newConfiguredDeployer(t) + d.skeClient = mockSKE + d.currentClusterName = "kt2-test123" + + mockSKE.EXPECT().DeleteCluster(gomock.Any(), "kt2-test123").Return(nil) + mockSKE.EXPECT().WaitClusterDeleted(gomock.Any(), "kt2-test123").Return(errors.New("timeout")) + + err := d.Down() + if err == nil || !strings.Contains(err.Error(), "timeout") { + t.Fatalf("expected timeout error, got %v", err) + } +} + +func TestRunIDReuseAcrossDeployers(t *testing.T) { + runDir := t.TempDir() + opts := fakeOptions{ + runID: "1234567890abcdef", + runDir: runDir, + } + + createCtrl := gomock.NewController(t) + createSKE := clientmock.NewMockSKEClient(createCtrl) + upDeployer := newConfiguredDeployerWithOptions(opts) + upDeployer.skeClient = createSKE + + createSKE.EXPECT().ListProviderOptions(gomock.Any()).Return( + baseProviderOptions("1.32.1", "eu01-1", "g1.2", "flatcar", "3033.2.4", "storage_premium_perf0"), + nil, + ) + createSKE.EXPECT(). + CreateOrUpdateCluster(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateOrUpdateClusterPayload{})). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_CREATING), nil) + createSKE.EXPECT(). + WaitClusterReady(gomock.Any(), "kt2-test123"). + Return(newCluster("kt2-test123", ske.CLUSTERSTATUSSTATE_STATE_HEALTHY), nil) + createSKE.EXPECT(). + CreateKubeconfig(gomock.Any(), "kt2-test123", gomock.AssignableToTypeOf(ske.CreateKubeconfigPayload{})). + Return(&ske.Kubeconfig{Kubeconfig: stringPtr("apiVersion: v1\n")}, nil) + + if err := upDeployer.Up(); err != nil { + t.Fatalf("Up() returned error: %v", err) + } + + deleteCtrl := gomock.NewController(t) + deleteSKE := clientmock.NewMockSKEClient(deleteCtrl) + downDeployer := newConfiguredDeployerWithOptions(opts) + downDeployer.skeClient = deleteSKE + + if err := upDeployer.initPaths(); err != nil { + t.Fatalf("initPaths() returned error: %v", err) + } + if err := downDeployer.initPaths(); err != nil { + t.Fatalf("initPaths() returned error: %v", err) + } + if upDeployer.clusterStatePath != downDeployer.clusterStatePath { + t.Fatalf("expected same cluster state path, got %q and %q", upDeployer.clusterStatePath, downDeployer.clusterStatePath) + } + + deleteSKE.EXPECT().DeleteCluster(gomock.Any(), "kt2-test123").Return(nil) + deleteSKE.EXPECT().WaitClusterDeleted(gomock.Any(), "kt2-test123").Return(nil) + + if err := downDeployer.Down(); err != nil { + t.Fatalf("Down() returned error: %v", err) + } +} + +func newConfiguredDeployer(t *testing.T) *Deployer { + t.Helper() + return newConfiguredDeployerWithOptions(fakeOptions{ + runID: "1234567890abcdef", + runDir: t.TempDir(), + }) +} + +func newConfiguredDeployerWithOptions(opts fakeOptions) *Deployer { + d := NewDeployer(opts) + d.ProjectID = "project-id" + d.Region = "eu01" + d.ClusterName = "kt2-test123" + d.KubernetesVersion = "1.32.1" + d.AvailabilityZone = "eu01-1" + d.MachineType = "g1.2" + d.NodeImageName = "flatcar" + d.NodeImageVersion = "3033.2.4" + d.NodepoolName = "np-default" + d.Nodes = 2 + d.VolumeSize = 150 + d.VolumeType = "storage_premium_perf0" + d.KubeconfigExpirationSeconds = 7200 + d.csiReconciler = &recordingCSIReconciler{writeTestDriver: true} + d.lookupEnv = func(string) (string, bool) { + return `{"credentials":{"private_key":"key"}}`, true + } + return d +} + +type recordingCSIReconciler struct { + calls []csiInstallConfig + err error + writeTestDriver bool +} + +func (r *recordingCSIReconciler) Reconcile(_ context.Context, cfg *csiInstallConfig) error { + r.calls = append(r.calls, *cfg) + if r.err != nil { + return r.err + } + if r.writeTestDriver { + return writeCSITestDriverConfig(cfg.TestDriverOutputPath) + } + return nil +} + +func writeClusterState(t *testing.T, d *Deployer, state clusterState) { + t.Helper() + + if err := d.initPaths(); err != nil { + t.Fatalf("initPaths() returned error: %v", err) + } + content, err := json.MarshalIndent(state, "", " ") + if err != nil { + t.Fatalf("failed marshalling cluster state: %v", err) + } + if err := os.WriteFile(d.clusterStatePath, content, 0o600); err != nil { + t.Fatalf("failed writing cluster state: %v", err) + } +} + +func baseProviderOptions(version, az, machineType, imageName, imageVersion, volumeType string) *ske.ProviderOptions { + return &ske.ProviderOptions{ + KubernetesVersions: []ske.KubernetesVersion{ + {Version: &version}, + }, + AvailabilityZones: []ske.AvailabilityZone{ + {Name: &az}, + }, + MachineTypes: []ske.MachineType{ + {Name: &machineType}, + }, + MachineImages: []ske.MachineImage{ + { + Name: &imageName, + Versions: []ske.MachineImageVersion{ + {Version: &imageVersion}, + }, + }, + }, + VolumeTypes: []ske.VolumeType{ + {Name: &volumeType}, + }, + } +} + +func newCluster(name string, state ske.ClusterStatusState) *ske.Cluster { + return &ske.Cluster{ + Name: &name, + Status: &ske.ClusterStatus{ + Aggregated: &state, + }, + Kubernetes: *ske.NewKubernetes("1.32.1"), + Nodepools: []ske.Nodepool{}, + } +} + +func notFoundError() error { + return &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound} +} + +func stringPtr(value string) *string { + return &value +} diff --git a/test/e2e/kustomize/kustomization.yaml b/test/e2e/kustomize/kustomization.yaml new file mode 100644 index 00000000..7ed9e5cb --- /dev/null +++ b/test/e2e/kustomize/kustomization.yaml @@ -0,0 +1,96 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v8.3.0 +- github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v8.3.0 +- ../../../deploy/csi-plugin +- snapshotclass.yaml + +images: +- name: ghcr.io/stackitcloud/cloud-provider-stackit/stackit-csi-plugin + newName: IMAGE_NAME_PLACEHOLDER + newTag: IMAGE_TAG_PLACEHOLDER + +patches: +- target: + kind: CSIDriver + patch: |- + - op: replace + path: /metadata/name + value: CSI_DRIVER_NAME_PLACEHOLDER +- target: + kind: StorageClass + patch: |- + - op: replace + path: /metadata/name + value: CSI_CLASS_NAME_PLACEHOLDER + - op: replace + path: /provisioner + value: CSI_DRIVER_NAME_PLACEHOLDER +- target: + kind: VolumeSnapshotClass + patch: |- + - op: replace + path: /metadata/name + value: CSI_CLASS_NAME_PLACEHOLDER + - op: replace + path: /driver + value: CSI_DRIVER_NAME_PLACEHOLDER +- target: + kind: Deployment + name: csi-stackit-controllerplugin + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: csi-stackit-controllerplugin + spec: + template: + spec: + containers: + - name: stackit-csi-plugin + imagePullPolicy: Always + env: + - name: CSI_DRIVER_NAME + value: CSI_DRIVER_NAME_PLACEHOLDER +- target: + kind: DaemonSet + name: csi-stackit-nodeplugin + patch: |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: csi-stackit-nodeplugin + spec: + template: + spec: + containers: + - name: stackit-csi-plugin + imagePullPolicy: Always + env: + - name: CSI_DRIVER_NAME + value: CSI_DRIVER_NAME_PLACEHOLDER + - name: node-driver-registrar + env: + - name: CSI_DRIVER_NAME + value: CSI_DRIVER_NAME_PLACEHOLDER + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/CSI_DRIVER_NAME_PLACEHOLDER/csi.sock + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/CSI_DRIVER_NAME_PLACEHOLDER +- target: + kind: Deployment + name: snapshot-controller + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: snapshot-controller + spec: + template: + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet diff --git a/test/e2e/kustomize/snapshotclass.yaml b/test/e2e/kustomize/snapshotclass.yaml new file mode 100644 index 00000000..89d6dfb2 --- /dev/null +++ b/test/e2e/kustomize/snapshotclass.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: stackit +driver: block-storage.csi.stackit.cloud +deletionPolicy: Delete +parameters: + type: "snapshot"