feat: provider migration testing

Signed-off-by: Artur Troian <[email protected]>
akash-network · Sep 1, 2023 · 5834b4b · 5834b4b
1 parent 7432e8b
commit 5834b4b
Show file tree

Hide file tree

Showing 22 changed files with 522 additions and 223 deletions.
diff --git a/_run/common-commands.mk b/_run/common-commands.mk
@@ -18,6 +18,13 @@ JWT_AUTH_HOSTNAME    ?= localhost
 JWT_AUTH_HOST        ?= $(JWT_AUTH_HOSTNAME):8444
 RESOURCE_SERVER_HOST ?= localhost:8445
 
+BLOCK_PERIOD               := 6.5
+LATEST_BLOCK_HEIGHT         = $(shell curl -sS "$(AKASH_NODE)/status?" | jq -r '.result.sync_info.latest_block_height')
+VOTING_PERIOD               = $(shell $(AKASH) query params subspace gov votingparams -o json | jq -r '.value' | sed 's/\\//g' | jq -r '.voting_period')
+MIN_DEPOSIT                 = $(shell $(AKASH) query params subspace gov depositparams -o json | jq -r '.value' | sed 's/\\//g' | jq -r '.min_deposit[0] | (.amount + .denom)')
+UPGRADE_HEIGHT_AFTER_PASS  ?= 20
+CALCULATE_UPGRADE_HEIGHT    = $(shell echo "$(LATEST_BLOCK_HEIGHT) + $(UPGRADE_HEIGHT_AFTER_PASS) + (($(VOTING_PERIOD) / 1000000000) / $(BLOCK_PERIOD))" | bc)
+
 .PHONY: multisig-send
 multisig-send:
 	$(AKASH) tx send \
@@ -84,7 +91,7 @@ send-manifest:
 
 .PHONY: deployment-create
 deployment-create:
-	$(AKASH) tx deployment create "$(SDL_PATH)" \
+	$(PROVIDER_SERVICES) tx deployment create "$(SDL_PATH)" \
 		--dseq "$(DSEQ)" \
 		--from "$(KEY_NAME)"
 
@@ -262,6 +269,12 @@ query-account-certificates:
 create-server-certificate:
 	$(AKASH) tx cert create server $(CERT_HOSTNAME) --from=$(KEY_NAME) --rie
 
+.PHONY: create-client-certificate
+create-client-certificate:
+	$(AKASH) tx cert generate client --from=$(KEY_NAME)
+	$(AKASH) tx cert publish client --from=$(KEY_NAME)
+
+
 .PHONY: revoke-certificate
 revoke-certificate:
 	$(AKASH) tx cert revoke --from=$(KEY_NAME)
@@ -304,3 +317,43 @@ provider-service-status:
 		--oseq      "$(OSEQ)"        \
 		--from      "$(KEY_NAME)" \
 		--provider  "$(PROVIDER_ADDRESS)"
+
+UPGRADE_NAME   ?= v0.24.0
+PROP_SUBMITTER ?= main
+
+.PHONY: prepare
+prepare:
+	docker pull $(AKASH_DOCKER_IMAGE)
+	docker pull $(DOCKER_IMAGE)
+	curl https://raw.githubusercontent.com/akash-network/provider/main/install.sh | bash -s -- -b $(AP_DEVCACHE_BIN) v0.2.1
+	curl https://raw.githubusercontent.com/akash-network/node/main/install.sh | bash -s -- -b $(AP_DEVCACHE_BIN) v0.22.9
+
+.PHONY: send-upgrade
+send-upgrade:
+	$(AKASH) tx gov submit-proposal software-upgrade $(UPGRADE_NAME) \
+		--yes \
+		--title=$(UPGRADE_NAME) \
+		--description=$(UPGRADE_NAME) \
+		--upgrade-height=$(CALCULATE_UPGRADE_HEIGHT) \
+		--deposit=$(MIN_DEPOSIT) \
+		--from=$(PROP_SUBMITTER)
+	$(AKASH) tx gov vote 1 yes -y --from validator
+
+PROVIDER_MIGRATE_DRYRUN ?= true
+
+.PHONY: provider-migrate
+provider-migrate:
+	$(PROVIDER_SERVICES) migrate v2beta2 \
+		--crd-dry-run=$(PROVIDER_MIGRATE_DRYRUN) \
+		--crd-v2beta1=https://raw.githubusercontent.com/akash-network/provider/v0.2.1/pkg/apis/akash.network/crd.yaml \
+		--crd-v2beta2=../../pkg/apis/akash.network/crd.yaml \
+		--crd-backup-path=$(AP_RUN_DIR)/migrate \
+		--from=$(PROVIDER_KEY_NAME)
+
+.PHONY: post-upgrade
+post-upgrade: post-upgrade-clean bins
+
+.PHONY: post-upgrade-clean
+post-upgrade-clean:
+	rm -f $(AP_DEVCACHE_BIN)/akash
+	rm -f $(AP_DEVCACHE_BIN)/provider-services
diff --git a/_run/common-kube.mk b/_run/common-kube.mk
@@ -85,7 +85,8 @@ kube-upload-images-default:
 kube-upload-crd:
 	$(SETUP_KUBE) --crd=$(CRD_FILE) $(KUBE_SSH_NODE_NAME) init
 
-$(KUBE_CREATE): $(AP_RUN_DIR) kube-cluster-create-$(KUBE_CLUSTER_CREATE_TARGET) kube-upload-crd
+$(KUBE_CREATE): $(AP_RUN_DIR) kube-cluster-create-$(KUBE_CLUSTER_CREATE_TARGET)
+	$(SETUP_KUBE) --crd=$(CRD_FILE) $(KUBE_SSH_NODE_NAME) init
 	touch $@
 
 .INTERMEDIATE: kube-cluster-create-default

diff --git a/_run/kube/deployment.yaml b/_run/kube/deployment.yaml
@@ -1,55 +1,39 @@
----
 version: "2.0"
-
 services:
-  web:
-    image: quay.io/ovrclk/demo-app
+  supermario1:
+    image: scarruthers/simplewebsite:3
     expose:
       - port: 80
-        as: 80
-        http_options:
-          max_body_size: 2097152
-          next_cases:
-            - off
-        accept:
-          - hello.localhost
         to:
           - global: true
-  bew:
-    image: quay.io/ovrclk/demo-app
+  supermario2:
+    image: scarruthers/simplewebsite:4
     expose:
       - port: 80
-        as: 80
-        accept:
-          - hello1.localhost
         to:
           - global: true
-
 profiles:
   compute:
-    web:
+    supermarioprofile:
       resources:
         cpu:
           units: 0.1
         memory:
-          size: 16Mi
+          size: 1Gi
         storage:
-          size: 128Mi
+          size: 1Gi
   placement:
-    westcoast:
-      attributes:
-        region: us-west
+    akash:
       pricing:
-        web: 
+        supermarioprofile:
           denom: uakt
-          amount: 1000
-
+          amount: 100000
 deployment:
-  web:
-    westcoast:
-      profile: web
+  supermario1:
+    akash:
+      profile: supermarioprofile
+      count: 2
+  supermario2:
+    akash:
+      profile: supermarioprofile
       count: 1
-  bew:
-    westcoast:
-      profile: web
-      count: 1
diff --git a/_run/kube/grafana.yaml b/_run/kube/grafana.yaml
@@ -4,6 +4,8 @@ version: "2.0"
 services:
   grafana:
     image: grafana/grafana
+    env:
+      - "BLA=test"
     expose:
       - port: 3000
         as: 80

diff --git a/_run/kube/provider-migration.md b/_run/kube/provider-migration.md
@@ -0,0 +1,135 @@
+### Resetting state
+if at any point fresh start needed:
+ - make sure node is not-running - `__t2` in the guide below
+ - make sure provider is not-running - `__t3` in the guide below
+ - `make clean`
+ - `make kube-cluster-delete`
+
+
+### SDL
+SDL_PATH is set to `grafana.yaml`
+it can be opted by setting SDL_PATH to custom SDL prior every make command
+
+### Working dir
+```shell
+cd _run/kube
+```
+
+###  __t1 Step 1__
+```shell
+AKASH_DOCKER_IMAGE=ghcr.io/akash-network/node:0.22.9 DOCKER_IMAGE=ghcr.io/akash-network/provider:0.2.1 make prepare
+```
+
+### __t1 Step 2__
+```shell
+SKIP_BUILD=true \
+AKASH_DOCKER_IMAGE=ghcr.io/akash-network/node:0.22.9 \
+DOCKER_IMAGE=ghcr.io/akash-network/provider:0.2.1 \
+CRD_FILE=https://raw.githubusercontent.com/akash-network/provider/v0.2.1/pkg/apis/akash.network/crd.yaml \
+make kube-cluster-setup
+```
+
+### __t1 Step 3__
+Ensure versions
+
+#### akash expected to be v0.22.9
+```shell
+akash version
+```
+#### provider-services expected to be v0.2.1
+```shell
+provider-services version
+```
+
+### __t2 Step 4__ Run node
+```shell
+make node-run
+```
+
+### __t1 Step 5__ Create provider
+```shell
+make provider-create
+```
+
+### __t3 Step 6__ Run provider
+```shell
+make provider-run
+```
+
+### __t1 Step 7__ Create deployment
+```shell
+make deployment-create
+make lease-create
+make send-manifest
+```
+
+Ensure deployment has started
+
+### __t1 Step 8__ Submit network upgrade
+```shell
+make send-upgrade
+```
+wait till node in __t2 fails with `CONSENSUS FAILURE UPGRADE REQUIRED`
+
+
+### __t1 Step 9 Post Upgrade
+```shell
+make post-upgrade
+```
+Ensure versions
+
+#### akash expected to be v0.24.0
+```shell
+akash version
+```
+#### provider-services expected to be v0.3.2-rcX
+```shell
+provider-services version
+```
+
+### __t2 Step 10__ Run node
+```shell
+make node-run
+```
+
+### __t3 Step 11__ Dry run migration
+```shell
+make provider-migrate
+
+# should see smth like below
+✓        loaded CRDs
+✓        loaded active leases for provider "akash1muc4ckm26kyw975vffzp3a8ls34ew0n565ksyy"
+✓        loaded CRDs
+✓        backup manifests DONE: 1
+✓        backup providers hosts DONE: 4
+✓        migrated manifests to v2beta2
+✓        migrated hosts to v2beta2
+
+```
+
+the tool does backup of all existing manifests into `.cache/run/kube/crd/v2beta1`
+patched manifests are located in `.cache/run/kube/crd/v2beta2`
+
+### __t3 Step 12__ Run migration
+```shell
+PROVIDER_MIGRATE_DRYRUN=false make provider-migrate
+# will be asked to override backup. press Y
+```
+
+### __t1 Step 13__ Update operators
+```shell
+make kube-prepare-images kube-upload-images
+make kustomize-init kustomize-deploy-services
+```
+
+### __t3 Step 14__ Start the provider
+```shell
+make provider-run
+```
+
+### Step 15 Inspect workload
+Ensure provider pick up on upgraded manifests and workloads are up and running.
+
+
+### Step 16 Perform deployment-update
+make sure provider processes it and update deployment. make sure that manifest in kubernetes is updated as well
diff --git a/cluster/kube/client_hostname_connections.go b/cluster/kube/client_hostname_connections.go
@@ -70,7 +70,6 @@ func (c *client) DeclareHostname(ctx context.Context, lID mtypes.LeaseID, host s
 			ServiceName:  serviceName,
 			ExternalPort: externalPort,
 		},
-		Status: crd.ProviderHostStatus{},
 	}
 
 	c.log.Info("declaring hostname", "lease", lID, "service-name", serviceName, "external-port", externalPort, "host", host)

diff --git a/cluster/kube/cluster_ip_connections.go b/cluster/kube/cluster_ip_connections.go
@@ -99,7 +99,6 @@ func (c *client) DeclareIP(ctx context.Context, lID mtypes.LeaseID, serviceName
 			Protocol:     proto.ToString(),
 			Port:         port,
 		},
-		Status: akashtypes.ProviderLeasedIPStatus{},
 	}
 
 	c.log.Info("declaring leased ip", "lease", lID,

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.21.0
 
 require (
-	github.com/akash-network/akash-api v0.0.28
+	github.com/akash-network/akash-api v0.0.29
 	github.com/akash-network/node v0.24.2
 	github.com/avast/retry-go/v4 v4.5.0
 	github.com/boz/go-lifecycle v0.1.1

diff --git a/go.sum b/go.sum
@@ -200,8 +200,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
-github.com/akash-network/akash-api v0.0.28 h1:QEoQJI02NqJgxNPH4C10XSEB7VWVnEh8nUVML7wdqME=
-github.com/akash-network/akash-api v0.0.28/go.mod h1:9aIEhpvAyGQJ4/p1cPo18lG67S0QGT//JLueAXqk6Kw=
+github.com/akash-network/akash-api v0.0.29 h1:S+GiYldK1UlpfHsSKOccQWGPsW3xqztwi5745lumgv4=
+github.com/akash-network/akash-api v0.0.29/go.mod h1:9aIEhpvAyGQJ4/p1cPo18lG67S0QGT//JLueAXqk6Kw=
 github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0=
 github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
 github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=