fix: failed release upgrade (#2338)

framework/app-service/.olares/config/cluster/deploy/appservice_deploy.yaml

@@ -170,7 +170,7 @@ spec:
       priorityClassName: "system-cluster-critical"
       containers:
       - name: app-service
-        image: beclab/app-service:0.4.69
+        image: beclab/app-service:0.4.70
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 0

framework/app-service/pkg/apiserver/handler_installer_upgrade.go

@@ -146,6 +146,16 @@ func (h *upgradeHandlerHelper) setAndEncodingAppCofnig(prevCfg *appcfg.Applicati
 			}
 		}
 	}
+
+	prevPortsMap := apputils.BuildPrevPortsMap(prevCfg)
+
+	// Set expose ports for upgrade, preserving existing ports with same name
+	err := apputils.SetExposePorts(context.TODO(), h.appConfig, prevPortsMap)
+	if err != nil {
+		klog.Errorf("set expose ports failed %v", err)
+		return "", err
+	}
+
 	encoding, err := json.Marshal(h.appConfig)
 	if err != nil {
 		klog.Errorf("Failed to marshal app config err=%v", err)

framework/app-service/pkg/appstate/installing_app.go

@@ -13,6 +13,7 @@ import (
 	"github.com/beclab/Olares/framework/app-service/pkg/appinstaller/versioned"
 	"github.com/beclab/Olares/framework/app-service/pkg/constants"
 	"github.com/beclab/Olares/framework/app-service/pkg/errcode"
+	apputils "github.com/beclab/Olares/framework/app-service/pkg/utils/app"
 
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
@@ -59,12 +60,25 @@ func (p *InstallingApp) Exec(ctx context.Context) (StatefulInProgressApp, error)
 		klog.Errorf("get kube config failed %v", err)
 		return nil, err
 	}
-	err = setExposePorts(ctx, appCfg)
+	err = apputils.SetExposePorts(ctx, appCfg, nil)
 	if err != nil {
 		klog.Errorf("set expose ports failed %v", err)
 		return nil, err
 	}
 
+	updatedConfig, err := json.Marshal(appCfg)
+	if err != nil {
+		klog.Errorf("marshal appConfig failed %v", err)
+		return nil, err
+	}
+	managerCopy := p.manager.DeepCopy()
+	managerCopy.Spec.Config = string(updatedConfig)
+	err = p.client.Patch(ctx, managerCopy, client.MergeFrom(p.manager))
+	if err != nil {
+		klog.Errorf("update ApplicationManager config failed %v", err)
+		return nil, err
+	}
+
 	opCtx, cancel := context.WithCancel(context.Background())
 
 	ops, err := versioned.NewHelmOps(opCtx, kubeConfig, appCfg, token,

framework/app-service/pkg/appstate/upgrading_app.go

@@ -169,6 +169,14 @@ func (p *UpgradingApp) exec(ctx context.Context) error {
 			klog.Errorf("get app config failed %v", err)
 			return err
 		}
+		var cfg *appcfg.ApplicationConfig
+		err = json.Unmarshal([]byte(p.manager.Spec.Config), &cfg)
+		if err != nil {
+			klog.Errorf("unmarshal to appConfig failed %v", err)
+			return err
+		}
+		appConfig.Ports = cfg.Ports
+
 	} else {
 		_, err = apputils.GetIndexAndDownloadChart(ctx, &apputils.ConfigOptions{
 			App:          p.manager.Spec.AppName,

framework/app-service/pkg/appstate/utils.go

@@ -6,9 +6,6 @@ import (
 
 	appv1alpha1 "github.com/beclab/Olares/framework/app-service/api/app.bytetrade.io/v1alpha1"
 	"github.com/beclab/Olares/framework/app-service/pkg/apiserver/api"
-	"github.com/beclab/Olares/framework/app-service/pkg/appcfg"
-	"github.com/beclab/Olares/framework/app-service/pkg/utils"
-
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -17,7 +14,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -25,96 +21,6 @@ import (
 const suspendAnnotation = "bytetrade.io/suspend-by"
 const suspendCauseAnnotation = "bytetrade.io/suspend-cause"
 
-type portKey struct {
-	port     int32
-	protocol string
-}
-
-func setExposePorts(ctx context.Context, appConfig *appcfg.ApplicationConfig) error {
-	existPorts := make(map[portKey]struct{})
-	client, err := utils.GetClient()
-	if err != nil {
-		return err
-	}
-	apps, err := client.AppV1alpha1().Applications().List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
-
-	for _, app := range apps.Items {
-		for _, p := range app.Spec.Ports {
-			protos := []string{p.Protocol}
-			if p.Protocol == "" {
-				protos = []string{"tcp", "udp"}
-			}
-			for _, proto := range protos {
-				key := portKey{
-					port:     p.ExposePort,
-					protocol: proto,
-				}
-				existPorts[key] = struct{}{}
-			}
-		}
-	}
-	klog.Infof("existPorts: %v", existPorts)
-
-	for i := range appConfig.Ports {
-		port := &appConfig.Ports[i]
-		if port.ExposePort == 0 {
-			var exposePort int32
-			protos := []string{port.Protocol}
-			if port.Protocol == "" {
-				protos = []string{"tcp", "udp"}
-			}
-
-			for i := 0; i < 5; i++ {
-				exposePort, err = genPort(protos)
-				if err != nil {
-					continue
-				}
-				for _, proto := range protos {
-					key := portKey{port: exposePort, protocol: proto}
-					if _, ok := existPorts[key]; !ok && err == nil {
-						break
-					}
-				}
-			}
-			for _, proto := range protos {
-				key := portKey{port: exposePort, protocol: proto}
-				if _, ok := existPorts[key]; ok || err != nil {
-					return fmt.Errorf("%d port is not available", key.port)
-				}
-				existPorts[key] = struct{}{}
-				port.ExposePort = exposePort
-			}
-		}
-	}
-
-	// add exposePort to tailscale acls
-	for i := range appConfig.Ports {
-		if appConfig.Ports[i].AddToTailscaleAcl {
-			appConfig.TailScale.ACLs = append(appConfig.TailScale.ACLs, appv1alpha1.ACL{
-				Action: "accept",
-				Src:    []string{"*"},
-				Proto:  appConfig.Ports[i].Protocol,
-				Dst:    []string{fmt.Sprintf("*:%d", appConfig.Ports[i].ExposePort)},
-			})
-		}
-	}
-	klog.Infof("appConfig.TailScale: %v", appConfig.TailScale)
-	return nil
-}
-
-func genPort(protos []string) (int32, error) {
-	exposePort := int32(rand.IntnRange(46800, 50000))
-	for _, proto := range protos {
-		if !utils.IsPortAvailable(proto, int(exposePort)) {
-			return 0, fmt.Errorf("failed to allocate an available port after 5 attempts")
-		}
-	}
-	return exposePort, nil
-}
-
 func suspendOrResumeApp(ctx context.Context, cli client.Client, am *appv1alpha1.ApplicationManager, replicas int32) error {
 	suspend := func(list client.ObjectList) error {
 		namespace := am.Spec.AppNamespace

framework/app-service/pkg/helm/helm.go

@@ -85,7 +85,7 @@ func UpgradeCharts(ctx context.Context, actionConfig *action.Configuration, sett
 	ctrl.Log.Info("helm action config", "reachable", actionConfig.KubeClient.IsReachable())
 	client := action.NewUpgrade(actionConfig)
 	client.Namespace = namespace
-	client.Timeout = 300 * time.Second
+	client.Timeout = 2400 * time.Second
 	client.Recreate = false
 	client.Atomic = true
 	if reuseValue {

framework/app-service/pkg/utils/app/app.go

@@ -35,6 +35,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/retry"
@@ -1094,3 +1095,118 @@ func GetRawAppName(AppName, rawAppName string) string {
 	return rawAppName
 
 }
+
+type portKey struct {
+	port     int32
+	protocol string
+}
+
+func genPort(protos []string) (int32, error) {
+	exposePort := int32(rand.IntnRange(46800, 50000))
+	for _, proto := range protos {
+		if !utils.IsPortAvailable(proto, int(exposePort)) {
+			return 0, fmt.Errorf("failed to allocate an available port after 5 attempts")
+		}
+	}
+	return exposePort, nil
+}
+
+// SetExposePorts sets expose ports for app config.
+func SetExposePorts(ctx context.Context, appConfig *appcfg.ApplicationConfig, prevPortsMap map[string]int32) error {
+	existPorts := make(map[portKey]struct{})
+	client, err := utils.GetClient()
+	if err != nil {
+		return err
+	}
+	apps, err := client.AppV1alpha1().Applications().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, app := range apps.Items {
+		for _, p := range app.Spec.Ports {
+			protos := []string{p.Protocol}
+			if p.Protocol == "" {
+				protos = []string{"tcp", "udp"}
+			}
+			for _, proto := range protos {
+				key := portKey{
+					port:     p.ExposePort,
+					protocol: proto,
+				}
+				existPorts[key] = struct{}{}
+			}
+		}
+	}
+	klog.Infof("existPorts: %v", existPorts)
+
+	for i := range appConfig.Ports {
+		port := &appConfig.Ports[i]
+
+		// For upgrade: if port with same name exists in prevPortsMap, preserve its ExposePort
+		if prevPortsMap != nil && port.Name != "" {
+			if prevExposePort, exists := prevPortsMap[port.Name]; exists && prevExposePort != 0 {
+				klog.Infof("preserving ExposePort %d for port %s from previous config", prevExposePort, port.Name)
+				port.ExposePort = prevExposePort
+				continue
+			}
+		}
+
+		if port.ExposePort == 0 {
+			var exposePort int32
+			protos := []string{port.Protocol}
+			if port.Protocol == "" {
+				protos = []string{"tcp", "udp"}
+			}
+
+			for i := 0; i < 5; i++ {
+				exposePort, err = genPort(protos)
+				if err != nil {
+					continue
+				}
+				for _, proto := range protos {
+					key := portKey{port: exposePort, protocol: proto}
+					if _, ok := existPorts[key]; !ok && err == nil {
+						break
+					}
+				}
+			}
+			for _, proto := range protos {
+				key := portKey{port: exposePort, protocol: proto}
+				if _, ok := existPorts[key]; ok || err != nil {
+					return fmt.Errorf("%d port is not available", key.port)
+				}
+				existPorts[key] = struct{}{}
+				port.ExposePort = exposePort
+			}
+		}
+	}
+
+	// add exposePort to tailscale acls
+	for i := range appConfig.Ports {
+		if appConfig.Ports[i].AddToTailscaleAcl {
+			appConfig.TailScale.ACLs = append(appConfig.TailScale.ACLs, v1alpha1.ACL{
+				Action: "accept",
+				Src:    []string{"*"},
+				Proto:  appConfig.Ports[i].Protocol,
+				Dst:    []string{fmt.Sprintf("*:%d", appConfig.Ports[i].ExposePort)},
+			})
+		}
+	}
+	klog.Infof("appConfig.TailScale: %v", appConfig.TailScale)
+	return nil
+}
+
+// BuildPrevPortsMap builds a map of port name -> expose port from previous config.
+func BuildPrevPortsMap(prevConfig *appcfg.ApplicationConfig) map[string]int32 {
+	if prevConfig == nil {
+		return nil
+	}
+	m := make(map[string]int32)
+	for _, p := range prevConfig.Ports {
+		if p.Name != "" && p.ExposePort != 0 {
+			m[p.Name] = p.ExposePort
+		}
+	}
+	return m
+}