fix(gpu): handle scheduler inconsistency and device stuck in unhealthy

* fix: failed release upgrade

* fix: helm upgrade do not use atomic param and allow upgrade failed release

* fix: app upgrade set tailscale acl (#2357)

* fix: increase wait timeout for namespace delete

* fix: update app-service image tag to 0.4.73

apps/.olares/config/user/helm-charts/system-apps/templates/olares-app.yaml

@@ -317,7 +317,7 @@ spec:
             chown -R 1000:1000 /uploadstemp && \
             chown -R 1000:1000 /appdata 
         - name: olares-app-init
-          image: beclab/system-frontend:v1.6.37
+          image: beclab/system-frontend:v1.6.38
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh

daemon/pkg/utils/drivers_linux.go

@@ -199,7 +199,7 @@ func MountedHddPath(ctx context.Context) ([]string, error) {
 
 func FilterBySerial(serial string) func(dev storageDevice) bool {
 	return func(dev storageDevice) bool {
-		return dev.IDSerial == serial || dev.IDSerialShort == serial
+		return strings.HasSuffix(serial, dev.IDSerial) || strings.HasSuffix(serial, dev.IDSerialShort)
 	}
 }
 

framework/app-service/.olares/config/cluster/deploy/appservice_deploy.yaml

@@ -170,7 +170,7 @@ spec:
       priorityClassName: "system-cluster-critical"
       containers:
       - name: app-service
-        image: beclab/app-service:0.4.72
+        image: beclab/app-service:0.4.73
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 0

framework/app-service/controllers/load.go

@@ -146,7 +146,7 @@ func LoadStatefulApp(ctx context.Context, appmgr *ApplicationManagerController,
 		case appv1alpha1.ApplyingEnvCanceling:
 			return appstate.NewApplyingEnvCancelingApp(appmgr, &am)
 		case appv1alpha1.Uninstalling:
-			return appstate.NewUninstallingApp(appmgr, &am, 15*time.Minute)
+			return appstate.NewUninstallingApp(appmgr, &am, 30*time.Minute)
 		case appv1alpha1.StopFailed:
 			return appstate.NewSuspendFailedApp(appmgr, &am)
 		case appv1alpha1.UninstallFailed:

framework/app-service/pkg/appstate/types.go

@@ -20,6 +20,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -138,6 +139,13 @@ func (p *baseStatefulApp) forceDeleteApp(ctx context.Context) error {
 			return err
 		}
 	}
+
+	// Wait for namespace to be fully deleted before updating status
+	if err = p.waitForNamespaceDeleted(ctx); err != nil {
+		klog.Errorf("wait for namespace %s deleted failed %v", p.manager.Spec.AppNamespace, err)
+		return err
+	}
+
 	err = p.updateStatus(ctx, p.manager, appsv1.Uninstalled, nil, appsv1.Uninstalled.String(), "")
 	if err != nil {
 		klog.Errorf("update app manager %s to state %s failed", p.manager.Name, appsv1.Uninstalled)
@@ -146,6 +154,32 @@ func (p *baseStatefulApp) forceDeleteApp(ctx context.Context) error {
 	return nil
 }
 
+// waitForNamespaceDeleted waits for the namespace to be completely deleted
+func (p *baseStatefulApp) waitForNamespaceDeleted(ctx context.Context) error {
+	namespace := p.manager.Spec.AppNamespace
+	if apputils.IsProtectedNamespace(namespace) {
+		return nil
+	}
+
+	klog.Infof("waiting for namespace %s to be fully deleted", namespace)
+	err := utilwait.PollImmediate(time.Second, 30*time.Minute, func() (done bool, err error) {
+		var ns corev1.Namespace
+		err = p.client.Get(ctx, types.NamespacedName{Name: namespace}, &ns)
+		if err != nil && !apierrors.IsNotFound(err) {
+			klog.Errorf("failed to get namespace %s: %v", namespace, err)
+			return false, err
+		}
+		if apierrors.IsNotFound(err) {
+			klog.Infof("namespace %s has been fully deleted", namespace)
+			return true, nil
+		}
+		klog.Infof("namespace %s still exists, waiting...", namespace)
+		return false, nil
+	})
+
+	return err
+}
+
 type OperationApp interface {
 	StatefulApp
 	IsTimeout() bool

framework/app-service/pkg/appstate/uninstalling_app.go

@@ -100,7 +100,7 @@ func (p *UninstallingApp) waitForDeleteNamespace(ctx context.Context) error {
 	if apputils.IsProtectedNamespace(p.manager.Spec.AppNamespace) {
 		return nil
 	}
-	err := utilwait.PollImmediate(time.Second, 15*time.Minute, func() (done bool, err error) {
+	err := utilwait.PollImmediate(time.Second, 30*time.Minute, func() (done bool, err error) {
 		klog.Infof("waiting for namespace %s to be deleted", p.manager.Spec.AppNamespace)
 		nsName := p.manager.Spec.AppNamespace
 		var ns corev1.Namespace

framework/app-service/pkg/appstate/upgrading_app.go

@@ -176,6 +176,7 @@ func (p *UpgradingApp) exec(ctx context.Context) error {
 			return err
 		}
 		appConfig.Ports = cfg.Ports
+		appConfig.TailScale = cfg.TailScale
 
 	} else {
 		_, err = apputils.GetIndexAndDownloadChart(ctx, &apputils.ConfigOptions{

framework/authelia/.olares/config/user/helm-charts/auth/templates/auth_deploy.yaml

@@ -29,7 +29,7 @@ spec:
         name: check-auth
       containers:
       - name: auth-front
-        image: beclab/login:v1.6.30
+        image: beclab/login:v1.6.38
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80

infrastructure/gpu/.olares/config/gpu/hami/values.yaml

@@ -4,7 +4,7 @@ nameOverride: ""
 fullnameOverride: ""
 namespaceOverride: ""
 imagePullSecrets: []
-version: "v2.6.6"
+version: "v2.6.7"
 
 # Nvidia GPU Parameters
 resourceName: "nvidia.com/gpu"

platform/hami/.olares/Olares.yaml

@@ -3,7 +3,7 @@ target: prebuilt
 output:
   containers:
     - 
-      name: beclab/hami:v2.6.6
+      name: beclab/hami:v2.6.7
     - 
       name: beclab/hami-webui-fe-oss:v1.0.8
     -