fix: handle case for system applications without configuration in provider list (#2507)

* main:
  Modify release-daemon.yaml for arm64 support
  fix: seafile trim commit_id for syncing and change psql ccnet init (#2495)
  backup: sync systemEnv default remote url (#2492)
  download-server: nats message publish modify (#2489)
  fix(cli): clear master host config when uninstalling (#2488)
  market, settings: support optional data deletion and fix bugs. (#2486)
  fix(cli): do not override upgrade target version by config file (#2483)
  app-service: feat app uninstall delete data (#2480)

.github/workflows/release-daemon.yaml

@@ -46,7 +46,15 @@ jobs:
 
       - name: install udev-devel and pcap-devel
         run: |
-          sudo apt update && sudo apt install -y libudev-dev libpcap-dev
+          sudo dpkg --add-architecture arm64
+          # Only modify ubuntu official sources, not third-party sources
+          sudo sed -i 's/^deb http:\/\/azure.archive.ubuntu.com/deb [arch=amd64] http:\/\/azure.archive.ubuntu.com/' /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true
+          sudo sed -i 's/^Types: deb$/Types: deb\nArchitectures: amd64/' /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true
+          # Add arm64 sources
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports noble main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports noble-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
+          sudo apt update
+          sudo apt install -y libudev-dev libpcap-dev libudev-dev:arm64 libpcap-dev:arm64
 
       - name: Install x86_64 cross-compiler
         run: sudo apt-get update && sudo apt-get install -y build-essential

apps/.olares/config/user/helm-charts/system-apps/templates/olares-app.yaml

@@ -317,7 +317,7 @@ spec:
             chown -R 1000:1000 /uploadstemp && \
             chown -R 1000:1000 /appdata 
         - name: olares-app-init
-          image: beclab/system-frontend:v1.8.5
+          image: beclab/system-frontend:v1.8.7
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh

build/user-env.yaml

@@ -43,6 +43,15 @@ userEnvs:
   - envName: OLARES_USER_SMTP_SECURITY_PROTOCOLS
     type: string
     editable: true        
+    options:
+    - title: "TLS"
+      value: "tls"
+    - title: "SSL"
+      value: "ssl"
+    - title: "StartTLS"
+      value: "starttls"
+    - title: "None"
+      value: "none"
   - envName: OLARES_USER_OPENAI_APIKEY
     type: password
     editable: true

cli/cmd/ctl/root.go

@@ -13,6 +13,7 @@ import (
 	"github.com/beclab/Olares/cli/cmd/ctl/user"
 	"github.com/beclab/Olares/cli/version"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 func NewDefaultCommand() *cobra.Command {
@@ -25,6 +26,11 @@ func NewDefaultCommand() *cobra.Command {
 		Short:             "Olares Installer",
 		CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true},
 		Version:           version.VERSION,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlags(cmd.InheritedFlags())
+			viper.BindPFlags(cmd.PersistentFlags())
+			viper.BindPFlags(cmd.Flags())
+		},
 		Run: func(cmd *cobra.Command, args []string) {
 			if showVendor {
 				fmt.Println(version.VENDOR)

cli/pkg/common/kube_runtime.go

@@ -211,6 +211,9 @@ func NewArgument() *Argument {
 	arg.IsOlaresInContainer = os.Getenv(ENV_CONTAINER_MODE) == "oic"
 	si.IsOIC = arg.IsOlaresInContainer
 
+	// Ensure BaseDir is initialized before loading master.conf
+	// so master host config can be loaded from ${base-dir}/master.conf reliably.
+	arg.SetBaseDir(viper.GetString(FlagBaseDir))
 	arg.loadMasterHostConfig()
 	return arg
 }
@@ -382,6 +385,10 @@ func (a *Argument) loadMasterHostConfig() {
 	}
 }
 
+func (a *Argument) ClearMasterHostConfig() {
+	a.MasterHostConfig = &MasterHostConfig{}
+}
+
 func (a *Argument) SetManifest(manifest string) {
 	a.Manifest = manifest
 }

cli/pkg/pipelines/add_node.go

@@ -18,7 +18,6 @@ func AddNodePipeline() error {
 	}
 
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("addnode.log", true)
 
 	if err := arg.MasterHostConfig.Validate(); err != nil {

cli/pkg/pipelines/changeip.go

@@ -19,7 +19,6 @@ func ChangeIPPipeline() error {
 
 	var arg = common.NewArgument()
 	arg.SetOlaresVersion(terminusVersion)
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("changeip.log", true)
 	arg.SetKubeVersion(kubeType)
 	arg.SetMinikubeProfile(viper.GetString(common.FlagMiniKubeProfile))

cli/pkg/pipelines/check_download.go

@@ -12,7 +12,6 @@ import (
 func CheckDownloadInstallationPackage() error {
 	arg := common.NewArgument()
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 
 	runtime, err := common.NewKubeRuntime(*arg)
 	if err != nil {

cli/pkg/pipelines/download_package.go

@@ -13,7 +13,6 @@ import (
 
 func DownloadInstallationPackage() error {
 	arg := common.NewArgument()
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
 	arg.SetOlaresCDNService(viper.GetString(common.FlagCDNService))
 

cli/pkg/pipelines/download_wizard.go

@@ -13,7 +13,6 @@ import (
 func DownloadInstallationWizard() error {
 	arg := common.NewArgument()
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetOlaresCDNService(viper.GetString(common.FlagCDNService))
 
 	runtime, err := common.NewKubeRuntime(*arg)

cli/pkg/pipelines/gpu_install.go

@@ -15,7 +15,6 @@ import (
 func InstallGpuDrivers() error {
 	arg := common.NewArgument()
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("gpuinstall.log", true)
 	runtime, err := common.NewKubeRuntime(*arg)
 	if err != nil {

cli/pkg/pipelines/install_terminus.go

@@ -20,7 +20,6 @@ func CliInstallTerminusPipeline() error {
 	}
 
 	arg := common.NewArgument()
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetKubeVersion(viper.GetString(common.FlagKubeType))
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
 	arg.SetMinikubeProfile(viper.GetString(common.FlagMiniKubeProfile))

cli/pkg/pipelines/masterinfo.go

@@ -8,7 +8,6 @@ import (
 	"github.com/beclab/Olares/cli/pkg/core/module"
 	"github.com/beclab/Olares/cli/pkg/core/pipeline"
 	"github.com/beclab/Olares/cli/pkg/terminus"
-	"github.com/spf13/viper"
 )
 
 func MasterInfoPipeline() error {
@@ -17,7 +16,6 @@ func MasterInfoPipeline() error {
 		fmt.Println("error: Only Linux nodes can be added to an Olares cluster!")
 		os.Exit(1)
 	}
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("masterinfo.log", true)
 
 	if err := arg.MasterHostConfig.Validate(); err != nil {

cli/pkg/pipelines/precheck.go

@@ -11,7 +11,6 @@ import (
 func StartPreCheckPipeline() error {
 	var arg = common.NewArgument()
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("precheck.log", true)
 
 	runtime, err := common.NewKubeRuntime(*arg)

cli/pkg/pipelines/prepare_system.go

@@ -28,7 +28,6 @@ func PrepareSystemPipeline(components []string) error {
 	}
 
 	var arg = common.NewArgument()
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetKubeVersion(viper.GetString(common.FlagKubeType))
 	arg.SetMinikubeProfile(viper.GetString(common.FlagMiniKubeProfile))
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))

cli/pkg/pipelines/storage.go

@@ -18,7 +18,6 @@ func CliInstallStoragePipeline() error {
 	}
 
 	arg := common.NewArgument()
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
 	arg.SetStorage(getStorageConfig())
 

cli/pkg/pipelines/uninstall_terminus.go

@@ -20,10 +20,10 @@ func UninstallTerminusPipeline() error {
 
 	var arg = common.NewArgument()
 	arg.SetOlaresVersion(version)
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetConsoleLog("uninstall.log", true)
 	arg.SetKubeVersion(kubeType)
 	arg.SetStorage(getStorageConfig())
+	arg.ClearMasterHostConfig()
 
 	phase := viper.GetString(common.FlagUninstallPhase)
 	all := viper.GetBool(common.FlagUninstallAll)

cli/pkg/pipelines/upgrade.go

@@ -30,13 +30,7 @@ func UpgradeOlaresPipeline() error {
 		return fmt.Errorf("error parsing current Olares version: %v", err)
 	}
 
-	// should only be and defaults to the current cli version
-	// this argument is for backwards-compatibility with older olaresd
-	targetVersionStr := viper.GetString(common.FlagVersion)
-	if targetVersionStr == "" {
-		targetVersionStr = version.VERSION
-	}
-	targetVersion, err := utils.ParseOlaresVersionString(targetVersionStr)
+	targetVersion, err := utils.ParseOlaresVersionString(version.VERSION)
 	if err != nil {
 		return fmt.Errorf("error parsing target Olares version: %v", err)
 	}
@@ -46,7 +40,6 @@ func UpgradeOlaresPipeline() error {
 	}
 
 	arg := common.NewArgument()
-	arg.SetBaseDir(viper.GetString(common.FlagBaseDir))
 	arg.SetOlaresVersion(viper.GetString(common.FlagVersion))
 	arg.SetConsoleLog("upgrade.log", true)
 	arg.SetKubeVersion(phase.GetKubeType())

cli/pkg/release/app/app.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/beclab/Olares/cli/pkg/common"
 	"github.com/beclab/Olares/cli/pkg/core/util"
 )
 
@@ -57,6 +58,10 @@ func (m *Manager) Package() error {
 		return err
 	}
 
+	if err := m.packageEnvConfig(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -121,3 +126,19 @@ func (m *Manager) packageGPU() error {
 		filepath.Join(m.distPath, "wizard/config/gpu"),
 	)
 }
+
+func (m *Manager) packageEnvConfig() error {
+	fmt.Println("packaging env config ...")
+
+	systemEnvSrc := filepath.Join(m.olaresRepoRoot, "build", common.OLARES_SYSTEM_ENV_FILENAME)
+	userEnvSrc := filepath.Join(m.olaresRepoRoot, "build", common.OLARES_USER_ENV_FILENAME)
+
+	if err := util.CopyFile(systemEnvSrc, filepath.Join(m.distPath, common.OLARES_SYSTEM_ENV_FILENAME)); err != nil {
+		return err
+	}
+	if err := util.CopyFile(userEnvSrc, filepath.Join(m.distPath, common.OLARES_USER_ENV_FILENAME)); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/pkg/terminus/ossystem.go

@@ -125,6 +125,11 @@ func (t *CreateUserEnvConfigMap) Execute(runtime connector.Runtime) error {
 		return nil
 	}
 
+	desiredBytes, err := os.ReadFile(userEnvPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to read user env config file")
+	}
+
 	configK8s, err := ctrl.GetConfig()
 	if err != nil {
 		return errors.Wrap(err, "failed to get kubernetes config")
@@ -144,17 +149,24 @@ func (t *CreateUserEnvConfigMap) Execute(runtime connector.Runtime) error {
 	defer cancel()
 
 	name := "user-env"
+	namespace := common.NamespaceOsFramework
 	cm := &corev1.ConfigMap{}
-	err = ctrlclient.Get(ctx, types.NamespacedName{Name: name, Namespace: common.NamespaceOsFramework}, cm)
+	err = ctrlclient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cm)
 	if apierrors.IsNotFound(err) {
-		// create via kubectl from file
-		kubectl, _ := util.GetCommand(common.CommandKubectl)
-		cmd := fmt.Sprintf("%s -n %s create configmap %s --from-file=%s=%s",
-			kubectl, common.NamespaceOsFramework, name, common.OLARES_USER_ENV_FILENAME, userEnvPath,
-		)
-		if _, cerr := runtime.GetRunner().SudoCmd(cmd, false, true); cerr != nil {
-			return errors.Wrap(errors.WithStack(cerr), "failed to create user-env configmap")
+		cm = &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+			Data: map[string]string{
+				common.OLARES_USER_ENV_FILENAME: string(desiredBytes),
+			},
 		}
+
+		if err := ctrlclient.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) {
+			return errors.Wrap(err, "failed to create user-env configmap")
+		}
+
 		logger.Infof("Created user env configmap from %s", userEnvPath)
 		return nil
 	}
@@ -162,57 +174,21 @@ func (t *CreateUserEnvConfigMap) Execute(runtime connector.Runtime) error {
 		return errors.Wrap(err, "failed to get user-env configmap")
 	}
 
-	// If exists, merge missing envs and update via client
-	newDataBytes, err := os.ReadFile(userEnvPath)
-	if err != nil {
-		return errors.Wrap(err, "failed to read user env config file")
-	}
-
-	var newCfg UserEnvConfig
-	if err := yaml.Unmarshal(newDataBytes, &newCfg); err != nil {
-		return errors.Wrap(err, "failed to parse user env config file")
-	}
-
-	var existingCfg UserEnvConfig
-	existingContent := cm.Data[common.OLARES_USER_ENV_FILENAME]
-	if existingContent != "" {
-		if err := yaml.Unmarshal([]byte(existingContent), &existingCfg); err != nil {
-			return errors.Wrap(err, "failed to parse existing user env configmap data")
-		}
-	}
-
-	existingSet := make(map[string]struct{}, len(existingCfg.UserEnvs))
-	for _, e := range existingCfg.UserEnvs {
-		existingSet[e.EnvName] = struct{}{}
-	}
-
-	missing := 0
-	for _, e := range newCfg.UserEnvs {
-		if _, ok := existingSet[e.EnvName]; !ok {
-			existingCfg.UserEnvs = append(existingCfg.UserEnvs, e)
-			missing++
-		}
-	}
-
-	if missing == 0 {
-		logger.Infof("No new user envs to add; configmap up to date")
-		return nil
-	}
-
-	updatedBytes, err := yaml.Marshal(existingCfg)
-	if err != nil {
-		return errors.Wrap(err, "failed to marshal updated user env config")
-	}
 	if cm.Data == nil {
 		cm.Data = map[string]string{}
 	}
-	cm.Data[common.OLARES_USER_ENV_FILENAME] = string(updatedBytes)
+	if cm.Data[common.OLARES_USER_ENV_FILENAME] == string(desiredBytes) {
+		logger.Infof("user-env configmap is up to date")
+		return nil
+	}
+
+	cm.Data[common.OLARES_USER_ENV_FILENAME] = string(desiredBytes)
 
 	if err := ctrlclient.Update(ctx, cm); err != nil {
 		return errors.Wrap(err, "failed to update user-env configmap")
 	}
 
-	logger.Infof("Appended %d missing user env(s) and updated configmap", missing)
+	logger.Infof("Updated user env configmap from %s", userEnvPath)
 	return nil
 }
 

cli/pkg/upgrade/1_12_5_20260122.go

@@ -7,15 +7,15 @@ import (
 	"github.com/beclab/Olares/cli/pkg/core/task"
 )
 
-type upgrader_1_12_6_20260122 struct {
+type upgrader_1_12_5_20260122 struct {
 	breakingUpgraderBase
 }
 
-func (u upgrader_1_12_6_20260122) Version() *semver.Version {
-	return semver.MustParse("1.12.6-20260122")
+func (u upgrader_1_12_5_20260122) Version() *semver.Version {
+	return semver.MustParse("1.12.5-20260122")
 }
 
-func (u upgrader_1_12_6_20260122) UpgradeSystemComponents() []task.Interface {
+func (u upgrader_1_12_5_20260122) UpgradeSystemComponents() []task.Interface {
 	pre := []task.Interface{
 		&task.LocalTask{
 			Name:   "UpgradeL4BFLProxy",
@@ -28,5 +28,5 @@ func (u upgrader_1_12_6_20260122) UpgradeSystemComponents() []task.Interface {
 }
 
 func init() {
-	registerDailyUpgrader(upgrader_1_12_6_20260122{})
+	registerDailyUpgrader(upgrader_1_12_5_20260122{})
 }

framework/app-service/.olares/config/cluster/deploy/appservice_deploy.yaml

@@ -170,7 +170,7 @@ spec:
       priorityClassName: "system-cluster-critical"
       containers:
       - name: app-service
-        image: beclab/app-service:0.4.78
+        image: beclab/app-service:0.5.1
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 6755

framework/app-service/controllers/userenv_sync_controller.go

@@ -40,7 +40,7 @@ type UserEnvSyncController struct {
 
 //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
 //+kubebuilder:rbac:groups=iam.kubesphere.io,resources=users,verbs=get;list;watch
-//+kubebuilder:rbac:groups=sys.bytetrade.io,resources=userenvs,verbs=get;list;watch;create
+//+kubebuilder:rbac:groups=sys.bytetrade.io,resources=userenvs,verbs=get;list;watch;create;patch;update
 
 func (r *UserEnvSyncController) SetupWithManager(mgr ctrl.Manager) error {
 	cmPred := predicate.NewPredicateFuncs(func(obj client.Object) bool {
@@ -164,14 +164,63 @@ func (r *UserEnvSyncController) syncUserEnvForUser(ctx context.Context, username
 		return 0, fmt.Errorf("list userenvs in %s failed: %w", userNs, err)
 	}
 
-	existSet := make(map[string]struct{}, len(existing.Items))
+	existByName := make(map[string]*sysv1alpha1.UserEnv, len(existing.Items))
 	for i := range existing.Items {
-		existSet[existing.Items[i].EnvName] = struct{}{}
+		existByName[existing.Items[i].EnvName] = &existing.Items[i]
 	}
 
 	created := 0
 	for _, spec := range base {
-		if _, ok := existSet[spec.EnvName]; ok {
+		if ue, ok := existByName[spec.EnvName]; ok {
+			original := ue.DeepCopy()
+			updated := false
+
+			if ue.Default == "" && spec.Default != "" {
+				ue.Default = spec.Default
+				updated = true
+			}
+			if ue.Type == "" && spec.Type != "" {
+				ue.Type = spec.Type
+				updated = true
+			}
+			if ue.Title == "" && spec.Title != "" {
+				ue.Title = spec.Title
+				updated = true
+			}
+			if ue.Description == "" && spec.Description != "" {
+				ue.Description = spec.Description
+				updated = true
+			}
+			if ue.RemoteOptions == "" && spec.RemoteOptions != "" {
+				ue.RemoteOptions = spec.RemoteOptions
+				updated = true
+			}
+			if ue.Regex == "" && spec.Regex != "" {
+				ue.Regex = spec.Regex
+				updated = true
+			}
+
+			if len(spec.Options) > 0 {
+				existOpt := make(map[string]struct{}, len(ue.Options))
+				for _, it := range ue.Options {
+					existOpt[it.Value] = struct{}{}
+				}
+				for _, it := range spec.Options {
+					if _, exists := existOpt[it.Value]; exists {
+						continue
+					}
+					ue.Options = append(ue.Options, it)
+					existOpt[it.Value] = struct{}{}
+					updated = true
+				}
+			}
+
+			if updated {
+				if err := r.Patch(ctx, ue, client.MergeFrom(original)); err != nil {
+					return created, fmt.Errorf("patch userenv %s/%s failed: %w", ue.Namespace, ue.Name, err)
+				}
+				klog.Infof("UserEnvSync: patched userenv %s/%s for user %s", ue.Namespace, ue.Name, username)
+			}
 			continue
 		}
 		name, err := apputils.EnvNameToResourceName(spec.EnvName)

framework/app-service/pkg/apiserver/api/types.go

@@ -13,6 +13,7 @@ const (
 	AppMarketSourceKey  = constants.AppMarketSourceKey
 	AppInstallSourceKey = "bytetrade.io/install-source"
 	AppUninstallAllKey  = "bytetrade.io/uninstall-all"
+	AppDeleteDataKey    = "bytetrade.io/delete-data"
 	AppStopAllKey       = "bytetrade.io/stop-all"
 	AppResumeAllKey     = "bytetrade.io/resume-all"
 	AppImagesKey        = "bytetrade.io/images"
@@ -145,7 +146,8 @@ type Image struct {
 
 // UninstallRequest represents a request to uninstall an application.
 type UninstallRequest struct {
-	All bool `json:"all"`
+	All        bool `json:"all"`
+	DeleteData bool `json:"deleteData"`
 }
 
 // StopRequest represents a request to stop an application.

framework/app-service/pkg/apiserver/handler_appenv.go

@@ -6,10 +6,12 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"sync"
 	"time"
 
 	sysv1alpha1 "github.com/beclab/Olares/framework/app-service/api/sys.bytetrade.io/v1alpha1"
 	"github.com/beclab/Olares/framework/app-service/pkg/apiserver/api"
+	"github.com/beclab/Olares/framework/app-service/pkg/constants"
 	"github.com/beclab/Olares/framework/app-service/pkg/utils"
 	apputils "github.com/beclab/Olares/framework/app-service/pkg/utils/app"
 	"github.com/emicklei/go-restful/v3"
@@ -76,32 +78,74 @@ func (h *Handler) updateAppEnv(req *restful.Request, resp *restful.Response) {
 		return
 	}
 
+	var refEnvOnce sync.Once
+	var listErr error
+	refEnvs := make(map[string]string)
+
 	updated := false
 	original := targetAppEnv.DeepCopy()
 	for i, existingEnv := range targetAppEnv.Envs {
 		for _, env := range updatedEnvs {
-			if existingEnv.EnvName == env.EnvName {
-				if !existingEnv.Editable {
-					api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' is not editable", env.EnvName))
-					return
-				}
-				if existingEnv.Required && existingEnv.Default == "" && env.Value == "" {
-					api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' is required", env.EnvName))
-					return
-				}
-				if existingEnv.Value != env.Value {
-					if err := existingEnv.ValidateValue(env.Value); err != nil {
-						api.HandleBadRequest(resp, req, fmt.Errorf("failed to update app env '%s': %v", env.EnvName, err))
+			if existingEnv.EnvName != env.EnvName {
+				continue
+			}
+			if !existingEnv.Editable {
+				api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' is not editable", env.EnvName))
+				return
+			}
+			if existingEnv.Required && existingEnv.Default == "" && env.Value == "" && (env.ValueFrom == nil || env.ValueFrom.EnvName == "") {
+				api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' is required", env.EnvName))
+				return
+			}
+			if env.ValueFrom != nil && env.ValueFrom.EnvName != "" && (existingEnv.ValueFrom == nil || existingEnv.ValueFrom.EnvName != env.ValueFrom.EnvName) {
+				refEnvOnce.Do(func() {
+					sysenvs := new(sysv1alpha1.SystemEnvList)
+					listErr = h.ctrlClient.List(req.Request.Context(), sysenvs)
+					if listErr != nil {
 						return
 					}
-					targetAppEnv.Envs[i].Value = env.Value
-					updated = true
-					if existingEnv.ApplyOnChange {
-						targetAppEnv.NeedApply = true
+					userenvs := new(sysv1alpha1.UserEnvList)
+					listErr = h.ctrlClient.List(req.Request.Context(), userenvs, client.InNamespace(utils.UserspaceName(owner)))
+					for _, sysenv := range sysenvs.Items {
+						refEnvs[sysenv.EnvName] = sysenv.GetEffectiveValue()
 					}
+					for _, userenv := range userenvs.Items {
+						refEnvs[userenv.EnvName] = userenv.GetEffectiveValue()
+					}
+				})
+				if listErr != nil {
+					api.HandleInternalError(resp, req, fmt.Errorf("failed to list referenced envs: %s", listErr))
+					return
 				}
-				break
+				value, ok := refEnvs[env.ValueFrom.EnvName]
+				if !ok {
+					api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' references unknown env '%s'", env.EnvName, env.ValueFrom.EnvName))
+					return
+				}
+				if existingEnv.Required && value == "" {
+					api.HandleBadRequest(resp, req, fmt.Errorf("required app env '%s' references empty env '%s'", env.EnvName, env.ValueFrom.EnvName))
+					return
+				}
+				if existingEnv.ValidateValue(value) != nil {
+					api.HandleBadRequest(resp, req, fmt.Errorf("app env '%s' references invalid value '%s' from '%s': %v", env.EnvName, value, env.ValueFrom.EnvName, err))
+					return
+				}
+				targetAppEnv.Envs[i].ValueFrom = env.ValueFrom
+				targetAppEnv.Envs[i].Value = value
+				targetAppEnv.Envs[i].ValueFrom.Status = constants.EnvRefStatusSynced
+				updated = true
+			} else if existingEnv.Value != env.Value {
+				if err := existingEnv.ValidateValue(env.Value); err != nil {
+					api.HandleBadRequest(resp, req, fmt.Errorf("failed to update app env '%s': %v", env.EnvName, err))
+					return
+				}
+				targetAppEnv.Envs[i].Value = env.Value
+				updated = true
 			}
+			if updated && existingEnv.ApplyOnChange {
+				targetAppEnv.NeedApply = true
+			}
+			break
 		}
 	}
 

framework/app-service/pkg/apiserver/handler_installer_uninstall.go

@@ -77,6 +77,7 @@ func (h *Handler) uninstall(req *restful.Request, resp *restful.Response) {
 	}
 	am.Annotations[api.AppTokenKey] = token
 	am.Annotations[api.AppUninstallAllKey] = fmt.Sprintf("%t", request.All)
+	am.Annotations[api.AppDeleteDataKey] = fmt.Sprintf("%t", request.DeleteData)
 	err = h.ctrlClient.Update(req.Request.Context(), &am)
 	if err != nil {
 		api.HandleError(resp, req, err)

framework/app-service/pkg/apiserver/handler_settings.go

@@ -586,6 +586,18 @@ func (h *Handler) getApplicationPermission(req *restful.Request, resp *restful.R
 		return
 	}
 
+	// sys app does not have app config
+	if am.Spec.Config == "" {
+		ret := &applicationPermission{
+			App:         am.Spec.AppName,
+			Owner:       owner,
+			Permissions: []permission{},
+		}
+
+		resp.WriteAsJson(ret)
+		return
+	}
+
 	var appConfig appcfg.ApplicationConfig
 	err = am.GetAppConfig(&appConfig)
 	if err != nil {
@@ -717,6 +729,12 @@ func (h *Handler) getApplicationProviderList(req *restful.Request, resp *restful
 		return
 	}
 
+	// sys app does not have app config
+	if am.Spec.Config == "" {
+		resp.WriteAsJson([]providerRegistry{})
+		return
+	}
+
 	var appConfig appcfg.ApplicationConfig
 	err = am.GetAppConfig(&appConfig)
 	if err != nil {

framework/app-service/pkg/appinstaller/helm_ops_uninstall.go

@@ -29,8 +29,14 @@ func (h *HelmOps) UninstallAll() error {
 	if err != nil {
 		return err
 	}
+	appName := fmt.Sprintf("%s-%s", h.app.Namespace, h.app.AppName)
+	appmgr, err := h.client.AppClient.AppV1alpha1().ApplicationManagers().Get(h.ctx, appName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	deleteData := appmgr.Annotations["bytetrade.io/delete-data"] == "true"
 
-	appCacheDirs, err := apputils.TryToGetAppdataDirFromDeployment(h.ctx, h.app.Namespace, h.app.AppName, h.app.OwnerName)
+	appCacheDirs, appDataDirs, err := apputils.TryToGetAppdataDirFromDeployment(h.ctx, h.app.Namespace, h.app.AppName, h.app.OwnerName, deleteData)
 	if err != nil {
 		klog.Warningf("get app %s cache dir failed %v", h.app.AppName, err)
 	}
@@ -48,6 +54,13 @@ func (h *HelmOps) UninstallAll() error {
 		klog.Errorf("Failed to clear app cache dirs %v err=%v", appCacheDirs, err)
 		return err
 	}
+	if deleteData {
+		h.ClearData(client, appDataDirs)
+		if err != nil {
+			klog.Errorf("Failed to clear app data dirs %v err=%v", appDataDirs, err)
+			return err
+		}
+	}
 
 	err = h.DeleteNamespace(client, h.app.Namespace)
 	if err != nil {
@@ -117,7 +130,7 @@ func (h *HelmOps) ClearCache(client kubernetes.Interface, appCacheDirs []string)
 			formattedAppCacheDirs := apputils.FormatCacheDirs(appCacheDirs)
 
 			for _, n := range nodes.Items {
-				URL := fmt.Sprintf(constants.AppDataDirURL, n.Name)
+				URL := fmt.Sprintf(constants.AppCacheDirURL, n.Name)
 				c.SetHeader("X-Terminus-Node", n.Name)
 				c.SetHeader("X-Bfl-User", h.app.OwnerName)
 				res, e := c.R().SetBody(map[string]interface{}{
@@ -137,6 +150,32 @@ func (h *HelmOps) ClearCache(client kubernetes.Interface, appCacheDirs []string)
 	return nil
 }
 
+func (h *HelmOps) ClearData(client kubernetes.Interface, appDataDirs []string) error {
+	if len(appDataDirs) > 0 {
+		klog.Infof("clear app data dirs: %v", appDataDirs)
+
+		c := resty.New().SetTimeout(2 * time.Second).
+			SetAuthToken(h.token)
+
+		formattedAppDataDirs := apputils.FormatCacheDirs(appDataDirs)
+
+		URL := constants.AppDataDirURL
+		c.SetHeader("X-Bfl-User", h.app.OwnerName)
+		res, e := c.R().SetBody(map[string]interface{}{
+			"dirents": formattedAppDataDirs,
+		}).Delete(URL)
+		if e != nil {
+			klog.Errorf("Failed to delete data dir err=%v", e)
+			return nil
+		}
+		if res.StatusCode() != http.StatusOK {
+			klog.Infof("delete app data failed with: %v", res.String())
+		}
+	}
+
+	return nil
+}
+
 func (h *HelmOps) ClearMiddlewareRequests(middlewareNamespace string) {
 	// delete middleware requests crd
 	for _, mt := range middlewareTypes {

framework/app-service/pkg/constants/constants.go

@@ -93,7 +93,8 @@ const (
 
 	DependencyTypeSystem = "system"
 	DependencyTypeApp    = "application"
-	AppDataDirURL        = "http://files-service.os-framework/api/resources/cache/%s/"
+	AppCacheDirURL       = "http://files-service.os-framework/api/resources/cache/%s/"
+	AppDataDirURL        = "http://files-service.os-framework/api/resources/drive/Data/"
 
 	UserSpaceDirKey   = "userspace_hostpath"
 	UserAppDataDirKey = "appcache_hostpath"

framework/app-service/pkg/utils/app/app.go

@@ -22,7 +22,6 @@ import (
 	"github.com/beclab/Olares/framework/app-service/api/app.bytetrade.io/v1alpha1"
 	"github.com/beclab/Olares/framework/app-service/pkg/appcfg"
 	"github.com/beclab/Olares/framework/app-service/pkg/constants"
-	"github.com/beclab/Olares/framework/app-service/pkg/generated/clientset/versioned"
 	"github.com/beclab/Olares/framework/app-service/pkg/users/userspace"
 	"github.com/beclab/Olares/framework/app-service/pkg/utils"
 	"github.com/beclab/Olares/framework/app-service/pkg/utils/files"
@@ -554,7 +553,7 @@ func parseDestination(dest string) (string, string, error) {
 	return alias, tokens[len(tokens)-1], nil
 }
 
-func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owner string) (appdirs []string, err error) {
+func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owner string, appData bool) (appCacheDirs []string, appDataDirs []string, err error) {
 	userspaceNs := utils.UserspaceName(owner)
 	config, err := ctrl.GetConfig()
 	if err != nil {
@@ -568,7 +567,6 @@ func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owne
 	if err != nil {
 		return
 	}
-	appName := fmt.Sprintf("%s-%s", namespace, name)
 	appCachePath := sts.GetAnnotations()["appcache_hostpath"]
 	if len(appCachePath) == 0 {
 		err = errors.New("empty appcache_hostpath")
@@ -577,20 +575,23 @@ func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owne
 	if !strings.HasSuffix(appCachePath, "/") {
 		appCachePath += "/"
 	}
-	dClient, err := versioned.NewForConfig(config)
-	if err != nil {
+
+	userspacePath := sts.GetAnnotations()["userspace_hostpath"]
+	if len(userspacePath) == 0 {
+		err = errors.New("empty userspace_hostpath annotation")
 		return
 	}
-	appCRD, err := dClient.AppV1alpha1().Applications().Get(ctx, appName, metav1.GetOptions{})
-	if err != nil {
-		return
+	appDataPath := filepath.Join(userspacePath, "Data")
+	if !strings.HasSuffix(appDataPath, "/") {
+		appDataPath += "/"
 	}
-	deploymentName := appCRD.Spec.DeploymentName
+
+	deploymentName := name
 	deployment, err := clientset.AppsV1().Deployments(namespace).
 		Get(context.Background(), deploymentName, metav1.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			return tryToGetAppdataDirFromSts(ctx, namespace, deploymentName, appCachePath)
+			return tryToGetAppdataDirFromSts(ctx, namespace, deploymentName, appCachePath, appDataPath)
 		}
 		return
 	}
@@ -602,15 +603,31 @@ func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owne
 				if appDirSet.Has(appDir) {
 					continue
 				}
-				appdirs = append(appdirs, appDir)
+				appCacheDirs = append(appCacheDirs, appDir)
 				appDirSet.Insert(appDir)
 			}
 		}
 	}
-	return appdirs, nil
+	if appData {
+		appDirSet := sets.NewString()
+
+		for _, v := range deployment.Spec.Template.Spec.Volumes {
+			if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appDataPath) && len(v.HostPath.Path) > len(appDataPath) {
+				appDir := GetFirstSubDir(v.HostPath.Path, appDataPath)
+				if appDir != "" {
+					if appDirSet.Has(appDir) {
+						continue
+					}
+					appDataDirs = append(appDataDirs, appDir)
+					appDirSet.Insert(appDir)
+				}
+			}
+		}
+	}
+	return appCacheDirs, appDataDirs, nil
 }
 
-func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, baseDir string) (appdirs []string, err error) {
+func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, appCacheDir, appDataDir string) (appCacheDirs []string, appDataDirs []string, err error) {
 	config, err := ctrl.GetConfig()
 	if err != nil {
 		return
@@ -627,18 +644,32 @@ func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, baseDir
 	}
 	appDirSet := sets.NewString()
 	for _, v := range sts.Spec.Template.Spec.Volumes {
-		if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, baseDir) && len(v.HostPath.Path) > len(baseDir) {
-			appDir := GetFirstSubDir(v.HostPath.Path, baseDir)
+		if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appCacheDir) && len(v.HostPath.Path) > len(appCacheDir) {
+			appDir := GetFirstSubDir(v.HostPath.Path, appCacheDir)
 			if appDir != "" {
 				if appDirSet.Has(appDir) {
 					continue
 				}
-				appdirs = append(appdirs, appDir)
+				appCacheDirs = append(appCacheDirs, appDir)
 				appDirSet.Insert(appDir)
 			}
 		}
 	}
-	return appdirs, nil
+	appDirSet = sets.NewString()
+
+	for _, v := range sts.Spec.Template.Spec.Volumes {
+		if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appDataDir) && len(v.HostPath.Path) > len(appDataDir) {
+			appDir := GetFirstSubDir(v.HostPath.Path, appDataDir)
+			if appDir != "" {
+				if appDirSet.Has(appDir) {
+					continue
+				}
+				appDataDirs = append(appDataDirs, appDir)
+				appDirSet.Insert(appDir)
+			}
+		}
+	}
+	return appCacheDirs, appDataDirs, nil
 }
 
 func GetFirstSubDir(fullPath, basePath string) string {

framework/backup-server/.olares/config/cluster/deploy/backup_server.yaml

@@ -1,6 +1,6 @@
 
 
-{{ $backupVersion := "0.3.61" }}
+{{ $backupVersion := "0.3.62" }}
 {{ $backup_server_rootpath := printf "%s%s" .Values.rootPath "/rootfs/backup-server" }}
 
 {{- $backup_nats_secret := (lookup "v1" "Secret" .Release.Namespace "backup-nats-secret") -}}

framework/backup-server/pkg/watchers/systemenv/systemenv.go

@@ -57,6 +57,7 @@ func (s *Subscriber) Do(_ context.Context, obj interface{}, action watchers.Acti
 		return fmt.Errorf("invalid object type")
 	}
 	m := *mPtr
+	log.Infof("Sysenv data: %v", m)
 
 	// effective value can be from value or default
 	var newValue string
@@ -66,11 +67,14 @@ func (s *Subscriber) Do(_ context.Context, obj interface{}, action watchers.Acti
 	} else if d, ok := m["default"].(string); ok && d != "" {
 		newValue = d
 	}
+
 	if newValue == "" {
+		constant.OlaresRemoteService = constant.DefaultSyncServerURL
+		constant.SyncServerURL = constant.DefaultSyncServerURL
 		return nil
 	}
 
-	if constant.OlaresRemoteService == newValue {
+	if constant.SyncServerURL == newValue {
 		return nil
 	}
 

framework/download-server/.olares/config/cluster/deploy/download_deploy.yaml

@@ -180,7 +180,7 @@ spec:
             memory: 300Mi
             
       - name: download-server
-        image: "beclab/download-server:v0.1.20"
+        image: "beclab/download-server:v0.1.21"
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 0

framework/market/.olares/config/cluster/deploy/market_deploy.yaml

@@ -46,6 +46,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - iam.kubesphere.io
+    resources:
+      - users
+    verbs:
+      - get
+      - list
+      - watch
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -140,7 +148,7 @@ spec:
           name: check-chart-repo
       containers:
       - name: appstore-backend
-        image: beclab/market-backend:v0.6.17
+        image: beclab/market-backend:v0.6.18
         imagePullPolicy: IfNotPresent
         ports:
           - containerPort: 81

framework/seahub/.olares/config/cluster/deploy/seafile_deploy.yaml

@@ -166,6 +166,7 @@ data:
     user = seafile_os_framework
     password = {{ $pg_password | b64dec }}
     db_name = os_framework_seafile
+    ccnet_db_name = os_framework_ccnet
     connection_charset = utf8
     create_tables = true
   ccnet.conf: |-
@@ -248,7 +249,7 @@ spec:
 
       containers:
       - name: seafile-server
-        image: beclab/pg_seafile_server:v0.0.18
+        image: beclab/pg_seafile_server:v0.0.19
         imagePullPolicy: IfNotPresent
         ports:
           - containerPort: 8082