app-service: inject nvshare environment duplicately

feat: add node to a cluster (#868)

.github/workflows/build-redis.yaml

@@ -20,7 +20,7 @@ jobs:
           bash scripts/build-redis.sh linux/amd64
 
   push-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: Clean

.github/workflows/check.yaml

@@ -68,22 +68,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          coscmd config -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       # test
       - env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -93,7 +77,7 @@ jobs:
           bash scripts/image-manifest.sh && bash scripts/upload-images.sh .manifest/images.mf
 
   push-image-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: 'Checkout source code'
@@ -103,22 +87,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
 
       - env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -140,22 +108,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          coscmd config -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       # test
       - env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -165,7 +117,7 @@ jobs:
           bash scripts/deps-manifest.sh && bash scripts/upload-deps.sh
 
   push-deps-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: "Checkout source code"
@@ -178,20 +130,6 @@ jobs:
       - name: Install coscmd
         run: pip install coscmd        
 
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       # test
       - env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/push-deps-to-s3.yml

@@ -36,7 +36,7 @@ jobs:
           bash scripts/deps-manifest.sh && bash scripts/upload-deps.sh
 
   push-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: "Checkout source code"

.github/workflows/push-to-s3.yaml

@@ -36,7 +36,7 @@ jobs:
           bash scripts/image-manifest.sh && bash scripts/upload-images.sh .manifest/images.mf
 
   push-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: "Checkout source code"

.github/workflows/release-daily.yaml

@@ -16,22 +16,6 @@ jobs:
       - name: 'Checkout source code'
         uses: actions/checkout@v3
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          coscmd config -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       - env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -40,29 +24,12 @@ jobs:
           bash scripts/image-manifest.sh && bash scripts/upload-images.sh .manifest/images.mf
 
   push-images-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: 'Checkout source code'
         uses: actions/checkout@v3
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       - env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -78,22 +45,6 @@ jobs:
       - name: "Checkout source code"
         uses: actions/checkout@v3
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          coscmd config -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       # test
       - env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -103,29 +54,12 @@ jobs:
           bash scripts/deps-manifest.sh && bash scripts/upload-deps.sh
 
   push-deps-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: "Checkout source code"
         uses: actions/checkout@v3
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       # test
       - env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -154,29 +88,6 @@ jobs:
         run: |
           bash scripts/build.sh ${{ steps.vars.outputs.tag_version }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-          
-      # - name: Upload to COS
-      #   run: |
-      #     md5sum install-wizard-v${{ steps.vars.outputs.tag_version }}.tar.gz > install-wizard-v${{ steps.vars.outputs.tag_version }}.md5sum.txt && \
-      #     coscmd upload ./install-wizard-v${{ steps.vars.outputs.tag_version }}.md5sum.txt /install-wizard-v${{ steps.vars.outputs.tag_version }}.md5sum.txt
-      #     coscmd upload ./install-wizard-v${{ steps.vars.outputs.tag_version }}.tar.gz /install-wizard-v${{ steps.vars.outputs.tag_version }}.tar.gz
-
       - name: Upload to S3
         env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -230,6 +141,7 @@ jobs:
             build/installer/publicInstaller.sh
             build/installer/install.sh
             build/installer/install.ps1
+            build/installer/joincluster.sh
             build/installer/publicAddnode.sh
             build/installer/version.hint
             build/installer/publicRestoreInstaller.sh

.github/workflows/release.yaml

@@ -18,22 +18,6 @@ jobs:
         with:
           ref: ${{ github.event.inputs.tags }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          coscmd config -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       - env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -42,7 +26,7 @@ jobs:
           bash scripts/image-manifest.sh && bash scripts/upload-images.sh .manifest/images.mf
 
   push-arm64:
-    runs-on: self-hosted
+    runs-on: [self-hosted, linux, ARM64]
 
     steps:
       - name: 'Checkout source code'
@@ -50,23 +34,6 @@ jobs:
         with:
           ref: ${{ github.event.inputs.tags }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-
       - env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -89,29 +56,6 @@ jobs:
         run: |
           bash scripts/build.sh ${{ github.event.inputs.tags }}
 
-      - name: Install coscmd
-        run: pip install coscmd        
-
-      - name: Configure coscmd
-        env:
-          TENCENT_SECRET_ID: ${{ secrets.TENCENT_SECRET_ID }}
-          TENCENT_SECRET_KEY: ${{ secrets.TENCENT_SECRET_KEY }}
-          COS_BUCKET: ${{ secrets.COS_BUCKET }}
-          COS_REGION: ${{ secrets.COS_REGION }}
-          END_POINT: ${{ secrets.END_POINT }}
-        run: |
-          export PATH=$PATH:/usr/local/bin:/home/ubuntu/.local/bin
-          coscmd config -m 10 -p 10 -a $TENCENT_SECRET_ID \
-                        -s $TENCENT_SECRET_KEY \
-                        -b $COS_BUCKET \
-                        -r $COS_REGION 
-          
-      # - name: Upload to COS
-      #   run: |
-      #     md5sum install-wizard-v${{ github.event.inputs.tags }}.tar.gz > install-wizard-v${{ github.event.inputs.tags }}.md5sum.txt && \
-      #     coscmd upload ./install-wizard-v${{ github.event.inputs.tags }}.md5sum.txt /install-wizard-v${{ github.event.inputs.tags }}.md5sum.txt
-      #     coscmd upload ./install-wizard-v${{ github.event.inputs.tags }}.tar.gz /install-wizard-v${{ github.event.inputs.tags }}.tar.gz
-
       - name: Upload to S3
         env: 
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -174,6 +118,7 @@ jobs:
             build/installer/publicInstaller.latest.ps1
             build/installer/install.ps1
             build/installer/publicAddnode.sh
+            build/installer/joincluster.sh
             build/installer/version.hint
             build/installer/publicRestoreInstaller.sh
           prerelease: true

apps/desktop/config/user/helm-charts/desktop/templates/desktop_deploy.yaml

@@ -23,6 +23,7 @@ spec:
         runAsUser: 1000
         runAsGroup: 1000
         fsGroup: 1000
+      priorityClassName: "system-cluster-critical"
       initContainers:
       - args:
         - -it
@@ -65,7 +66,7 @@ spec:
 
       containers:
       - name: edge-desktop
-        image: beclab/desktop:v0.2.45
+        image: beclab/desktop:v0.2.46
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsNonRoot: false
@@ -77,7 +78,7 @@ spec:
             value: http://bfl.{{ .Release.Namespace }}:8080
 
       - name: desktop-server
-        image: beclab/desktop-server:v0.2.45
+        image: beclab/desktop-server:v0.2.46
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

apps/download/config/user/helm-charts/download/templates/download_deploy.yaml

@@ -146,7 +146,7 @@ spec:
             value: user_space_{{ .Values.bfl.username }}_knowledge
       containers:
       - name: aria2
-        image: "beclab/aria2:v0.0.3"
+        image: "beclab/aria2:v0.0.4"
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsNonRoot: false
@@ -172,7 +172,7 @@ spec:
             cpu: "1"
             memory: 300Mi
       - name: yt-dlp
-        image: "beclab/yt-dlp:v0.0.16"
+        image: "beclab/yt-dlp:v0.0.19"
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false
@@ -220,7 +220,7 @@ spec:
             cpu: "1"
             memory: 300Mi
       - name: download-spider
-        image: "beclab/download-spider:v0.0.15"
+        image: "beclab/download-spider:v0.0.16"
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false

apps/files/config/cluster/deploy/files_deploy.yaml

@@ -15,6 +15,14 @@
 {{ $files_redis_password = randAlphaNum 16 | b64enc }}
 {{- end -}}
 
+{{- $files_nats_secret := (lookup "v1" "Secret" "os-system" "files-nats-secrets") -}}
+{{- $files_nats_password := "" -}}
+{{ if $files_nats_secret -}}
+{{ $files_nats_password = (index $files_nats_secret "data" "files_nats_password") }}
+{{ else -}}
+{{ $files_nats_password = randAlphaNum 16 | b64enc }}
+{{- end -}}
+
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -61,7 +69,7 @@ spec:
             chown -R 1000:1000 /appdata; chown -R 1000:1000 /appcache; chown -R 1000:1000 /data
       containers:
         - name: gateway
-          image: beclab/appdata-gateway:0.1.15
+          image: beclab/appdata-gateway:0.1.16
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: false
@@ -70,7 +78,7 @@ spec:
             - containerPort: 8080
           env:
             - name: FILES_SERVER_TAG
-              value: 'beclab/files-server:v0.2.45'
+              value: 'beclab/files-server:v0.2.54'
             - name: NAMESPACE
               valueFrom:
                 fieldRef:
@@ -88,6 +96,10 @@ spec:
             value: seafile
           image: beclab/media-server:v0.1.10
           imagePullPolicy: IfNotPresent
+          securityContext:
+            allowPrivilegeEscalation: true
+            runAsUser: 1000
+            privileged: true
           ports:
           - containerPort: 9090
           volumeMounts:
@@ -98,10 +110,11 @@ spec:
 {{ if .Values.sharedlib }}
           - name: shared-lib
             mountPath: /data/External
+            mountPropagation: Bidirectional
 {{ end }}
 
         - name: files
-          image: beclab/files-server:v0.2.45
+          image: beclab/files-server:v0.2.54
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: true
@@ -191,6 +204,20 @@ spec:
               # use redis db 0 for redis cache
             - name: REDIS_DB
               value: '0'
+            - name: NATS_HOST
+              value: nats
+            - name: NATS_PORT
+              value: '4222'
+            - name: NATS_USERNAME
+              value: os-system-files-server
+            - name: NATS_PASSWORD
+              value: {{ $files_nats_password | b64dec }}
+            - name: NATS_SUBJECT
+              value: terminus.os-system.files-notify
+            - name: RESERVED_SPACE
+              value: '1000'
+            - name: OLARES_VERSION
+              value: '1.11'
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -207,12 +234,14 @@ spec:
             - /filebrowser
             - --noauth
         - name: uploader
-          image: beclab/upload:v1.0.7
+          image: beclab/upload:v1.0.8
           env:
             - name: UPLOAD_FILE_TYPE
               value: '*'
             - name: UPLOAD_LIMITED_SIZE
-              value: '21474836481'
+              value: '118111600640'
+            - name: RESERVED_SPACE
+              value: '1000'
           volumeMounts:
             - name: fb-data
               mountPath: /appdata
@@ -223,13 +252,18 @@ spec:
 {{ if .Values.sharedlib }}
             - name: shared-lib
               mountPath: /data/External
+              mountPropagation: Bidirectional
 {{ end }}
           resources: { }
           terminationMessagePath: /dev/termination-log
           terminationMessagePolicy: File
           imagePullPolicy: IfNotPresent
+          securityContext:
+            allowPrivilegeEscalation: true
+            runAsUser: 1000
+            privileged: true
         - name: nginx
-          image: 'beclab/nginx-lua:n0.0.4'
+          image: 'nginx:stable-alpine3.17-slim'
           securityContext:
             runAsNonRoot: false
             runAsUser: 0
@@ -237,6 +271,10 @@ spec:
             - containerPort: 80
               protocol: TCP
           volumeMounts:
+            - name: files-nginx-config
+              readOnly: true
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
             - name: files-nginx-config
               mountPath: /etc/nginx/conf.d/default.conf
               subPath: default.conf
@@ -261,6 +299,8 @@ spec:
           configMap:
             name: files-nginx-config
             items:
+              - key: nginx.conf
+                path: nginx.conf
               - key: default.conf
                 path: default.conf
             defaultMode: 420
@@ -345,10 +385,16 @@ spec:
           - sh
           - -c
           - |
-            chown -R 1000:1000 /appdata 
+            chown -R 1000:1000 /appdata
+        - args:
+          - -it
+          - nats.os-system:4222
+          image: owncloudci/wait-for:latest
+          imagePullPolicy: IfNotPresent
+          name: check-nats
       containers:
         - name: files
-          image: beclab/files-server:v0.2.45
+          image: beclab/files-server:v0.2.54
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: false
@@ -367,6 +413,8 @@ spec:
               value: /appdata/config/settings.json
             - name: FB_ROOT
               value: /data
+            - name: OLARES_VERSION
+              value: '1.11'
             - name: NODE_NAME
               valueFrom:
                 fieldRef:
@@ -412,6 +460,16 @@ data:
   password: {{ $password }}
   files_redis_password: {{ $files_redis_password }}
 
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: files-nats-secrets
+  namespace: os-system
+data:
+  files_nats_password: {{ $files_nats_password }}
+type: Opaque
+
 ---
 apiVersion: apr.bytetrade.io/v1alpha1
 kind: MiddlewareRequest
@@ -430,6 +488,37 @@ spec:
           name: files-secrets
     namespace: files-redis
 
+---
+apiVersion: apr.bytetrade.io/v1alpha1
+kind: MiddlewareRequest
+metadata:
+  name: files-server-nat
+  namespace: os-system
+spec:
+  app: files-server
+  appNamespace: os-system
+  middleware: nats
+  nats:
+    password:
+      valueFrom:
+        secretKeyRef:
+          key: files_nats_password
+          name: files-nats-secrets
+    refs: []
+    subjects:
+      - export:
+          - appName: files-frontend
+            pub: allow
+            sub: allow
+          - appName: vault
+            pub: allow
+            sub: allow
+        name: files-notify
+        permission:
+          pub: allow
+          sub: allow
+    user: os-system-files-server
+
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -439,6 +528,37 @@ metadata:
   annotations:
     kubesphere.io/creator: bytetrade.io
 data:
+  nginx.conf: |-
+    user  nginx;
+    worker_processes  4;
+
+    error_log  /var/log/nginx/error.log notice;
+    pid        /var/run/nginx.pid;
+
+    events {
+        worker_connections  1024;
+    }
+
+    http {
+        include       /etc/nginx/mime.types;
+        default_type  application/octet-stream;
+
+        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                          '$status $body_bytes_sent "$http_referer" '
+                          '"$http_user_agent" "$http_x_forwarded_for"';
+
+        access_log  /var/log/nginx/access.log  main;
+
+        sendfile        on;
+        #tcp_nopush     on;
+
+        keepalive_timeout  2700;
+
+        #gzip  on;
+        client_max_body_size 4000M;
+
+        include /etc/nginx/conf.d/*.conf;
+    }
   default.conf: |-
     server {
     	listen 80 default_server;
@@ -505,12 +625,77 @@ data:
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Host $host;
 
-            client_body_timeout 60s;
-            client_max_body_size 2000M;
+            client_body_timeout 1800s;
+            client_max_body_size 4000M;
             proxy_request_buffering off;
-            keepalive_timeout 75s;
-            proxy_read_timeout 60s;
-            proxy_send_timeout 60s;
+            keepalive_timeout 2700s;
+            proxy_read_timeout 1800s;
+            proxy_send_timeout 1800s;
+        }
+
+        location /api/raw {
+            proxy_pass http://127.0.0.1:8080;
+            # rewrite ^/server(.*)$ $1 break;
+
+            # Add original-request-related headers
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+
+            client_body_timeout 1800s;
+            client_max_body_size 4000M;
+            proxy_request_buffering off;
+            keepalive_timeout 2700s;
+            proxy_read_timeout 1800s;
+            proxy_send_timeout 1800s;
+        }
+
+        location /api/md5 {
+            proxy_pass http://127.0.0.1:8080;
+            # rewrite ^/server(.*)$ $1 break;
+            # Add original-request-related headers
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+            add_header Accept-Ranges bytes;
+            client_body_timeout 1800s;
+            client_max_body_size 4000M;
+            proxy_request_buffering off;
+            keepalive_timeout 2700s;
+            proxy_read_timeout 1800s;
+            proxy_send_timeout 1800s;
+        }
+
+        location /api/paste {
+            proxy_pass http://127.0.0.1:8080;
+            # rewrite ^/server(.*)$ $1 break;
+            # Add original-request-related headers
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+            add_header Accept-Ranges bytes;
+            client_body_timeout 1800s;
+            client_max_body_size 4000M;
+            proxy_request_buffering off;
+            keepalive_timeout 2700s;
+            proxy_read_timeout 1800s;
+            proxy_send_timeout 1800s;
+        }
+
+        location /api/cache {
+            proxy_pass http://127.0.0.1:8080;
+            # rewrite ^/server(.*)$ $1 break;
+            # Add original-request-related headers
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+            add_header Accept-Ranges bytes;
+            client_body_timeout 1800s;
+            client_max_body_size 4000M;
+            proxy_request_buffering off;
+            keepalive_timeout 2700s;
+            proxy_read_timeout 1800s;
+            proxy_send_timeout 1800s;
         }
 
         location /provider {

apps/files/config/user/helm-charts/files/templates/files_fe_deploy.yaml

@@ -27,6 +27,14 @@
 {{ $pg_password = randAlphaNum 16 | b64enc }}
 {{- end -}}
 
+{{- $files_frontend_nats_secret := (lookup "v1" "Secret" $namespace "files-frontend-nats-secrets") -}}
+{{- $files_frontend_nats_password := "" -}}
+{{ if $files_frontend_nats_secret -}}
+{{ $files_frontend_nats_password = (index $files_frontend_nats_secret "data" "files_frontend_nats_password") }}
+{{ else -}}
+{{ $files_frontend_nats_password = randAlphaNum 16 | b64enc }}
+{{- end -}}
+
 
 ---
 apiVersion: v1
@@ -134,6 +142,12 @@ spec:
         image: owncloudci/wait-for:latest
         imagePullPolicy: IfNotPresent
         name: check-auth
+      - args:
+        - -it
+        - nats.user-system-{{ .Values.bfl.username }}:4222
+        image: owncloudci/wait-for:latest
+        imagePullPolicy: IfNotPresent
+        name: check-nats
       - name: terminus-sidecar-init
         image: openservicemesh/init:v1.2.3
         imagePullPolicy: IfNotPresent
@@ -283,13 +297,24 @@ spec:
 #        - /filebrowser
 #        - --noauth
       - name: files-frontend
-        image: beclab/files-frontend:v1.2.69
+        image: beclab/files-frontend-1.11:v1.3.24
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsNonRoot: false
           runAsUser: 0
         ports:
         - containerPort: 80
+        env:
+          - name: NATS_HOST
+            value: nats.user-system-{{ .Values.bfl.username }}
+          - name: NATS_PORT
+            value: '4222'
+          - name: NATS_USERNAME
+            value: user-system-{{ .Values.bfl.username }}-files-frontend
+          - name: NATS_PASSWORD
+            value: {{ $files_frontend_nats_password | b64dec }}
+          - name: NATS_SUBJECT
+            value: terminus.os-system.files-notify
         volumeMounts:
         - name: userspace-dir
           mountPath: /data
@@ -606,6 +631,16 @@ data:
   redis_password: {{ $redis_password }}
   pg_password: {{ $pg_password }}
 
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: files-frontend-nats-secrets
+  namespace: user-system-{{ .Values.bfl.username }}
+data:
+  files_frontend_nats_password: {{ $files_frontend_nats_password }}
+type: Opaque
+
 #---
 #apiVersion: apr.bytetrade.io/v1alpha1
 #kind: MiddlewareRequest
@@ -646,6 +681,31 @@ spec:
           name: zinc-files-secrets
     namespace: zinc-files
 
+---
+apiVersion: apr.bytetrade.io/v1alpha1
+kind: MiddlewareRequest
+metadata:
+  name: files-frontend-nat
+  namespace: user-system-{{ .Values.bfl.username }}
+spec:
+  app: files-frontend
+  appNamespace: user-space-{{ .Values.bfl.username }}
+  middleware: nats
+  nats:
+    password:
+      valueFrom:
+        secretKeyRef:
+          key: files_frontend_nats_password
+          name: files-frontend-nats-secrets
+    refs:
+      - appName: files-server
+        appNamespace: os-system
+        subjects:
+          - name: files-notify
+            perm:
+              - pub
+              - sub
+    user: user-system-{{ .Values.bfl.username }}-files-frontend
 
 ---
 apiVersion: v1

apps/knowledge/config/user/helm-charts/knowledge/templates/knowledge_deployment.yaml

@@ -168,7 +168,7 @@ spec:
             value: user_space_{{ .Values.bfl.username }}_knowledge
       containers:
       - name: knowledge
-        image: "beclab/knowledge-base-api:v0.1.56"
+        image: "beclab/knowledge-base-api:v0.1.61"
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false
@@ -236,7 +236,7 @@ spec:
             memory: 1Gi
 
       - name: backend-server
-        image: "beclab/recommend-backend:v0.0.24"
+        image: "beclab/recommend-backend:v0.0.25"
         imagePullPolicy: IfNotPresent
         securityContext:
           allowPrivilegeEscalation: false
@@ -367,7 +367,7 @@ spec:
             memory: 800Mi
 
       - name: terminus-ws-sidecar
-        image: 'beclab/ws-gateway:v1.0.3'
+        image: 'beclab/ws-gateway:v1.0.4'
         imagePullPolicy: IfNotPresent
         command:
           - /ws-gateway
@@ -421,6 +421,10 @@ spec:
       protocol: TCP
       port: 3010
       targetPort: 3010
+    - name: "knowledge-websocket"
+      protocol: TCP
+      port: 40010
+      targetPort: 40010
 
 ---
 apiVersion: v1

apps/market/config/user/helm-charts/market/templates/market_deploy.yaml

@@ -1,9 +1,8 @@
-{{- $namespace := printf "%s%s" "user-system-" .Values.bfl.username -}}
-{{- $market_secret := (lookup "v1" "Secret" $namespace "market-secrets") -}}
+{{- $market_secret := (lookup "v1" "Secret" .Release.Namespace "market-secrets") -}}
 
 {{- $redis_password := "" -}}
 {{ if $market_secret -}}
-{{ $redis_password = (index $market_secret "data" "redis_password") }}
+{{ $redis_password = (index $market_secret "data" "redis-passwords") }}
 {{ else -}}
 {{ $redis_password = randAlphaNum 16 | b64enc }}
 {{- end -}}
@@ -85,12 +84,12 @@ spec:
                 fieldPath: status.podIP
       containers:
       - name: appstore
-        image: beclab/market-frontend:v0.2.30
+        image: beclab/market-frontend:v0.3.4
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80
       - name: appstore-backend
-        image: beclab/market-backend:v0.2.30
+        image: beclab/market-backend:v0.3.4
         imagePullPolicy: IfNotPresent
         ports:
           - containerPort: 81

apps/rss/config/cluster/deploy/rss_deploy.yaml

@@ -24,7 +24,7 @@ spec:
     spec:
       containers:
         - name: rss-server
-          image: beclab/rsshub-server:v0.0.2
+          image: beclab/rsshub-server:v0.0.3
           imagePullPolicy: IfNotPresent
           ports:
           - containerPort: 1200

apps/system-apps/config/cluster/deploy/ks_server_deploy.yaml

@@ -22,7 +22,7 @@ spec:
     spec:
       containers:
       - name: monitoring-server
-        image: beclab/monitoring-server-v1:v0.2.3
+        image: beclab/monitoring-server-v1:v0.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8000

apps/system-apps/config/user/helm-charts/monitoring/templates/system-frontend.yaml

@@ -137,6 +137,7 @@ spec:
         app: system-frontend
         io.bytetrade.app: "true"
     spec:
+      priorityClassName: "system-cluster-critical"
       initContainers:
         - args:
             - -it
@@ -177,7 +178,7 @@ spec:
                   apiVersion: v1
                   fieldPath: status.podIP
         - name: dashboard-init
-          image: beclab/dashboard-frontend-v1:v0.4.4
+          image: beclab/dashboard-frontend-v1:v0.4.9
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -189,7 +190,7 @@ spec:
             - mountPath: /www
               name: www-dir
         - name: control-hub-init
-          image: beclab/admin-console-frontend-v1:v0.4.8
+          image: beclab/admin-console-frontend-v1:v0.4.12
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -201,7 +202,7 @@ spec:
             - mountPath: /www
               name: www-dir
         - name: profile-editor-init
-          image: beclab/profile-editor:v0.2.0
+          image: beclab/profile-editor:v0.2.1
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -213,7 +214,7 @@ spec:
             - mountPath: /www
               name: www-dir
         - name: profile-preview-init
-          image: beclab/profile-preview:v0.2.0
+          image: beclab/profile-preview:v0.2.1
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -225,7 +226,7 @@ spec:
             - mountPath: /www
               name: www-dir
         - name: wise-init
-          image: beclab/wise:v1.2.69
+          image: beclab/wise:v1.3.24
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -237,7 +238,7 @@ spec:
             - mountPath: /www
               name: www-dir
         - name: settings-init
-          image: beclab/settings:v0.2.0
+          image: beclab/settings:v0.2.10
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -298,7 +299,7 @@ spec:
             - name: www-dir
               mountPath: /www
             - name: wise-download-dir
-              mountPath: /data/Home/Downloads
+              mountPath: /data/Home
             - name: system-frontend-nginx-config
               mountPath: /etc/nginx/nginx.conf
               subPath: nginx.conf
@@ -338,7 +339,7 @@ spec:
                 fieldRef:
                   fieldPath: status.podIP
         - name: terminus-ws-sidecar
-          image: 'beclab/ws-gateway:v1.0.3'
+          image: 'beclab/ws-gateway:v1.0.4'
           imagePullPolicy: IfNotPresent
           command:
             - /ws-gateway
@@ -351,7 +352,7 @@ spec:
           terminationMessagePath: /dev/termination-log
           terminationMessagePolicy: File
         - name: settings-server
-          image: beclab/settings-server:v0.2.0
+          image: beclab/settings-server:v0.2.10
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 3000
@@ -394,7 +395,7 @@ spec:
             path: {{ .Values.userspace.userData }}
         - name: terminus-sidecar-config
           configMap:
-            name: sidecar-ws-configs
+            name: sidecar-configs
             items:
               - key: envoy.yaml
                 path: envoy.yaml
@@ -403,7 +404,7 @@ spec:
         - name: wise-download-dir
           hostPath:
             type: Directory
-            path: {{ .Values.userspace.userData }}/Downloads
+            path: {{ .Values.userspace.userData }}
         - name: system-frontend-nginx-config
           configMap:
             name: system-frontend-nginx-config
@@ -622,6 +623,11 @@ spec:
           - settings-event
       op: Create
       uri: /api/event/app_installation_event
+    - filters:
+        type:
+          - entrance-state-event
+      op: Create
+      uri: /api/event/entrance_state_event
     - filters:
         type:
           - system-upgrade-event
@@ -766,6 +772,14 @@ data:
         expires 0;
       }
 
+      location /ws {
+        proxy_pass http://127.0.0.1:40010;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+      }
+
       location /bfl {
         add_header 'Access-Control-Allow-Headers' 'x-api-nonce,x-api-ts,x-api-ver,x-api-source';
         proxy_pass http://bfl;
@@ -779,6 +793,13 @@ data:
       location /kapis {
         proxy_pass http://SettingsServer;
       }
+    
+      location /api/profile/init {
+        proxy_pass http://127.0.0.1:3010;
+        proxy_set_header            Host $host;
+        proxy_set_header            X-real-ip $remote_addr;
+        proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+      }
 
       location /api {
         proxy_pass http://SettingsServer;
@@ -1048,6 +1069,15 @@ data:
         expires 0;
       }
 
+      location /ws {
+        proxy_pass http://rss-svc:40010;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+      }
+
+
         location /knowledge {
           proxy_pass http://KnowledgeServer;
 
@@ -1079,9 +1109,9 @@ data:
             proxy_pass http://ArgoworkflowsSever;
         }
 
-        location ~ ^/download/preview/Downloads/(.*)$
+        location ~ ^/download/preview/(.*)$
         {
-          alias /data/Home/Downloads/$1;
+          alias /data/Home/$1;
         }
 
         location /videos/ {
@@ -1102,6 +1132,44 @@ data:
             proxy_pass http://media-server-service.os-system:9090;
         }
 
+        location /api {
+            proxy_pass http://files-service.os-system:80;
+            # rewrite ^/server(.*)$ $1 break;
+
+            # Add original-request-related headers
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+
+            add_header Accept-Ranges bytes;
+
+            client_body_timeout 600s;
+            client_max_body_size 4000M;
+            proxy_request_buffering off;
+            keepalive_timeout 750s;
+            proxy_read_timeout 600s;
+            proxy_send_timeout 600s;
+        }
+
+        location /upload {
+           proxy_pass http://files-service.os-system:80;
+           # rewrite ^/server(.*)$ $1 break;
+
+           # Add original-request-related headers
+           proxy_set_header X-Real-IP $remote_addr;
+           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+           proxy_set_header X-Forwarded-Host $host;
+
+           add_header Accept-Ranges bytes;
+
+           client_body_timeout 600s;
+           client_max_body_size 4000M;
+           proxy_request_buffering off;
+           keepalive_timeout 750s;
+           proxy_read_timeout 600s;
+           proxy_send_timeout 600s;
+        }
+
         # # files
         # # for all routes matching a dot, check for files and return 404 if not found
         # # e.g. /file.js returns a 404 if not found
@@ -1173,6 +1241,15 @@ data:
             expires 0;
         }
 
+        location /ws {
+          proxy_pass http://127.0.0.1:40010;
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+          proxy_set_header Host $host;
+        }
+
+
         location /kapis {
             proxy_pass http://SettingsServer_Monitoring;
             # rewrite ^/server(.*)$ $1 break;

apps/vault/config/cluster/deploy/vault_server_deploy.yaml

@@ -83,7 +83,7 @@ spec:
             value: os_system_vault
       containers:
       - name: vault-server
-        image: beclab/vault-server:v1.2.69
+        image: beclab/vault-server:v1.3.24
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 3000
@@ -114,7 +114,7 @@ spec:
         - name: vault-attach
           mountPath: /padloc/packages/server/attachments
       - name: vault-admin
-        image: beclab/vault-admin:v1.2.69
+        image: beclab/vault-admin:v1.3.24
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 3010

apps/vault/config/user/helm-charts/vault/templates/vault_deploy.yaml

@@ -1,3 +1,13 @@
+{{- $namespace := printf "%s%s" "user-system-" .Values.bfl.username -}}
+
+{{- $vault_nats_secret := (lookup "v1" "Secret" $namespace "vault-nats-secrets") -}}
+{{- $vault_nats_password := "" -}}
+{{ if $vault_nats_secret -}}
+{{ $vault_nats_password = (index $vault_nats_secret "data" "vault_nats_password") }}
+{{ else -}}
+{{ $vault_nats_password = randAlphaNum 16 | b64enc }}
+{{- end -}}
+
 
 
 ---
@@ -36,6 +46,12 @@ spec:
         image: owncloudci/wait-for:latest
         imagePullPolicy: IfNotPresent
         name: check-auth
+      - args:
+        - -it
+        - nats.user-system-{{ .Values.bfl.username }}:4222
+        image: owncloudci/wait-for:latest
+        imagePullPolicy: IfNotPresent
+        name: check-nats
       - name: terminus-sidecar-init
         image: openservicemesh/init:v1.2.3
         imagePullPolicy: IfNotPresent
@@ -72,13 +88,13 @@ spec:
 
       containers:
       - name: vault-frontend
-        image: beclab/vault-frontend:v1.2.69
+        image: beclab/vault-frontend:v1.3.24
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80
       
       - name: notification-server
-        image: beclab/vault-notification:v1.2.69
+        image: beclab/vault-notification:v1.3.24
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 3010
@@ -93,6 +109,17 @@ spec:
             value: '{{ .Values.os.vault.appSecret }}'
           - name: OS_APP_KEY
             value: {{ .Values.os.vault.appKey }}
+          - name: NATS_HOST
+            value: nats.user-system-{{ .Values.bfl.username }}
+          - name: NATS_PORT
+            value: '4222'
+          - name: NATS_USERNAME
+            value: user-system-{{ .Values.bfl.username }}-vault
+          - name: NATS_PASSWORD
+            value: {{ $vault_nats_password | b64dec }}
+          - name: NATS_SUBJECT
+            value: terminus.os-system.files-notify
+          
 
       - name: terminus-envoy-sidecar
         image: bytetrade/envoy:v1.25.11
@@ -238,3 +265,38 @@ spec:
     version: v1
 status:
   state: active
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-nats-secrets
+  namespace: user-system-{{ .Values.bfl.username }}
+data:
+  vault_nats_password: {{ $vault_nats_password }}
+type: Opaque
+
+---
+apiVersion: apr.bytetrade.io/v1alpha1
+kind: MiddlewareRequest
+metadata:
+  name: vault-nat
+  namespace: user-system-{{ .Values.bfl.username }}
+spec:
+  app: vault
+  appNamespace: user-space-{{ .Values.bfl.username }}
+  middleware: nats
+  nats:
+    password:
+      valueFrom:
+        secretKeyRef:
+          key: vault_nats_password
+          name: vault-nats-secrets
+    refs:
+      - appName: files-server
+        appNamespace: os-system
+        subjects:
+          - name: files-notify
+            perm:
+              - pub
+              - sub
+    user: user-system-{{ .Values.bfl.username }}-vault

apps/wizard/config/user/helm-charts/wizard/templates/wizard_deploy.yaml

@@ -61,7 +61,7 @@ spec:
 
       containers:
       - name: wizard
-        image: beclab/wizard:v0.5.11
+        image: beclab/wizard:v0.5.12
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80

build/installer/deploy/device-plugin.yaml

@@ -28,6 +28,8 @@ spec:
     spec:
       runtimeClassName: nvidia # Explicitly request the runtime
       priorityClassName: system-node-critical
+      nodeSelector:
+        gpu.bytetrade.io/cuda-supported: 'true'
       initContainers:
       - name: init-dir
         image: busybox:1.28
@@ -40,7 +42,7 @@ spec:
         - "[ -d /var/run/nvshare/libnvshare.so ] && rm -rf /var/run/nvshare/libnvshare.so || true"
       containers:
       - name: nvshare-lib
-        image: beclab/nvshare:libnvshare-v0.0.2
+        image: beclab/nvshare:libnvshare-v0.0.1
         command:
         - sleep
         - infinity
@@ -50,7 +52,7 @@ spec:
               command:
               - "/bin/sh"
               - "-c"
-              - "test -f /host-var-run-nvshare/libnvshare.so || touch /host-var-run-nvshare/libnvshare.so && mount -v --bind /libnvshare.so /host-var-run-nvshare/libnvshare.so"
+              - "test -f /host-var-run-nvshare/libnvshare.so || ( test -d /host-var-run-nvshare/libnvshare.so && rm -rf /host-var-run-nvshare/libnvshare.so && false ) || touch /host-var-run-nvshare/libnvshare.so && mount -v --bind /libnvshare.so /host-var-run-nvshare/libnvshare.so"
           preStop:
             exec:
               command:

build/installer/deploy/nvidia-device-plugin.yml

@@ -44,6 +44,8 @@ spec:
       # be rescheduled after a failure.
       # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
       priorityClassName: "system-node-critical"
+      nodeSelector:
+        gpu.bytetrade.io/cuda-supported: 'true'
       containers:
       - image: nvcr.io/nvidia/k8s-device-plugin:v0.16.1
         name: nvidia-device-plugin-ctr

build/installer/deploy/scheduler.yaml

@@ -28,6 +28,8 @@ spec:
     spec:
       runtimeClassName: nvidia # Explicitly request the runtime
       priorityClassName: system-node-critical
+      nodeSelector:
+        gpu.bytetrade.io/cuda-supported: 'true'
       initContainers:
       - name: init-dir
         image: busybox:1.28
@@ -46,6 +48,10 @@ spec:
           allowPrivilegeEscalation: false
           capabilities:
             drop: ["ALL"]
+        command:
+        - sh
+        - -c
+        - "test -f /var/run/nvshare/scheduler.sock && rm -rf /var/run/nvshare/scheduler.sock; pid1 nvshare-scheduler"
         volumeMounts:
           - name: nvshare-socket-directory
             mountPath: /var/run/nvshare

build/installer/install.ps1

@@ -1,6 +1,8 @@
 $currentPath = Get-Location
 $architecture = $env:PROCESSOR_ARCHITECTURE
+$downloadCdnUrlFromEnv = $env:DOWNLOAD_CDN_URL
 $version = "#__VERSION__"
+$downloadUrl = "https://dc3p1870nn3cj.cloudfront.net"
 
 function Test-Wait {
   while ($true) {
@@ -8,42 +10,78 @@ function Test-Wait {
   }
 }
 
+$runAsAdmin = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())
+if (-not $runAsAdmin.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
+    Write-Host "`n`nThe installation script needs to be run as an administrator.`n"
+    Write-Host "Please try the following methods:`n"
+    Write-Host "1. Search for 'PowerShell' in the Start menu, right-click it, and select 'Run as administrator'. "
+    Write-Host "   Navigate to the directory where the installation script is located and run the installation script.`n"
+    Write-Host "2. Press Win + R, type 'powershell', and then press Ctrl + Shift + Enter. "
+    Write-Host "   Navigate to the directory where the installation script is located and run the installation script.`n"
+    Write-Host "`nPress Ctrl+C to exit.`n"
+    Test-Wait
+}
+
 $process = Get-Process -Name olares-cli -ErrorAction SilentlyContinue
 if ($process) {
   Write-Host "olares-cli.exe is running, Press Ctrl+C to exit."
   Test-Wait
 }
 
+$distro = wsl --list | Select-String -Pattern "^Ubuntu$"
+if (-not $distro -eq "") {
+  Write-Host "Distro Olares exists, please unregister it first."
+  exit 1
+}
+
 $arch = "amd64"
 if ($architecture -like "ARM") {
   $arch = "arm64"
 }
 
-$CLI_VERSION = "0.1.75"
+if (-Not $downloadCdnUrlFromEnv -eq "") {
+  $downloadUrl = $downloadCdnUrlFromEnv
+}
+
+$CLI_PROGRAM_PATH = "{0}\" -f $currentPath
+if (-Not (Test-Path $CLI_PROGRAM_PATH)) {
+  New-Item -Path $CLI_PROGRAM_PATH -ItemType Directory
+}
+
+$CLI_VERSION = "0.1.107"
 $CLI_FILE = "olares-cli-v{0}_windows_{1}.tar.gz" -f $CLI_VERSION, $arch
-$CLI_URL = "https://dc3p1870nn3cj.cloudfront.net/{0}" -f $CLI_FILE
-$CLI_PATH = "{0}\{1}"  -f $currentPath, $CLI_FILE
-if (-Not (Test-Path $CLI_FILE)) {
+$CLI_URL = "{0}/{1}" -f $downloadUrl, $CLI_FILE
+$CLI_PATH = "{0}{1}" -f $CLI_PROGRAM_PATH, $CLI_FILE
+
+$download = 0
+if (Test-Path $CLI_PATH) {
+  tar -xzf $CLI_PATH -C $CLI_PROGRAM_PATH *> $null
+  if (-Not ($LASTEXITCODE -eq 0)) {
+    Remove-Item -Path $CLI_PATH
+    $download = 1
+  }
+} else {
+  $download = 1
+}
+
+if ($download -eq 1) {
   curl -Uri $CLI_URL -OutFile $CLI_PATH
+  Write-Host "Downloading olares-cli.exe..."
+  if (-Not (Test-Path $CLI_PATH)) {
+    Write-Host "Download olares-cli.exe failed."
+    exit 1
+  }
+  tar -xzf $CLI_PATH -C $CLI_PROGRAM_PATH *> $null
+  $cliPath = "{0}\olares-cli.exe" -f $CLI_PROGRAM_PATH
+  if ( -Not (Test-Path $cliPath)) {
+    Write-Host "olares-cli.exe not found."
+    exit 1
+  }
 }
 
-if (-Not (Test-Path $CLI_PATH)) {
-  Write-Host "Download olares-cli.exe failed."
-  exit 1
-}
-
-tar -xf $CLI_PATH
-$cliPath = "{0}\olares-cli.exe" -f $currentPath
-if ( -Not (Test-Path $cliPath)) {
-  Write-Host "olares-cli.exe not found."
-  exit 1
-}
-
-wsl --unregister Ubuntu *> $null
-
 Start-Sleep -Seconds 3
 Write-Host ("Preparing to start the installation of Olares {0}. Depending on your network conditions, this process may take several minutes." -f $version)
 
-$command = "{0} olares install --version {1}" -f $cliPath, $version
+$command = "{0}\olares-cli.exe olares install --version {1}" -f $CLI_PROGRAM_PATH, $version
 Start-Process cmd -ArgumentList '/k',$command -Wait -Verb RunAs
 

build/installer/install.sh

@@ -28,16 +28,16 @@ fi
 os_type=$(uname -s)
 os_arch=$(uname -m)
 
-case "$os_arch" in 
-    arm64) ARCH=arm64; ;; 
-    x86_64) ARCH=amd64; ;; 
-    armv7l) ARCH=arm; ;; 
-    aarch64) ARCH=arm64; ;; 
-    ppc64le) ARCH=ppc64le; ;; 
-    s390x) ARCH=s390x; ;; 
+case "$os_arch" in
+    arm64) ARCH=arm64; ;;
+    x86_64) ARCH=amd64; ;;
+    armv7l) ARCH=arm; ;;
+    aarch64) ARCH=arm64; ;;
+    ppc64le) ARCH=ppc64le; ;;
+    s390x) ARCH=s390x; ;;
     *) echo "error: unsupported arch \"$os_arch\"";
     exit 1; ;;
-esac 
+esac
 
 # set shell execute command
 user="$(id -un 2>/dev/null || true)"
@@ -74,13 +74,14 @@ if [ -z ${cdn_url} ]; then
     cdn_url="https://dc3p1870nn3cj.cloudfront.net"
 fi
 
-CLI_VERSION="0.1.75"
+CLI_VERSION="0.1.107"
 CLI_FILE="olares-cli-v${CLI_VERSION}_linux_${ARCH}.tar.gz"
 if [[ x"$os_type" == x"Darwin" ]]; then
     CLI_FILE="olares-cli-v${CLI_VERSION}_darwin_${ARCH}.tar.gz"
 fi
 
 if command_exists olares-cli && [[ "$(olares-cli -v | awk '{print $3}')" == "$CLI_VERSION" ]]; then
+    INSTALL_OLARES_CLI=$(which olares-cli)
     echo "olares-cli already installed and is the expected version"
     echo ""
 else
@@ -136,16 +137,22 @@ else
             echo ""
         else
             echo "building local release ..."
-            $sh_c "olares-cli olares release $PARAMS $CDN"
+            $sh_c "$INSTALL_OLARES_CLI olares release $PARAMS $CDN"
             if [[ $? -ne 0 ]]; then
                 echo "error: failed to build local release"
                 exit 1
             fi
         fi
     else
+        echo "running system prechecks ..."
+        echo ""
+        $sh_c "$INSTALL_OLARES_CLI olares precheck $PARAMS"
+        if [[ $? -ne 0 ]]; then
+            exit 1
+        fi
         echo "downloading installation wizard..."
         echo ""
-        $sh_c "olares-cli olares download wizard $PARAMS $KUBE_PARAM $CDN"
+        $sh_c "$INSTALL_OLARES_CLI olares download wizard $PARAMS $KUBE_PARAM $CDN"
         if [[ $? -ne 0 ]]; then
             echo "error: failed to download installation wizard"
             exit 1
@@ -154,7 +161,7 @@ else
 
     echo "downloading installation packages..."
     echo ""
-    $sh_c "olares-cli olares download component $PARAMS $KUBE_PARAM $CDN"
+    $sh_c "$INSTALL_OLARES_CLI olares download component $PARAMS $KUBE_PARAM $CDN"
     if [[ $? -ne 0 ]]; then
         echo "error: failed to download installation packages"
         exit 1
@@ -166,10 +173,7 @@ else
     if [ x"$REGISTRY_MIRRORS" != x"" ]; then
         extra="--registry-mirrors $REGISTRY_MIRRORS"
     fi
-    if [[ "$JUICEFS" == "1" ]]; then
-        extra="$extra --with-juicefs=true"
-    fi
-    $sh_c "olares-cli olares prepare $PARAMS $KUBE_PARAM $extra"
+    $sh_c "$INSTALL_OLARES_CLI olares prepare $PARAMS $KUBE_PARAM $extra"
     if [[ $? -ne 0 ]]; then
         echo "error: failed to prepare installation environment"
         exit 1
@@ -185,9 +189,24 @@ if [ "$PREINSTALL" == "1" ]; then
     echo "Pre Install mode is specified by the \"PREINSTALL\" env var, skip installing"
     exit 0
 fi
+
+if [[ "$JUICEFS" == "1" ]]; then
+    echo "JuiceFS is enabled"
+    fsflag="--with-juicefs=true"
+    if [[ "$STORAGE" == "" ]]; then
+        echo "installing MinIO ..."
+    else
+        echo "checking storage config ..."
+    fi
+    $sh_c "$INSTALL_OLARES_CLI olares install storage $PARAMS"
+    if [[ $? -ne 0 ]]; then
+      exit 1
+    fi
+fi
+
 echo "installing Olares..."
 echo ""
-$sh_c "olares-cli olares install $PARAMS $KUBE_PARAM"
+$sh_c "$INSTALL_OLARES_CLI olares install $PARAMS $KUBE_PARAM $fsflag"
 
 if [[ $? -ne 0 ]]; then
     echo "error: failed to install Olares"

build/installer/joincluster.sh

@@ -0,0 +1,261 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+set -e
+
+function command_exists() {
+	  command -v "$@" > /dev/null 2>&1
+}
+
+function read_tty() {
+    echo -n $1
+    read $2 < /dev/tty
+}
+
+function confirm() {
+    if [[ "$QUIET" == "1" ]]; then
+        return 0
+    fi
+    answer=""
+    while :; do
+        read_tty "Do you confirm to continue? (y/n): " answer
+        if [[ "$answer" != "y" && "$answer" != "n" ]]; then
+            echo "Please input the letter y or n"
+            continue
+        fi
+        if [[ "$answer" == "y" ]]; then
+            return 0
+        fi
+        if [[ "$answer" == "n" ]]; then
+            exit 0
+        fi
+    done
+}
+
+function validate_ip() {
+    if [[ ! "$1" ]]; then
+        echo "invalid IP: empty address"
+        return 1
+    elif [[ ! $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "invalid IP: illegal format"
+        return 1
+    elif [[ $1 =~ ^127 ]]; then
+        echo "invalid IP: loopback address"
+        return 1
+    else
+        return 0
+    fi
+}
+
+MASTER_SSH_OPTIONS=""
+
+function add_master_host_ssh_options() {
+    MASTER_SSH_OPTIONS="$MASTER_SSH_OPTIONS --$1 $2"
+}
+
+function set_master_host_ssh_options() {
+    master_host="$MASTER_HOST"
+    if [[ ! "$master_host" ]]; then
+        read_tty "Enter the master node's IP: " master_host
+    fi
+
+    while :; do
+        if ! validate_ip "$master_host"; then
+            read_tty "Enter the master node's IP: " master_host
+        else
+            break
+        fi
+    done
+
+    add_master_host_ssh_options master-host "$master_host"
+
+    if [[ "$MASTER_NODE_NAME" ]]; then
+        add_master_host_ssh_options master-node-name "$MASTER_NODE_NAME"
+    fi
+
+    if [[ "$MASTER_SSH_USER" ]]; then
+        add_master_host_ssh_options master-ssh-user "$MASTER_SSH_USER"
+    else
+        echo "the environment variable \$MASTER_SSH_USER is not set"
+        echo "the default remote user \"root\" on the master node will be used to authenticate"
+        echo "if this is unexpected, please set it explicitly"
+        confirm
+    fi
+
+    if [[ "$MASTER_SSH_PASSWORD" ]]; then
+        add_master_host_ssh_options master-ssh-password "$MASTER_SSH_PASSWORD"
+    fi
+
+    if [[ "$MASTER_SSH_PRIVATE_KEY_PATH" ]]; then
+        add_master_host_ssh_options master-ssh-private-key-path "$MASTER_SSH_PRIVATE_KEY_PATH"
+    elif [[ ! "$MASTER_SSH_PASSWORD" ]]; then
+        echo "the environment variable \$MASTER_SSH_PRIVATE_KEY_PATH is not set"
+        echo "the default key in the local path /root/.ssh/id_rsa will be used to authenticate to the master"
+        echo "please make sure the key exists and the public key has already been added to the master node"
+        echo "if this is unexpected, please set it explicitly"
+        confirm
+    fi
+
+    if [[ "$MASTER_SSH_PORT" ]]; then
+        add_master_host_ssh_options master-ssh-port "$MASTER_SSH_PORT"
+    fi
+}
+
+function getmasterinfo() {
+    $sh_c "$INSTALL_OLARES_CLI node masterinfo $MASTER_SSH_OPTIONS" | tee /proc/$$/fd/1
+    if [[ $? -ne 0 ]]; then
+        exit 1
+    fi
+    echo "" > /proc/$$/fd/1
+}
+
+# check os type and arch
+os_type=$(uname -s)
+os_arch=$(uname -m)
+
+case "$os_arch" in
+    arm64) ARCH=arm64; ;;
+    x86_64) ARCH=amd64; ;;
+    armv7l) ARCH=arm; ;;
+    aarch64) ARCH=arm64; ;;
+    ppc64le) ARCH=ppc64le; ;;
+    s390x) ARCH=s390x; ;;
+    *) echo "error: unsupported arch \"$os_arch\"";
+    exit 1; ;;
+esac
+
+if [[ "$os_type" != "Linux" ]]; then
+    echo "error: only Linux machine can be added to the cluster"
+    exit 1
+fi
+
+# set shell execute command
+user="$(id -un 2>/dev/null || true)"
+sh_c='sh -c'
+if [ "$user" != 'root' ]; then
+    if ! command_exists sudo; then
+        echo "error: the ability to run as root is needed, but the command \"sudo\" can not be found"
+        exit 1
+    fi
+    sh_c='sudo -E sh -c'
+fi
+
+if ! command_exists tar; then
+    echo "error: the \"tar\" command is needed to unpack installation files, but can not be found"
+    exit 1
+fi
+
+BASE_DIR="$HOME/.olares"
+if [ ! -d $BASE_DIR ]; then
+    mkdir -p $BASE_DIR
+fi
+
+cdn_url=${DOWNLOAD_CDN_URL}
+if [[ -z "${cdn_url}" ]]; then
+    cdn_url="https://dc3p1870nn3cj.cloudfront.net"
+fi
+
+set_master_host_ssh_options
+
+CLI_VERSION="0.1.107"
+CLI_FILE="olares-cli-v${CLI_VERSION}_linux_${ARCH}.tar.gz"
+
+if command_exists olares-cli && [[ "$(olares-cli -v | awk '{print $3}')" == "$CLI_VERSION" ]]; then
+    INSTALL_OLARES_CLI=$(which olares-cli)
+    echo "olares-cli already installed and is the expected version"
+    echo ""
+else
+    if [[ ! -f ${CLI_FILE} ]]; then
+        CLI_URL="${cdn_url}/${CLI_FILE}"
+
+        echo "downloading Olares installer from ${CLI_URL} ..."
+        echo ""
+
+        curl -Lo ${CLI_FILE} ${CLI_URL}
+
+        if [[ $? -ne 0 ]]; then
+            echo "error: failed to download Olares installer"
+            exit 1
+        else
+            echo "Olares installer ${CLI_VERSION} download complete!"
+            echo ""
+        fi
+    fi
+    INSTALL_OLARES_CLI="/usr/local/bin/olares-cli"
+    echo "unpacking Olares installer to $INSTALL_OLARES_CLI..."
+    echo ""
+    tar -zxf ${CLI_FILE} olares-cli && chmod +x olares-cli
+    $sh_c "mv olares-cli $INSTALL_OLARES_CLI"
+
+    if [[ $? -ne 0 ]]; then
+        echo "error: failed to unpack Olares installer"
+        exit 1
+    fi
+fi
+
+echo "getting master info and checking current machine's eligibility to join the cluster"
+echo ""
+master_olares_version="$( getmasterinfo | grep OlaresVersion | awk '{print $2}' )"
+if [[ ! "$master_olares_version" ]]; then
+    echo "failed to fetch the version of Olares installed on master node"
+    exit 1
+fi
+PARAMS="--version $master_olares_version --base-dir $BASE_DIR"
+CDN="--download-cdn-url ${cdn_url}"
+
+if [[ -f $BASE_DIR/.prepared ]]; then
+    echo "file $BASE_DIR/.prepared detected, skip preparing phase"
+    echo ""
+    echo "please make sure the prepared Olares version is the same as the master, or there might be compatibility issues"
+    echo ""
+else
+    echo "running system prechecks ..."
+    echo ""
+    $sh_c "$INSTALL_OLARES_CLI olares precheck $PARAMS"
+    if [[ $? -ne 0 ]]; then
+        exit 1
+    fi
+
+    echo "downloading installation wizard..."
+    echo ""
+    $sh_c "$INSTALL_OLARES_CLI olares download wizard $PARAMS $CDN"
+    if [[ $? -ne 0 ]]; then
+        echo "error: failed to download installation wizard"
+        exit 1
+    fi
+
+    echo "downloading installation packages..."
+    echo ""
+    $sh_c "$INSTALL_OLARES_CLI olares download component $PARAMS $CDN"
+    if [[ $? -ne 0 ]]; then
+        echo "error: failed to download installation packages"
+        exit 1
+    fi
+
+    echo "preparing installation environment..."
+    echo ""
+    # env 'REGISTRY_MIRRORS' is a docker image cache mirrors, separated by commas
+    if [ x"$REGISTRY_MIRRORS" != x"" ]; then
+        extra="--registry-mirrors $REGISTRY_MIRRORS"
+    fi
+    $sh_c "$INSTALL_OLARES_CLI olares prepare $PARAMS $extra"
+    if [[ $? -ne 0 ]]; then
+        echo "error: failed to prepare installation environment"
+        exit 1
+    fi
+fi
+
+if [ -f $BASE_DIR/.installed ]; then
+    echo "file $BASE_DIR/.installed detected, skip installing"
+    echo "if it is left by an unclean uninstallation, please manually remove it and invoke the installer again"
+    exit 0
+fi
+
+echo "installing Kubernetes and joining Olares cluster..."
+echo ""
+$sh_c "$INSTALL_OLARES_CLI node add $PARAMS $MASTER_SSH_OPTIONS"
+
+if [[ $? -ne 0 ]]; then
+    echo "error: failed to install Olares"
+    exit 1
+fi

build/manifest/components

@@ -1,4 +1,4 @@
-olaresd-v0.0.50.tar.gz,pkg/components,https://dc3p1870nn3cj.cloudfront.net/olaresd-v0.0.50-linux-amd64.tar.gz,https://dc3p1870nn3cj.cloudfront.net/olaresd-v0.0.50-linux-arm64.tar.gz,olaresd
+olaresd-v0.0.57.tar.gz,pkg/components,https://dc3p1870nn3cj.cloudfront.net/olaresd-v0.0.57-linux-amd64.tar.gz,https://dc3p1870nn3cj.cloudfront.net/olaresd-v0.0.57-linux-arm64.tar.gz,olaresd
 socat-1.7.3.2.tar.gz,pkg/components,https://src.fedoraproject.org/lookaside/pkgs/socat/socat-1.7.3.2.tar.gz/sha512/540658b2a3d1b87673196282e5c62b97681bd0f1d1e4759ff9d72909d11060235ee9e9521a973603c1b00376436a9444248e5fbc0ffac65f8edb9c9bc28e7972/socat-1.7.3.2.tar.gz,https://src.fedoraproject.org/lookaside/pkgs/socat/socat-1.7.3.2.tar.gz/sha512/540658b2a3d1b87673196282e5c62b97681bd0f1d1e4759ff9d72909d11060235ee9e9521a973603c1b00376436a9444248e5fbc0ffac65f8edb9c9bc28e7972/socat-1.7.3.2.tar.gz,socat
 conntrack-tools-1.4.1.tar.gz,pkg/components,https://github.com/fqrouter/conntrack-tools/archive/refs/tags/conntrack-tools-1.4.1.tar.gz,https://github.com/fqrouter/conntrack-tools/archive/refs/tags/conntrack-tools-1.4.1.tar.gz,conntrack-tools
 minio.RELEASE.2023-05-04T21-44-30Z,pkg/components,https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-05-04T21-44-30Z,https://dl.min.io/server/minio/release/linux-arm64/archive/minio.RELEASE.2023-05-04T21-44-30Z,minio
@@ -14,8 +14,9 @@ ubuntu2204_cuda-keyring_1.1-1_all.deb,pkg/components,https://developer.download.
 ubuntu2204_cuda-keyring_1.0-1_all.deb,pkg/components,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.0-1_all.deb,ubuntu-22.04_cuda-keyring_1.0-1
 ubuntu2004_cuda-keyring_1.1-1_all.deb,pkg/components,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/arm64/cuda-keyring_1.1-1_all.deb,ubuntu-20.04_cuda-keyring_1.1-1
 ubuntu2004_cuda-keyring_1.0-1_all.deb,pkg/components,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/arm64/cuda-keyring_1.0-1_all.deb,ubuntu-20.04_cuda-keyring_1.0-1
+debian12_cuda-keyring_1.1-1_all.deb,pkg/components,https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb,debian-12_cuda-keyring_1.1-1
+debian11_cuda-keyring_1.1-1_all.deb,pkg/components,https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/cuda-keyring_1.1-1_all.deb,https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb,debian-11_cuda-keyring_1.1-1
 
-gpgkey,pkg/components,https://nvidia.github.io/libnvidia-container/gpgkey,https://nvidia.github.io/libnvidia-container/gpgkey,gpgkey
-ubuntu_22.04_libnvidia-container.list,pkg/components,https://nvidia.github.io/libnvidia-container/ubuntu22.04/libnvidia-container.list,https://nvidia.github.io/libnvidia-container/ubuntu22.04/libnvidia-container.list,ubuntu_22.04_libnvidia-container.list
-ubuntu_20.04_libnvidia-container.list,pkg/components,https://nvidia.github.io/libnvidia-container/ubuntu20.04/libnvidia-container.list,https://nvidia.github.io/libnvidia-container/ubuntu20.04/libnvidia-container.list,ubuntu_20.04_libnvidia-container.list
+libnvidia-gpgkey,pkg/components,https://nvidia.github.io/libnvidia-container/gpgkey,https://nvidia.github.io/libnvidia-container/gpgkey,libnvidia-gpgkey
+libnvidia-container.list,pkg/components,https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list,https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list,libnvidia-container.list
 

build/manifest/dependencies.amd64

@@ -1,53 +0,0 @@
-[components] format: url,filename
-https://github.com/beclab/Installer/releases/download/0.1.13/terminus-cli-v0.1.13_linux_amd64.tar.gz,terminus-cli-v0.1.13_linux_amd64.tar.gz
-https://src.fedoraproject.org/lookaside/pkgs/socat/socat-1.7.3.2.tar.gz/sha512/540658b2a3d1b87673196282e5c62b97681bd0f1d1e4759ff9d72909d11060235ee9e9521a973603c1b00376436a9444248e5fbc0ffac65f8edb9c9bc28e7972/socat-1.7.3.2.tar.gz,socat-1.7.3.2.tar.gz
-
-https://github.com/fqrouter/conntrack-tools/archive/refs/tags/conntrack-tools-1.4.1.tar.gz,conntrack-tools-1.4.1.tar.gz
-
-https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-05-04T21-44-30Z,minio.RELEASE.2023-05-04T21-44-30Z
-https://github.com/beclab/minio-operator/releases/download/v0.0.1/minio-operator-v0.0.1-linux-amd64.tar.gz,minio-operator-v0.0.1-linux-amd64.tar.gz
-
-https://download.redis.io/releases/redis-5.0.14.tar.gz,redis-5.0.14.tar.gz
-
-https://github.com/beclab/juicefs-ext/releases/download/v11.1.1/juicefs-v11.1.1-linux-amd64.tar.gz,juicefs-v11.1.1-linux-amd64.tar.gz
-
-https://github.com/beclab/velero/releases/download/v1.11.3/velero-v1.11.3-linux-amd64.tar.gz,velero-v1.11.3-linux-amd64.tar.gz
-
-https://launchpad.net/ubuntu/+source/apparmor/4.0.1-0ubuntu1/+build/28428840/+files/apparmor_4.0.1-0ubuntu1_amd64.deb,apparmor_4.0.1-0ubuntu1_amd64.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb,ubuntu_24.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb,ubuntu2404_cuda-keyring_1.1-1_all.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb,ubuntu_22.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb,ubuntu_22.04_cuda-keyring_1.0-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb,ubuntu2204_cuda-keyring_1.0-1_all.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb,ubuntu_20.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb,ubuntu_20.04_cuda-keyring_1.0-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb,ubuntu2004_cuda-keyring_1.0-1_all.deb
-https://nvidia.github.io/libnvidia-container/gpgkey,gpgkey
-https://nvidia.github.io/libnvidia-container/ubuntu22.04/libnvidia-container.list,ubuntu_22.04_libnvidia-container.list
-https://nvidia.github.io/libnvidia-container/ubuntu20.04/libnvidia-container.list,ubuntu_20.04_libnvidia-container.list
-
-
-[pkg] format: url,path,filename,special,cpname
-https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz,cni/v0.9.1,,,
-
-https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz,cni/v1.1.1,,,
-
-https://github.com/containerd/containerd/releases/download/v1.6.4/containerd-1.6.4-linux-amd64.tar.gz,containerd/1.6.4,,,
-
-https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.0/crictl-v1.24.0-linux-amd64.tar.gz,crictl/v1.24.0,,,
-
-https://github.com/coreos/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz,etcd/v3.4.13,,,
-
-https://get.helm.sh/helm-v3.9.0-linux-amd64.tar.gz,helm/v3.9.0,,helm,helm-v3.9.0
-
-https://github.com/k3s-io/k3s/releases/download/v1.21.5+k3s1/k3s,kube/v1.21.5,,,k3s-v1.21.5
-
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/amd64/kubeadm,kube/v1.22.10,,kubeadm,kubeadm-v1.22.10
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/amd64/kubelet,kube/v1.22.10,,kubelet,kubelet-v1.22.10
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/amd64/kubectl,kube/v1.22.10,,kubectl,kubectl-v1.22.10
-
-https://github.com/opencontainers/runc/releases/download/v1.1.1/runc.amd64,runc/v1.1.1,,,runc-v1.1.1
-https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64,runc/v1.1.4,,,runc-v1.1.4

build/manifest/dependencies.arm64

@@ -1,53 +0,0 @@
-[components] format: url,filename
-https://github.com/beclab/Installer/releases/download/0.1.13/terminus-cli-v0.1.13_linux_amd64.tar.gz,terminus-cli-v0.1.13_linux_amd64.tar.gz
-https://src.fedoraproject.org/lookaside/pkgs/socat/socat-1.7.3.2.tar.gz/sha512/540658b2a3d1b87673196282e5c62b97681bd0f1d1e4759ff9d72909d11060235ee9e9521a973603c1b00376436a9444248e5fbc0ffac65f8edb9c9bc28e7972/socat-1.7.3.2.tar.gz,socat-1.7.3.2.tar.gz
-
-https://github.com/fqrouter/conntrack-tools/archive/refs/tags/conntrack-tools-1.4.1.tar.gz,conntrack-tools-1.4.1.tar.gz
-
-https://dl.min.io/server/minio/release/linux-arm64/archive/minio.RELEASE.2023-05-04T21-44-30Z,
-https://github.com/beclab/minio-operator/releases/download/v0.0.1/minio-operator-v0.0.1-linux-arm64.tar.gz,minio-operator-v0.0.1-linux-arm64.tar.gz
-
-https://download.redis.io/releases/redis-5.0.14.tar.gz,redis-5.0.14.tar.gz
-
-https://github.com/beclab/juicefs-ext/releases/download/v11.1.1/juicefs-v11.1.1-linux-arm64.tar.gz,juicefs-v11.1.1-linux-arm64.tar.gz
-
-https://github.com/beclab/velero/releases/download/v1.11.3/velero-v1.11.3-linux-arm64.tar.gz,velero-v1.11.3-linux-arm64.tar.gz
-
-https://launchpad.net/ubuntu/+source/apparmor/4.0.1-0ubuntu1/+build/28428841/+files/apparmor_4.0.1-0ubuntu1_arm64.deb,apparmor_4.0.1-0ubuntu1_arm64.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/arm64/cuda-keyring_1.1-1_all.deb,ubuntu_24.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/arm64/cuda-keyring_1.1-1_all.deb,ubuntu2404_cuda-keyring_1.1-1_all.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb,ubuntu_22.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.0-1_all.deb,ubuntu_22.04_cuda-keyring_1.0-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.0-1_all.deb,ubuntu2204_cuda-keyring_1.0-1_all.deb
-
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/arm64/cuda-keyring_1.1-1_all.deb,ubuntu_20.04_cuda-keyring_1.1-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/arm64/cuda-keyring_1.0-1_all.deb,ubuntu_20.04_cuda-keyring_1.0-1_all.deb
-https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/arm64/cuda-keyring_1.0-1_all.deb,ubuntu2004_cuda-keyring_1.0-1_all.deb
-
-https://nvidia.github.io/libnvidia-container/gpgkey,gpgkey
-https://nvidia.github.io/libnvidia-container/ubuntu22.04/libnvidia-container.list,ubuntu_22.04_libnvidia-container.list
-https://nvidia.github.io/libnvidia-container/ubuntu20.04/libnvidia-container.list,ubuntu_20.04_libnvidia-container.list
-
-
-[pkg] format: url,path,filename,special,cpname
-https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz,cni/v0.9.1,,
-https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-arm64-v1.1.1.tgz,cni/v1.1.1,,
-
-https://github.com/containerd/containerd/releases/download/v1.6.4/containerd-1.6.4-linux-arm64.tar.gz,containerd/1.6.4,,
-
-https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.0/crictl-v1.24.0-linux-arm64.tar.gz,crictl/v1.24.0,,
-
-https://github.com/coreos/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-arm64.tar.gz,etcd/v3.4.13,,
-
-https://get.helm.sh/helm-v3.9.0-linux-arm64.tar.gz,helm/v3.9.0,,helm,helm-v3.9.0
-
-https://github.com/k3s-io/k3s/releases/download/v1.21.5+k3s1/k3s-arm64,kube/v1.21.5,,,k3s-v1.21.5
-
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/arm64/kubeadm,kube/v1.22.10,,kubeadm,kubeadm-v1.22.10
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/arm64/kubelet,kube/v1.22.10,,kubelet,kubelet-v1.22.10
-https://storage.googleapis.com/kubernetes-release/release/v1.22.10/bin/linux/arm64/kubectl,kube/v1.22.10,,kubectl,kubectl-v1.22.10
-
-https://github.com/opencontainers/runc/releases/download/v1.1.1/runc.arm64,runc/v1.1.1,,,runc-v1.1.1
-https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64,runc/v1.1.4,,,runc-v1.1.4

build/manifest/images

@@ -58,7 +58,7 @@ gcr.io/k8s-minikube/storage-provisioner:v5
 owncloudci/wait-for:latest
 beclab/recommend-argotask:v0.0.12
 nvcr.io/nvidia/k8s-device-plugin:v0.16.1
-beclab/nvshare:libnvshare-v0.0.2
+beclab/nvshare:libnvshare-v0.0.1
 bytetrade/nvshare:nvshare-device-plugin
 bytetrade/nvshare:nvshare-scheduler
 beclab/nats-server-config-reloader:v1

frameworks/app-service/config/cluster/deploy/appservice_deploy.yaml

@@ -146,9 +146,10 @@ spec:
     spec:
       serviceAccountName: os-internal
       serviceAccount: os-internal
+      priorityClassName: "system-cluster-critical"
       containers:
       - name: app-service
-        image: beclab/app-service:0.2.58
+        image: beclab/app-service:0.2.73
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 0
@@ -360,7 +361,7 @@ spec:
       hostNetwork: true
       containers:
         - name: image-service
-          image: beclab/image-service:0.2.51
+          image: beclab/image-service:0.2.66
           imagePullPolicy: IfNotPresent
           securityContext:
             runAsUser: 0

frameworks/bfl/config/launcher/templates/bfl_deploy.yaml

@@ -215,6 +215,7 @@ spec:
             weight: 10
 {{ end }}            
       serviceAccountName: bytetrade-controller
+      priorityClassName: "system-cluster-critical"
       initContainers:
       - name: init-userspace
         image: busybox:1.28
@@ -242,7 +243,7 @@ spec:
 
       containers:
       - name: api
-        image: beclab/bfl:v0.3.59
+        image: beclab/bfl:v0.3.63
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 1000
@@ -295,7 +296,7 @@ spec:
           value: {{ .Values.bfl.terminus_dns_service_api }}
 
       - name: ingress
-        image: beclab/bfl-ingress:v0.2.18
+        image: beclab/bfl-ingress:v0.2.19
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: ngxlog

frameworks/system-server/config/user/helm-charts/systemserver/templates/systemserver_deploy.yaml

@@ -44,6 +44,7 @@ spec:
     spec:
       serviceAccountName: bytetrade-sys-ops
       serviceAccount: bytetrade-sys-ops
+      priorityClassName: "system-cluster-critical"
       containers:
       - name: system-server
         image: beclab/system-server:0.1.19

frameworks/tapr/config/cluster/deploy/middleware_deploy.yaml

@@ -99,7 +99,7 @@ spec:
         - name: DISABLE_TELEMETRY
           value: "false"
       - name: operator-api
-        image: beclab/middleware-operator:0.1.37
+        image: beclab/middleware-operator:0.1.38
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9080

frameworks/tapr/config/cluster/deploy/nats_deployment.yaml

@@ -247,6 +247,24 @@ spec:
         app.kubernetes.io/name: nats
         app.kubernetes.io/instance: nats
     spec:
+      initContainers:
+        - name: generate-config
+          image: busybox:1.28
+          command:
+            - sh
+            - -c
+            - |
+              if [ ! -f /data/config/nats.conf ]; then
+                cat /etc/nats-config/nats.conf > /data/config/nats.conf
+              else
+                echo "nats config file already exists"
+              fi
+          volumeMounts:
+            - mountPath: /etc/nats-config
+              name: config
+              readOnly: false
+            - mountPath: /data
+              name: nats-data
       containers:
         - args:
             - --config

third-party/authelia/config/cluster/deploy/auth_backend_deploy.yaml

@@ -306,6 +306,7 @@ spec:
     spec:
       serviceAccountName: os-internal
       serviceAccount: os-internal
+      priorityClassName: "system-cluster-critical"      
       initContainers:
       - name: init-container
         image: 'postgres:16.0-alpine3.18'
@@ -423,6 +424,7 @@ spec:
       labels:
         app: redis
     spec:
+      priorityClassName: "system-cluster-critical"
       containers:
       - name: redis
         image: redis:6.2.13-alpine3.18

third-party/authelia/config/user/helm-charts/auth/templates/auth_deploy.yaml

@@ -28,7 +28,7 @@ spec:
         name: check-auth
       containers:
       - name: auth-front
-        image: beclab/login:v0.1.33
+        image: beclab/login:v0.1.34
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 80

third-party/headscale/config/user/helm-charts/headscale/templates/headscale_deploy.yaml

@@ -1,4 +1,42 @@
+{{- $namespace := printf "%s%s" "user-system-" .Values.bfl.username -}}
+{{- $headscale_secret := (lookup "v1" "Secret" $namespace "headscale-secrets") -}}
 
+{{- $pg_password := "" -}}
+{{ if $headscale_secret -}}
+{{ $pg_password = (index $headscale_secret "data" "pg_password") }}
+{{ else -}}
+{{ $pg_password = randAlphaNum 16 | b64enc }}
+{{- end -}}
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: headscale-secrets
+  namespace: user-system-{{ .Values.bfl.username }}
+type: Opaque
+data:
+  pg_password: {{ $pg_password }}
+
+---
+apiVersion: apr.bytetrade.io/v1alpha1
+kind: MiddlewareRequest
+metadata:
+  name: headscale-pg
+  namespace: user-system-{{ .Values.bfl.username }}
+spec:
+  app: headscale
+  appNamespace: {{ .Release.Namespace }}
+  middleware: postgres
+  postgreSQL:
+    user: headscale_{{ .Values.bfl.username }}
+    password:
+      valueFrom:
+        secretKeyRef:
+          key: pg_password
+          name: headscale-secrets
+    databases:
+    - name: headscale
 
 ---
 apiVersion: v1
@@ -36,8 +74,6 @@ spec:
   selector:
     matchLabels:
       app: headscale
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -68,7 +104,7 @@ spec:
         - |
           chown -R 1000:1000 /headscale 
       - name: init
-        image: beclab/headscale-init:v0.1.7
+        image: beclab/headscale-init:v0.1.9
         imagePullPolicy: IfNotPresent
         securityContext:
           privileged: true
@@ -79,9 +115,39 @@ spec:
         {{- end }}
         - name: NAMESPACE
           value: bfl.user-space-{{ .Values.bfl.username }}
+        - name: PG_HOST
+          value: citus-master-svc.user-system-{{ .Values.bfl.username }}
+        - name: PG_PORT
+          value: "5432"
+        - name: PG_USER
+          value: headscale_{{ .Values.bfl.username }}
+        - name: PG_PASS
+          value: "{{ $pg_password | b64dec }}"
+        - name: PG_DB
+          value: user_space_{{ .Values.bfl.username }}_headscale
         volumeMounts:
         - name: config
           mountPath: /etc/headscale
+      - name: wait-for-postgres
+        image: postgres:16.0-alpine3.18
+        command:
+          - sh
+          - '-c'
+          - >-
+            echo -e "Checking for the availability of PostgreSQL Server deployment"; until psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDB
+            -c "SELECT 1"; do sleep 1; printf "-"; done; echo -e " >> PostgreSQL DB Server has started";
+        env:
+        - name: PGHOST
+          value: citus-master-svc.user-system-{{ .Values.bfl.username }}
+        - name: PGPORT
+          value: "5432"
+        - name: PGUSER
+          value: headscale_{{ .Values.bfl.username }}
+        - name: PGPASSWORD
+          value: "{{ $pg_password | b64dec }}"
+        - name: PGDB
+          value: user_space_{{ .Values.bfl.username }}_headscale
+        imagePullPolicy: IfNotPresent
       containers:
       - name: headscale
         image: headscale/headscale:0.22.3
@@ -109,6 +175,9 @@ spec:
           mountPath: /etc/headscale
         - name: headscale-data
           mountPath: /var/lib/headscale
+        - name: acl-config
+          mountPath: /etc/headscale/acl
+          readOnly: true
         ports:
         - containerPort: 8080
       - args:
@@ -141,6 +210,13 @@ spec:
         hostPath:
           type: DirectoryOrCreate
           path: {{ .Values.userspace.appCache  }}/headscale
+      - name: acl-config
+        configMap:
+          defaultMode: 420
+          items:
+          - key: acl.json
+            path: acl.json
+          name: tailscale-acl
 
 ---
 apiVersion: apps/v1
@@ -198,7 +274,7 @@ spec:
         - name: TS_STATE_DIR
           value: "/var/lib/tailscale/"
         - name: TS_TAILSCALED_EXTRA_ARGS
-          value: "--no-logs-no-support --verbose=1"
+          value: "--no-logs-no-support  --verbose=1"
         - name: TS_ROUTES
           value: $(NODE_IP)/32
         - name: TS_EXTRA_ARGS
@@ -283,3 +359,26 @@ spec:
   version: v1
 status:
   state: active
+
+---
+
+apiVersion: v1
+data:
+  acl.json: |
+    {
+      "acls":[
+        { "action": "accept", "src": ["*"], "proto": "tcp", "dst": ["*:443"] }
+      ],
+      "autoApprovers": {
+        "routes": {
+          "10.0.0.0/8": ["default"],
+          "172.16.0.0/12": ["default"],
+          "192.168.0.0/16": ["default"]
+        },
+        "exitNode": []
+      }
+    }
+kind: ConfigMap
+metadata:
+  name: tailscale-acl
+  namespace: user-space-{{ .Values.bfl.username }}

third-party/infisical/config/user/helm-charts/infisical/templates/infisical_deploy.yaml

@@ -192,6 +192,7 @@ spec:
         io.bytetrade.app: "true"
     spec:
       serviceAccountName: infisical-sa
+      priorityClassName: "system-cluster-critical"
       initContainers:
       - name: init-container
         image: 'postgres:16.0-alpine3.18'

third-party/seahub/config/cluster/deploy/seafile_deploy.yaml

@@ -165,7 +165,7 @@ data:
 #    end
   nginx.conf: |-
     user  nginx;
-    worker_processes  auto;
+    worker_processes  4;
 
     error_log  /var/log/nginx/error.log notice;
     pid        /var/run/nginx.pid;