761 lines
27 KiB
Go
761 lines
27 KiB
Go
/*
|
|
Copyright 2021 The KubeSphere Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package kubernetes
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/beclab/Olares/cli/pkg/storage"
|
|
storagetpl "github.com/beclab/Olares/cli/pkg/storage/templates"
|
|
|
|
"github.com/beclab/Olares/cli/pkg/etcd"
|
|
"github.com/beclab/Olares/cli/pkg/manifest"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
kubekeyv1alpha2 "github.com/beclab/Olares/cli/apis/kubekey/v1alpha2"
|
|
"github.com/beclab/Olares/cli/pkg/common"
|
|
"github.com/beclab/Olares/cli/pkg/core/action"
|
|
"github.com/beclab/Olares/cli/pkg/core/connector"
|
|
"github.com/beclab/Olares/cli/pkg/core/logger"
|
|
"github.com/beclab/Olares/cli/pkg/core/util"
|
|
"github.com/beclab/Olares/cli/pkg/images"
|
|
"github.com/beclab/Olares/cli/pkg/kubernetes/templates"
|
|
"github.com/beclab/Olares/cli/pkg/kubernetes/templates/v1beta2"
|
|
"github.com/beclab/Olares/cli/pkg/utils"
|
|
)
|
|
|
|
type GetKubeType struct{}
|
|
|
|
func (g *GetKubeType) Execute() (kubeType string) {
|
|
kubeType = common.K3s
|
|
var getKubeVersion = new(GetKubeVersion)
|
|
_, kubeType, _ = getKubeVersion.Execute()
|
|
|
|
if kubeType != "" {
|
|
return
|
|
}
|
|
|
|
if util.IsExist("/etc/systemd/system/k3s.service") || util.IsExist("/usr/local/bin/k3s-uninstall.sh") {
|
|
kubeType = common.K3s
|
|
} else {
|
|
kubeType = common.K8s
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
type GetKubeVersion struct{}
|
|
|
|
func (g *GetKubeVersion) Execute() (string, string, error) {
|
|
var kubectlpath, err = util.GetCommand(common.CommandKubectl)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("kubectl not found, Olares might not be installed.")
|
|
}
|
|
|
|
var ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
cmd := exec.CommandContext(ctx, "/bin/sh", "-c", fmt.Sprintf("%s get nodes -l node-role.kubernetes.io/master -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}'", kubectlpath))
|
|
// the kubectl command may continue running after the context has timed out
|
|
// causing the cmd.Wait() to block for a long time
|
|
cmd.WaitDelay = 3 * time.Second
|
|
output, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return "", "", errors.Wrap(errors.WithStack(err), "get kube version failed")
|
|
}
|
|
|
|
if output == nil || len(output) == 0 {
|
|
return "", "", fmt.Errorf("get kube version failed")
|
|
}
|
|
|
|
var version = string(output)
|
|
var kubeVersion, kubeType = utils.KubeVersionAlias(version)
|
|
|
|
return kubeVersion, kubeType, nil
|
|
}
|
|
|
|
type GetClusterStatus struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (g *GetClusterStatus) Execute(runtime connector.Runtime) error {
|
|
exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/admin.conf")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !exist {
|
|
g.PipelineCache.Set(common.ClusterExist, false)
|
|
return nil
|
|
} else {
|
|
g.PipelineCache.Set(common.ClusterExist, true)
|
|
|
|
if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok {
|
|
cluster := v.(*KubernetesStatus)
|
|
if err := cluster.SearchVersion(runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := cluster.SearchKubeConfig(runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil {
|
|
return err
|
|
}
|
|
if err := cluster.SearchClusterInfo(runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := cluster.SearchNodesInfo(runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := cluster.SearchJoinInfo(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
g.PipelineCache.Set(common.ClusterStatus, cluster)
|
|
} else {
|
|
return errors.New("get kubernetes cluster status by pipeline cache failed")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type SyncKubeBinary struct {
|
|
common.KubeAction
|
|
manifest.ManifestAction
|
|
}
|
|
|
|
func (i *SyncKubeBinary) Execute(runtime connector.Runtime) error {
|
|
if err := utils.ResetTmpDir(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
binaryList := []string{"kubeadm", "kubelet", "kubectl", "helm", "cni-plugins"}
|
|
for _, name := range binaryList {
|
|
binary, err := i.Manifest.Get(name)
|
|
if err != nil {
|
|
return fmt.Errorf("get kube binary %s info failed: %w", name, err)
|
|
}
|
|
|
|
path := binary.FilePath(i.BaseDir)
|
|
|
|
fileName := binary.Filename
|
|
switch name {
|
|
//case "kubelet":
|
|
// if err := runtime.GetRunner().Scp(binary.Path, fmt.Sprintf("%s/%s", common.TmpDir, binary.Name)); err != nil {
|
|
// return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed"))
|
|
// }
|
|
case "cni-plugins":
|
|
dst := filepath.Join(common.TmpDir, fileName)
|
|
logger.Debugf("SyncKubeBinary cp %s from %s to %s", name, path, dst)
|
|
if err := runtime.GetRunner().Scp(path, dst); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed"))
|
|
}
|
|
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false, false); err != nil {
|
|
return err
|
|
}
|
|
case "helm":
|
|
dst := filepath.Join(common.TmpDir, fileName)
|
|
untarDst := filepath.Join(common.TmpDir, strings.TrimSuffix(fileName, ".tar.gz"))
|
|
logger.Debugf("SyncKubeBinary cp %s from %s to %s", name, path, dst)
|
|
if err := runtime.GetRunner().Scp(path, dst); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed"))
|
|
}
|
|
|
|
cmd := fmt.Sprintf("mkdir -p %s && tar -zxf %s -C %s && cd %s/linux-* && mv ./helm /usr/local/bin/.",
|
|
untarDst, dst, untarDst, untarDst)
|
|
if _, err := runtime.GetRunner().SudoCmd(cmd, false, false); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
dst := filepath.Join(common.BinDir, name)
|
|
if err := runtime.GetRunner().SudoScp(path, dst); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed"))
|
|
}
|
|
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false, false); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type SyncKubelet struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (s *SyncKubelet) Execute(runtime connector.Runtime) error {
|
|
if _, err := runtime.GetRunner().SudoCmd("chmod +x /usr/local/bin/kubelet", false, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "sync kubelet service failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type GenerateKubeletService struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (t *GenerateKubeletService) Execute(runtime connector.Runtime) error {
|
|
tplActions := &action.Template{
|
|
Name: "GenerateKubeletService",
|
|
Template: templates.KubeletService,
|
|
Dst: filepath.Join("/etc/systemd/system/", templates.KubeletService.Name()),
|
|
Data: util.Data{
|
|
"JuiceFSPreCheckEnabled": util.IsExist(storage.JuiceFsServiceFile),
|
|
"JuiceFSServiceUnit": storagetpl.JuicefsService.Name(),
|
|
"JuiceFSBinPath": storage.JuiceFsFile,
|
|
"JuiceFSMountPoint": storage.OlaresJuiceFSRootDir,
|
|
},
|
|
}
|
|
return tplActions.Execute(runtime)
|
|
}
|
|
|
|
type EnableKubelet struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (e *EnableKubelet) Execute(runtime connector.Runtime) error {
|
|
if _, err := runtime.GetRunner().SudoCmd("systemctl disable kubelet "+
|
|
"&& systemctl enable kubelet "+
|
|
"&& ln -snf /usr/local/bin/kubelet /usr/bin/kubelet", false, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "enable kubelet service failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type GenerateKubeletEnv struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (g *GenerateKubeletEnv) Execute(runtime connector.Runtime) error {
|
|
host := runtime.RemoteHost()
|
|
templateAction := action.Template{
|
|
Name: "GenerateKubeletEnv",
|
|
Template: templates.KubeletEnv,
|
|
Dst: filepath.Join("/etc/systemd/system/kubelet.service.d", templates.KubeletEnv.Name()),
|
|
Data: util.Data{
|
|
"NodeIP": host.GetInternalAddress(),
|
|
"Hostname": host.GetName(),
|
|
"ContainerRuntime": "",
|
|
},
|
|
}
|
|
|
|
templateAction.Init(nil, nil)
|
|
if err := templateAction.Execute(runtime); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type GenerateKubeadmConfig struct {
|
|
common.KubeAction
|
|
IsInitConfiguration bool
|
|
WithSecurityEnhancement bool
|
|
}
|
|
|
|
func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
|
|
host := runtime.RemoteHost()
|
|
|
|
localConfig := filepath.Join(runtime.GetWorkDir(), "kubeadm-config.yaml")
|
|
if util.IsExist(localConfig) {
|
|
// todo: if it is necessary?
|
|
if err := runtime.GetRunner().SudoScp(localConfig, "/etc/kubernetes/kubeadm-config.yaml"); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "scp local kubeadm config failed")
|
|
}
|
|
} else {
|
|
// generate etcd configuration
|
|
var externalEtcd kubekeyv1alpha2.ExternalEtcd
|
|
var endpointsList, etcdCertSANs []string
|
|
|
|
switch g.KubeConf.Cluster.Etcd.Type {
|
|
case kubekeyv1alpha2.KubeKey:
|
|
for _, host := range runtime.GetHostsByRole(common.ETCD) {
|
|
endpoint := fmt.Sprintf("https://%s:%s", host.GetInternalAddress(), kubekeyv1alpha2.DefaultEtcdPort)
|
|
endpointsList = append(endpointsList, endpoint)
|
|
}
|
|
externalEtcd.Endpoints = endpointsList
|
|
|
|
externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem"
|
|
externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", host.GetName())
|
|
externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", host.GetName())
|
|
case kubekeyv1alpha2.External:
|
|
externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints
|
|
|
|
if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 {
|
|
externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile))
|
|
externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile))
|
|
externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile))
|
|
}
|
|
case kubekeyv1alpha2.Kubeadm:
|
|
altNames := etcd.GenerateAltName(g.KubeConf, &runtime)
|
|
etcdCertSANs = append(etcdCertSANs, altNames.DNSNames...)
|
|
for _, ip := range altNames.IPs {
|
|
etcdCertSANs = append(etcdCertSANs, string(ip))
|
|
}
|
|
}
|
|
|
|
_, ApiServerArgs := util.GetArgs(v1beta2.GetApiServerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
|
|
_, ControllerManagerArgs := util.GetArgs(v1beta2.GetControllermanagerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
|
|
_, SchedulerArgs := util.GetArgs(v1beta2.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
|
|
|
|
checkCgroupDriver, err := v1beta2.GetKubeletCgroupDriver(runtime, g.KubeConf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var (
|
|
bootstrapToken, certificateKey string
|
|
// todo: if port needed
|
|
)
|
|
if !g.IsInitConfiguration {
|
|
if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok {
|
|
cluster := v.(*KubernetesStatus)
|
|
bootstrapToken = cluster.BootstrapToken
|
|
certificateKey = cluster.CertificateKey
|
|
} else {
|
|
return errors.New("get kubernetes cluster status by pipeline cache failed")
|
|
}
|
|
}
|
|
|
|
v1beta2.AdjustDefaultFeatureGates(g.KubeConf)
|
|
templateAction := action.Template{
|
|
Name: "GenerateKubeadmConfig",
|
|
Template: v1beta2.KubeadmConfig,
|
|
Dst: filepath.Join(common.KubeConfigDir, v1beta2.KubeadmConfig.Name()),
|
|
Data: util.Data{
|
|
"IsInitCluster": g.IsInitConfiguration,
|
|
"EtcdTypeIsKubeadm": g.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.Kubeadm,
|
|
"EtcdCertSANs": etcdCertSANs,
|
|
"EtcdRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "etcd").ImageRepo(), "/etcd"),
|
|
"EtcdTag": images.GetImage(runtime, g.KubeConf, "etcd").Tag,
|
|
"Version": g.KubeConf.Cluster.Kubernetes.Version,
|
|
"ClusterName": g.KubeConf.Cluster.Kubernetes.ClusterName,
|
|
"DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain,
|
|
"AdvertiseAddress": host.GetInternalAddress(),
|
|
"BindPort": kubekeyv1alpha2.DefaultApiserverPort,
|
|
"ControlPlaneEndpoint": fmt.Sprintf("%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port),
|
|
"PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR,
|
|
"ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR,
|
|
"CertSANs": g.KubeConf.Cluster.GenerateCertSANs(),
|
|
"ExternalEtcd": externalEtcd,
|
|
"NodeCidrMaskSize": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
|
|
"CriSock": g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint,
|
|
"ApiServerArgs": v1beta2.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
|
|
"ControllerManagerArgs": v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
|
|
"SchedulerArgs": v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
|
|
"KubeletConfiguration": v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
|
|
"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf, runtime.GetSystemInfo().IsPveLxc()),
|
|
"IsControlPlane": host.IsRole(common.Master),
|
|
"CgroupDriver": checkCgroupDriver,
|
|
"BootstrapToken": bootstrapToken,
|
|
"CertificateKey": certificateKey,
|
|
},
|
|
}
|
|
|
|
templateAction.Init(nil, nil)
|
|
if err := templateAction.Execute(runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type KubeadmInit struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (k *KubeadmInit) Execute(runtime connector.Runtime) error {
|
|
initCmd := "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull"
|
|
|
|
if k.KubeConf.Cluster.Kubernetes.DisableKubeProxy {
|
|
initCmd = initCmd + " --skip-phases=addon/kube-proxy"
|
|
}
|
|
|
|
// we manage the creation of coredns ourselves
|
|
initCmd = initCmd + " --skip-phases=addon/coredns"
|
|
|
|
if _, err := runtime.GetRunner().SudoCmd(initCmd, false, true); err != nil {
|
|
// kubeadm reset and then retry
|
|
resetCmd := "/usr/local/bin/kubeadm reset -f"
|
|
if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" {
|
|
resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint
|
|
}
|
|
_, _ = runtime.GetRunner().SudoCmd(resetCmd, false, true)
|
|
return errors.Wrap(errors.WithStack(err), "init kubernetes cluster failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type CopyKubeConfigForControlPlane struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (c *CopyKubeConfigForControlPlane) Execute(runtime connector.Runtime) error {
|
|
targetHome, targetUID, targetGID, err := utils.ResolveSudoUserHomeAndIDs(runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cmds := []string{
|
|
"mkdir -p /root/.kube",
|
|
"cp -f /etc/kubernetes/admin.conf /root/.kube/config",
|
|
"chmod 0600 /root/.kube/config",
|
|
fmt.Sprintf("mkdir -p %s", filepath.Join(targetHome, ".kube")),
|
|
fmt.Sprintf("cp -f /etc/kubernetes/admin.conf %s", filepath.Join(targetHome, ".kube", "config")),
|
|
fmt.Sprintf("chmod 0600 %s", filepath.Join(targetHome, ".kube", "config")),
|
|
fmt.Sprintf("chown -R %s:%s %s", targetUID, targetGID, filepath.Join(targetHome, ".kube")),
|
|
}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(cmds, " && "), false, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "copy kube config failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type RemoveMasterTaint struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (r *RemoveMasterTaint) Execute(runtime connector.Runtime) error {
|
|
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf(
|
|
"/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=:NoSchedule-",
|
|
runtime.RemoteHost().GetName()), false, true); err != nil {
|
|
logger.Warn(err.Error())
|
|
}
|
|
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf(
|
|
"/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/control-plane=:NoSchedule-",
|
|
runtime.RemoteHost().GetName()), false, true); err != nil {
|
|
logger.Warn(err.Error())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type AddWorkerLabel struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error {
|
|
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf(
|
|
"/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=",
|
|
runtime.RemoteHost().GetName()), false, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "add worker label failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type JoinNode struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (j *JoinNode) Execute(runtime connector.Runtime) error {
|
|
if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull",
|
|
true, false); err != nil {
|
|
resetCmd := "/usr/local/bin/kubeadm reset -f"
|
|
if j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" {
|
|
resetCmd = resetCmd + " --cri-socket " + j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint
|
|
}
|
|
_, _ = runtime.GetRunner().SudoCmd(resetCmd, true, false)
|
|
return errors.Wrap(errors.WithStack(err), "join node failed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type SyncKubeConfigToWorker struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (s *SyncKubeConfigToWorker) Execute(runtime connector.Runtime) error {
|
|
if v, ok := s.PipelineCache.Get(common.ClusterStatus); ok {
|
|
cluster := v.(*KubernetesStatus)
|
|
|
|
targetHome, targetUID, targetGID, err := utils.ResolveSudoUserHomeAndIDs(runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
targetKubeConfigPath := filepath.Join(targetHome, ".kube", "config")
|
|
|
|
cmds := []string{
|
|
"mkdir -p /root/.kube",
|
|
fmt.Sprintf("echo '%s' > %s", cluster.KubeConfig, "/root/.kube/config"),
|
|
"chmod 0600 /root/.kube/config",
|
|
fmt.Sprintf("mkdir -p %s", filepath.Join(targetHome, ".kube")),
|
|
fmt.Sprintf("echo '%s' > %s", cluster.KubeConfig, targetKubeConfigPath),
|
|
fmt.Sprintf("chmod 0600 %s", targetKubeConfigPath),
|
|
fmt.Sprintf("chown -R %s:%s %s", targetUID, targetGID, filepath.Join(targetHome, ".kube")),
|
|
}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(cmds, " && "), false, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "sync kube config failed")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type KubeadmReset struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (k *KubeadmReset) Execute(runtime connector.Runtime) error {
|
|
resetCmd := "/usr/local/bin/kubeadm reset -f"
|
|
if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" {
|
|
resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint
|
|
}
|
|
_, _ = runtime.GetRunner().SudoCmd(resetCmd, false, true)
|
|
return nil
|
|
}
|
|
|
|
type UmountKubelet struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (u *UmountKubelet) Execute(runtime connector.Runtime) error {
|
|
procMountsFile := "/proc/mounts"
|
|
targetPaths := []string{
|
|
"/var/lib/kubelet",
|
|
"/run/netns/cni",
|
|
}
|
|
f, err := os.Open(procMountsFile)
|
|
if err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return fmt.Errorf("failed to open %s: %w", procMountsFile, err)
|
|
}
|
|
return nil
|
|
}
|
|
defer f.Close()
|
|
|
|
var mounts []string
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
continue
|
|
}
|
|
for _, targetPath := range targetPaths {
|
|
if strings.HasPrefix(fields[1], targetPath) {
|
|
mounts = append(mounts, fields[1])
|
|
}
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return fmt.Errorf("failed to scan %s: %w", procMountsFile, err)
|
|
}
|
|
|
|
logger.Debugf("kubelet mounts %v", mounts)
|
|
|
|
for _, m := range mounts {
|
|
runtime.GetRunner().SudoCmd(fmt.Sprintf("umount %s && rm -rf %s", m, m), false, true)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type KubectlDeleteCurrentWorkerNode struct {
|
|
common.KubeAction
|
|
FailOnError bool
|
|
}
|
|
|
|
func (k *KubectlDeleteCurrentWorkerNode) Execute(runtime connector.Runtime) error {
|
|
// currently only master node has a redis server
|
|
if util.IsExist(storage.RedisServiceFile) {
|
|
return nil
|
|
}
|
|
|
|
kubectl, err := util.GetCommand(common.CommandKubectl)
|
|
// kubernetes very likely not installed
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
nodeName := runtime.GetSystemInfo().GetHostname()
|
|
if _, _, err := util.Exec(context.Background(), fmt.Sprintf(
|
|
"%s delete node %s", kubectl, nodeName),
|
|
true, false); err != nil {
|
|
if k.FailOnError {
|
|
return err
|
|
}
|
|
logger.Infof("failed to delete current node from kubernetes metadata, if this is a worker node, please delete it manually by \"kubectl delete node %s\" on the master to clean up", nodeName)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type ConfigureKubernetes struct {
|
|
common.KubeAction
|
|
}
|
|
|
|
func (c *ConfigureKubernetes) Execute(runtime connector.Runtime) error {
|
|
host := runtime.RemoteHost()
|
|
kubeHost := host.(*kubekeyv1alpha2.KubeHost)
|
|
for k, v := range kubeHost.Labels {
|
|
labelCmd := fmt.Sprintf("/usr/local/bin/kubectl label --overwrite node %s %s=%s", host.GetName(), k, v)
|
|
_, err := runtime.GetRunner().SudoCmd(labelCmd, true, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type EtcdSecurityEnhancemenAction struct {
|
|
common.KubeAction
|
|
ModuleName string
|
|
}
|
|
|
|
func (s *EtcdSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
|
|
chmodEtcdCertsDirCmd := "chmod 700 /etc/ssl/etcd/ssl"
|
|
chmodEtcdCertsCmd := "chmod 600 /etc/ssl/etcd/ssl/*"
|
|
chmodEtcdDataDirCmd := "chmod 700 /var/lib/etcd"
|
|
chmodEtcdCmd := "chmod 550 /usr/local/bin/etcd*"
|
|
|
|
chownEtcdCertsDirCmd := "chown root:root /etc/ssl/etcd/ssl"
|
|
chownEtcdCertsCmd := "chown root:root /etc/ssl/etcd/ssl/*"
|
|
chownEtcdDataDirCmd := "chown etcd:etcd /var/lib/etcd"
|
|
chownEtcdCmd := "chown root:root /usr/local/bin/etcd*"
|
|
|
|
ETCDcmds := []string{chmodEtcdCertsDirCmd, chmodEtcdCertsCmd, chmodEtcdDataDirCmd, chmodEtcdCmd, chownEtcdCertsDirCmd, chownEtcdCertsCmd, chownEtcdDataDirCmd, chownEtcdCmd}
|
|
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(ETCDcmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type MasterSecurityEnhancemenAction struct {
|
|
common.KubeAction
|
|
ModuleName string
|
|
}
|
|
|
|
func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
|
|
// Control-plane Security Enhancemen
|
|
chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes"
|
|
chownKubernetesDirCmd := "chown root:root /etc/kubernetes"
|
|
|
|
chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
|
|
chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes/*"
|
|
|
|
chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
|
|
chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
|
|
|
|
chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
|
|
chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
|
|
|
|
// node Security Enhancemen
|
|
chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
|
|
chownCniConfigDir := "chown root:root -R /etc/cni/net.d"
|
|
|
|
chmodBinDir := "chmod 550 /usr/local/bin/"
|
|
chownBinDir := "chown root:root /usr/local/bin/"
|
|
|
|
chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*"
|
|
chownKubeCmd := "chown root:root -R /usr/local/bin/kube*"
|
|
|
|
chmodHelmCmd := "chmod 550 /usr/local/bin/helm"
|
|
chownHelmCmd := "chown root:root /usr/local/bin/helm"
|
|
|
|
chmodCniDir := "chmod 550 -R /opt/cni/bin"
|
|
chownCniDir := "chown root:root -R /opt/cni/bin"
|
|
|
|
chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*"
|
|
chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*"
|
|
|
|
chmodCertsRenew := "chmod 640 /etc/systemd/system/k8s-certs-renew*"
|
|
chownCertsRenew := "chown root:root /etc/systemd/system/k8s-certs-renew*"
|
|
|
|
chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
|
|
chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig, chmodCertsRenew}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig, chownCertsRenew}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type NodesSecurityEnhancemenAction struct {
|
|
common.KubeAction
|
|
ModuleName string
|
|
}
|
|
|
|
func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error {
|
|
// Control-plane Security Enhancemen
|
|
chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes"
|
|
chownKubernetesDirCmd := "chown root:root /etc/kubernetes"
|
|
|
|
chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
|
|
chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes"
|
|
|
|
chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
|
|
chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
|
|
|
|
chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
|
|
chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
|
|
|
|
// node Security Enhancemen
|
|
chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
|
|
chownCniConfigDir := "chown root:root -R /etc/cni/net.d"
|
|
|
|
chmodBinDir := "chmod 550 /usr/local/bin/"
|
|
chownBinDir := "chown root:root /usr/local/bin/"
|
|
|
|
chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*"
|
|
chownKubeCmd := "chown root:root -R /usr/local/bin/kube*"
|
|
|
|
chmodHelmCmd := "chmod 550 /usr/local/bin/helm"
|
|
chownHelmCmd := "chown root:root /usr/local/bin/helm"
|
|
|
|
chmodCniDir := "chmod 550 -R /opt/cni/bin"
|
|
chownCniDir := "chown root:root -R /opt/cni/bin"
|
|
|
|
chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*"
|
|
chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*"
|
|
|
|
chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
|
|
chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig}
|
|
if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true, false); err != nil {
|
|
return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
|
|
}
|
|
|
|
return nil
|
|
}
|