k0s 折腾笔记
# 安装 k0sctl
wget https://github.com/k0sproject/k0sctl/releases/download/v0.9.0/k0sctl-linux-x64
chmod +x k0sctl-linux-x64
mv k0sctl-linux-x64 /usr/local/bin/k0sctl
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
- ssh:
address: 10.0.0.12
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
- ssh:
address: 10.0.0.13
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
- ssh:
address: 10.0.0.14
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: worker
- ssh:
address: 10.0.0.15
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: worker
k0s:
version: 1.21.2+k0s.1
config:
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s
spec:
api:
address: 10.0.0.11
port: 6443
k0sApiPort: 9443
sans:
- 10.0.0.11
- 10.0.0.12
- 10.0.0.13
storage:
type: etcd
etcd:
peerAddress: 10.0.0.11
network:
kubeProxy:
disabled: false
mode: ipvs
➜ tmp k0sctl apply -c bak.yaml
⠀⣿⣿⡇⠀⠀⢀⣴⣾⣿⠟⠁⢸⣿⣿⣿⣿⣿⣿⣿⡿⠛⠁⠀⢸⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠀█████████ █████████ ███
⠀⣿⣿⡇⣠⣶⣿⡿⠋⠀⠀⠀⢸⣿⡇⠀⠀⠀⣠⠀⠀⢀⣠⡆⢸⣿⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀███ ███ ███
⠀⣿⣿⣿⣿⣟⠋⠀⠀⠀⠀⠀⢸⣿⡇⠀⢰⣾⣿⠀⠀⣿⣿⡇⢸⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠀███ ███ ███
⠀⣿⣿⡏⠻⣿⣷⣤⡀⠀⠀⠀⠸⠛⠁⠀⠸⠋⠁⠀⠀⣿⣿⡇⠈⠉⠉⠉⠉⠉⠉⠉⠉⢹⣿⣿⠀███ ███ ███
⠀⣿⣿⡇⠀⠀⠙⢿⣿⣦⣀⠀⠀⠀⣠⣶⣶⣶⣶⣶⣶⣿⣿⡇⢰⣶⣶⣶⣶⣶⣶⣶⣶⣾⣿⣿⠀█████████ ███ ██████████
k0sctl 0.0.0 Copyright 2021, k0sctl authors.
Anonymized telemetry of usage will be sent to the authors.
By continuing to use k0sctl you agree to these terms:
https://k0sproject.io/licenses/eula
INFO ==> Running phase: Connect to hosts
INFO [ssh] 10.0.0.15:22: connected
INFO [ssh] 10.0.0.11:22: connected
INFO [ssh] 10.0.0.12:22: connected
INFO [ssh] 10.0.0.14:22: connected
INFO [ssh] 10.0.0.13:22: connected
INFO ==> Running phase: Detect host operating systems
INFO [ssh] 10.0.0.11:22: is running Ubuntu 20.04.2 LTS
INFO [ssh] 10.0.0.12:22: is running Ubuntu 20.04.2 LTS
INFO [ssh] 10.0.0.14:22: is running Ubuntu 20.04.2 LTS
INFO [ssh] 10.0.0.13:22: is running Ubuntu 20.04.2 LTS
INFO [ssh] 10.0.0.15:22: is running Ubuntu 20.04.2 LTS
INFO ==> Running phase: Prepare hosts
INFO ==> Running phase: Gather host facts
INFO [ssh] 10.0.0.11:22: discovered ens33 as private interface
INFO [ssh] 10.0.0.13:22: discovered ens33 as private interface
INFO [ssh] 10.0.0.12:22: discovered ens33 as private interface
INFO ==> Running phase: Download k0s on hosts
INFO [ssh] 10.0.0.11:22: downloading k0s 1.21.2+k0s.1
INFO [ssh] 10.0.0.13:22: downloading k0s 1.21.2+k0s.1
INFO [ssh] 10.0.0.12:22: downloading k0s 1.21.2+k0s.1
INFO [ssh] 10.0.0.15:22: downloading k0s 1.21.2+k0s.1
INFO [ssh] 10.0.0.14:22: downloading k0s 1.21.2+k0s.1
......
# 注意：目标机器 hostname 不应当为域名形式，这里的样例是已经修复了这个问题
k1.node ➜ ~ k0s kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k1.node Ready10m v1.21.2+k0s 10.0.0.11 Ubuntu 20.04.2 LTS 5.4.0-77-generic containerd://1.4.6
k2.node Ready10m v1.21.2+k0s 10.0.0.12 Ubuntu 20.04.2 LTS 5.4.0-77-generic containerd://1.4.6
k3.node Ready10m v1.21.2+k0s 10.0.0.13 Ubuntu 20.04.2 LTS 5.4.0-77-generic containerd://1.4.6
k4.node Ready10m v1.21.2+k0s 10.0.0.14 Ubuntu 20.04.2 LTS 5.4.0-77-generic containerd://1.4.6
k5.node Ready10m v1.21.2+k0s 10.0.0.15 Ubuntu 20.04.2 LTS 5.4.0-77-generic containerd://1.4.6
文件上传：k0sctl 允许定义在安装前的文件上传，在安装之前 k0sctl 会把已经定义的相关文件全部上传到目标主机，包括不限于 k0s 本身二进制文件、离线镜像包、其他安装文件、其他辅助脚本等。
Manifests 与 Helm：当将特定的文件上传到 Master 节点的 /var/lib/k0s/manifests 目录时，k0s 在安装过程中会自动应用这些配置，类似 kubelet 的 static pod 一样，只不过 k0s 允许全部资源（包括不限于 Deployment、DaemonSet、namespace 等）；同样也可以直接在 k0sctl.yaml 添加 Helm 配置，k0s 也会以同样的方式帮你管理。
辅助脚本：可以在每个主机下配置 hooks 选项来实现执行一些特定的脚本（文档里没有，需要看源码），以便在特定情况下做点骚操作。
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
# files 配置将会在安装前将相关文件上传到目标主机
files:
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
# 在该目录下的 image 压缩包将会被自动导入到 containerd 中
dstDir: /var/lib/k0s/images/
perm: 0755
......
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
files:
# 将 Flannel 的 yaml 放到 Manifests 里（需要单独创建一个目录）
- name: flannel
src: /Users/bleem/tmp/kube-flannel.yaml
dstDir: /var/lib/k0s/manifests/flannel
perm: 0644
# 自己安装一下 CNI 插件
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
k0s:
version: v1.21.2+k0s.1
config:
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s
spec:
api:
address: 10.0.0.11
port: 6443
k0sApiPort: 9443
sans:
- 10.0.0.11
- 10.0.0.12
- 10.0.0.13
storage:
type: etcd
network:
podCIDR: 10.244.0.0/16
serviceCIDR: 10.96.0.0/12
# 这里指定 CNI 为 custom 自定义类型，这样
# k0s 就不会安装 Calico/kube-router 了
provider: custom
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
# 声明需要上传二进制文件
uploadBinary: true
# 指定二进制文件位置
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: flannel
src: /Users/bleem/tmp/kube-flannel.yaml
dstDir: /var/lib/k0s/manifests/flannel
perm: 0644
......
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: flannel
src: /Users/bleem/tmp/kube-flannel.yaml
dstDir: /var/lib/k0s/manifests/flannel
perm: 0644
......
k0s:
version: v1.21.2+k0s.1
config:
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s
spec:
api:
address: 10.0.0.11
port: 6443
k0sApiPort: 9443
sans:
- 10.0.0.11
- 10.0.0.12
- 10.0.0.13
# 指定内部组件的镜像使用的版本
images:
#konnectivity:
# image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
# version: v0.0.21
#metricsserver:
# image: gcr.io/k8s-staging-metrics-server/metrics-server
# version: v0.3.7
kubeproxy:
image: k8s.gcr.io/kube-proxy
version: v1.21.3
#coredns:
# image: docker.io/coredns/coredns
# version: 1.7.0
#calico:
# cni:
# image: docker.io/calico/cni
# version: v3.18.1
# node:
# image: docker.io/calico/node
# version: v3.18.1
# kubecontrollers:
# image: docker.io/calico/kube-controllers
# version: v3.18.1
#kuberouter:
# cni:
# image: docker.io/cloudnativelabs/kube-router
# version: v1.2.1
# cniInstaller:
# image: quay.io/k0sproject/cni-node
# version: 0.1.0
default_pull_policy: IfNotPresent
#default_pull_policy: Never
spec.api.extraArgs：用于自定义 kube-apiserver 的自定义参数（KV map）
spec.scheduler.extraArgs：用于自定义 kube-scheduler 的自定义参数（KV map）
spec.controllerManager.extraArgs：用于自定义 kube-controller-manager 自定义参数（KV map）
spec.workerProfiles：用于覆盖 kubelet-config.yaml 中的配置，该配置最终将于默认的 kubelet-config.yaml 合并
6443（for Kubernetes API）：负载均衡器 6443 负载所有 Master 节点的 6443
9443（for controller join API）：负载均衡器 9443 负载所有 Master 节点的 9443
8132（for Konnectivity agent）：负载均衡器 8132 负载所有 Master 节点的 8132
8133（for Konnectivity server）：负载均衡器 8133 负载所有 Master 节点的 8133
error_log syslog:server=unix:/dev/log notice;
worker_processes auto;
events {
multi_accept on;
use epoll;
worker_connections 1024;
}
stream {
upstream kube_apiserver {
least_conn;
server 10.0.0.11:6443;
server 10.0.0.12:6443;
server 10.0.0.13:6443;
}
upstream konnectivity_agent {
least_conn;
server 10.0.0.11:8132;
server 10.0.0.12:8132;
server 10.0.0.13:8132;
}
upstream konnectivity_server {
least_conn;
server 10.0.0.11:8133;
server 10.0.0.12:8133;
server 10.0.0.13:8133;
}
upstream controller_join_api {
least_conn;
server 10.0.0.11:9443;
server 10.0.0.12:9443;
server 10.0.0.13:9443;
}
server {
listen 0.0.0.0:6443;
proxy_pass kube_apiserver;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
server {
listen 0.0.0.0:8132;
proxy_pass konnectivity_agent;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
server {
listen 0.0.0.0:8133;
proxy_pass konnectivity_server;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
server {
listen 0.0.0.0:9443;
proxy_pass controller_join_api;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
}
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s-cluster
spec:
hosts:
- ssh:
address: 10.0.0.11
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
# role 支持的值
# 'controller' 单 Master
# 'worker' 单 Worker
# 'controller + worker' Master 和 Worker 都运行
role: controller+worker
# 从本地 上传 k0s bin 文件，不要在目标机器下载
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
# 上传其他文件
files:
# 上传 Flannel 配置，使用自定的 Flannel 替换内置的 Calico
- name: flannel
src: /Users/bleem/tmp/kube-flannel.yaml
dstDir: /var/lib/k0s/manifests/flannel
perm: 0644
# 上传打包好的 image 镜像包，k0s 会自动导入到 containerd
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
dstDir: /var/lib/k0s/images/
perm: 0755
# 使用 Flannel 后每个机器要上传对应的 CNI 插件
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
- ssh:
address: 10.0.0.12
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
dstDir: /var/lib/k0s/images/
perm: 0755
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
- ssh:
address: 10.0.0.13
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: controller+worker
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
dstDir: /var/lib/k0s/images/
perm: 0755
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
- ssh:
address: 10.0.0.14
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: worker
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
dstDir: /var/lib/k0s/images/
perm: 0755
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
- ssh:
address: 10.0.0.15
user: root
port: 22
keyPath: /Users/bleem/.ssh/id_rsa
role: worker
uploadBinary: true
k0sBinaryPath: /Users/bleem/tmp/k0s
files:
- name: image-bundle
src: /Users/bleem/tmp/bundle_file
dstDir: /var/lib/k0s/images/
perm: 0755
- name: cni-plugins
src: /Users/bleem/tmp/cni-plugins/*
dstDir: /opt/cni/bin/
perm: 0755
k0s:
version: v1.21.2+k0s.1
config:
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s
spec:
api:
# 此处填写外部的负载均衡器地址，所有 kubelet 会链接这个地址
externalAddress: 10.0.0.20
# 不要忘了为外部负载均衡器添加 API 证书的 SAN
sans:
- 10.0.0.11
- 10.0.0.12
- 10.0.0.13
- 10.0.0.20
# 存储类型使用 etcd，etcd 集群由 k0s 自动管理
storage:
type: etcd
network:
podCIDR: 10.244.0.0/16
serviceCIDR: 10.96.0.0/12
# 网络插件使用 custom，然后让 Flannel 接管
provider: custom
kubeProxy:
disabled: false
# 开启 kubelet 的 ipvs 模式
mode: ipvs
# 不发送任何匿名统计信息
telemetry:
enabled: false
images:
default_pull_policy: IfNotPresent
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: k0s
spec:
storage:
type: kine
kine:
dataSource: "sqlite:///var/lib/k0s/db/state.db?more=rwc&_journal=WAL&cache=shared"
k0s kubeconfig create --groups "system:masters" testUser > k0s.config
https://docs.k0sproject.io/v1.21.2+k0s.1/airgap-install/
https://github.com/containernetworking/plugins/releases
●这可能是最为详细的Docker入门笔记，Docker学习看这一篇就够了