From 772c556517f0c21c055e2b47f816665d546477fd Mon Sep 17 00:00:00 2001 From: egg12138 Date: Mon, 17 Nov 2025 09:50:35 +0800 Subject: [PATCH 1/7] yocto: update comment for hi309a.conf - as a fixup Signed-off-by: egg12138 --- bsp/meta-hisilicon/conf/machine/hi309a.conf | 24 ++++++++++----------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/bsp/meta-hisilicon/conf/machine/hi309a.conf b/bsp/meta-hisilicon/conf/machine/hi309a.conf index 97b8ddd0a3d..16795fe2147 100644 --- a/bsp/meta-hisilicon/conf/machine/hi309a.conf +++ b/bsp/meta-hisilicon/conf/machine/hi309a.conf @@ -4,11 +4,12 @@ require conf/machine/include/hi309a.inc KERNEL_MODULE_AUTOLOAD = " ${@bb.utils.contains('MCS_FEATURES', 'openamp', 'mcs_km', '', d)} " #================================================================================== -# README: How to Add a New hi309x Machine (hi3093 Configuration As Template) +# README: Add a New hi309x Machine (hi3093 Configuration As Template) #================================================================================== # # This file demonstrates the pattern for adding a new hi309x variant machine -# by inheriting from the base hi3093 configuration with minimal customizations. +# by inheriting from the base hi3093 configuration with basic customizations, using +# similar file system contents and structure. # # BACKGROUND: # ----------- @@ -16,6 +17,8 @@ KERNEL_MODULE_AUTOLOAD = " ${@bb.utils.contains('MCS_FEATURES', 'openamp', 'mcs_ # To add a new variant (like hi309a, hi309b, etc.) with subtle difference, # you mirror the structure and use inheritance to avoid code duplication. # +# +# Take hi309a as an example, # FILES CREATED/ADDED FOR hi309a: # -------------------------------- # 1. conf/machine/hi309a.conf - Main machine config (this file) @@ -25,6 +28,7 @@ KERNEL_MODULE_AUTOLOAD = " ${@bb.utils.contains('MCS_FEATURES', 'openamp', 'mcs_ # 5. recipes-core/images/image-hi309a.inc - Image composition config, (mainly kernel modules) # 6. recipes-core/images/image-early-config-hi309a.inc - Early image config # 7. .oebuild/platform/hi309a.yaml - Platform config for oebuild +# files above plays a basic role for hi309a divergence from hi3093. # # Common customization areas: # 1. MACHINEOVERRIDES - Add your machine name to the override chain @@ -34,22 +38,17 @@ KERNEL_MODULE_AUTOLOAD = " ${@bb.utils.contains('MCS_FEATURES', 'openamp', 'mcs_ # 5. KERNEL_MODULE_AUTOLOAD - Auto-load specific modules # 6. DISTRO_FEATURES - Add variant-specific distro features # -# EXAMPLE (hi309a customizations): -# --------------------------------- -# MACHINEOVERRIDES =. "hi309a:march64le:" -# (Adds hi309a to the override chain, inherits all hi3093 settings) -# -# Kernel config optimization: -# Instead of creating a duplicate defconfig, hi309a uses hi3093's directly: -# OPENEULER_KERNEL_CONFIG = "file://config/hi3093/defconfig" -# (Saves ~4591 lines of duplication when configurations are identical) # # MAINTENANCE TIP: # ---------------- # When hi3093 base configuration is updated, review if changes need to be -# propagated to or are compatible with your hi309x variant. Since most +# **propagated** to or are compatible with your hi309x variant. Since most # configuration is inherited automatically, updates are usually minimal. # +# TODO: +# ---------------- +# If nessesary, we will write a hi309x-common instead of considering hi3093 as base for all hi309x variants. +# # TROUBLESHOOTING: # ---------------- # - If a package isn't working as expected, check MACHINEOVERRIDES includes @@ -59,6 +58,5 @@ KERNEL_MODULE_AUTOLOAD = " ${@bb.utils.contains('MCS_FEATURES', 'openamp', 'mcs_ # - Use bitbake-getvar to inspect variable values # # For more information, see hi3093.inc and hi309a.inc for examples. -# #================================================================================== -- Gitee From fadc9fd713bb55f891ec7761ca0923806dfbc990 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Mon, 17 Nov 2025 21:22:17 +0800 Subject: [PATCH 2/7] k3s: refactor recipes with more flexible customization This major restructuring rewrites the k3s BitBake recipes to provide better modularity, enhanced container runtime engine support, and improved deployment experience. Key changes: - Move k3s recipes to dynamic-layers/virtualization-layer for better modularity - Completely rewrite k3s_%.bbappend with multi-runtime support (isulad/containerd/bundle-containerd), easier for future extension - Add comprehensive deployment scripts (k3s-install-agent, k3s-clean, k3s-kill-agent) - Support phytiumpi platform in k3s.yaml feature configuration - Update kernel configs support for k3s (ip_vs modules) - Update package dependencies and runtime configurations - Support go vendor/mod build mode now - Add a README.md as basic building guideline Benefits: - Enhanced container runtime flexibility with automatic endpoint configuration - Improved deployment experience with comprehensive installation scripts - Modular architecture for easier maintenance - Extended platform support for embedded use cases Signed-off-by: egg12138 --- .oebuild/features/k3s.yaml | 5 +- ...01-Finding-host-local-in-usr-libexec.patch | 0 ...inding-host-local-in-usr-libexec.patch.oee | 24 + .../recipes-containers/k3s/README.md | 148 +++++ .../k3s/cni-containerd-net.conf | 0 .../k3s/cni-containerd-net.conf.oee | 24 + .../k3s/isulad-daemon-setting.patch | 0 .../recipes-containers/k3s/k3s-agent.oee} | 0 .../recipes-containers/k3s/k3s-agent.service | 24 + .../k3s/k3s-agent.service.oee} | 0 .../recipes-containers/k3s/k3s-clean} | 0 .../recipes-containers/k3s/k3s-install-agent | 271 +++++++++ .../recipes-containers/k3s/k3s-kill-agent | 30 + .../recipes-containers}/k3s/k3s-killall.sh | 0 .../recipes-containers}/k3s/k3s.service | 4 +- .../recipes-containers/k3s/k3s_%.bbappend | 328 +++++++++++ .../packagegroup-basic-containerd.bb | 18 +- .../recipes-containers/cni-plugins/cni_git.bb | 6 +- .../cri-tools/cri-tools_git.bb | 2 +- .../recipes-containers/k3s/README.md | 355 ------------ .../k3s/k3s/k3s-install-agent | 512 ------------------ .../recipes-containers/k3s/k3s/modules.txt | 202 ------- .../recipes-containers/k3s/k3s_%.bbappend | 30 - .../recipes-containers/k3s/k3s_v1.22.17.bb | 439 --------------- .../recipes-core/images/openeuler-image.bb | 2 +- .../packagegroups/packagegroup-k3s.bb | 23 +- .../files/meta-data/features/k3s/k3s.scc | 2 +- .../meta-data/features/k3s/net/ip_vs.cfg | 37 ++ .../meta-data/features/k3s/net/ip_vs.scc | 5 + 29 files changed, 925 insertions(+), 1566 deletions(-) rename meta-openeuler/{recipes-containers/k3s => dynamic-layers/virtualization-layer/recipes-containers}/k3s/0001-Finding-host-local-in-usr-libexec.patch (100%) create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md rename meta-openeuler/{recipes-containers/k3s => dynamic-layers/virtualization-layer/recipes-containers}/k3s/cni-containerd-net.conf (100%) create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee rename meta-openeuler/{recipes-containers/k3s => dynamic-layers/virtualization-layer/recipes-containers}/k3s/isulad-daemon-setting.patch (100%) rename meta-openeuler/{recipes-containers/k3s/k3s/k3s-agent => dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee} (100%) create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service rename meta-openeuler/{recipes-containers/k3s/k3s/k3s-agent.service => dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee} (100%) rename meta-openeuler/{recipes-containers/k3s/k3s/k3s-kill-agent => dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-clean} (100%) create mode 100755 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-install-agent create mode 100755 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-kill-agent rename meta-openeuler/{recipes-containers/k3s => dynamic-layers/virtualization-layer/recipes-containers}/k3s/k3s-killall.sh (100%) rename meta-openeuler/{recipes-containers/k3s => dynamic-layers/virtualization-layer/recipes-containers}/k3s/k3s.service (90%) create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend delete mode 100644 meta-openeuler/recipes-containers/k3s/README.md delete mode 100644 meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent delete mode 100644 meta-openeuler/recipes-containers/k3s/k3s/modules.txt delete mode 100644 meta-openeuler/recipes-containers/k3s/k3s_%.bbappend delete mode 100644 meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb create mode 100644 meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.cfg create mode 100644 meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.scc diff --git a/.oebuild/features/k3s.yaml b/.oebuild/features/k3s.yaml index b91f8a5250b..b63f6f56a00 100644 --- a/.oebuild/features/k3s.yaml +++ b/.oebuild/features/k3s.yaml @@ -1,6 +1,7 @@ type: feature -support: qemu-aarch64 +support: qemu-aarch64|phytiumpi local_conf: | - DISTRO_FEATURES:append = " k3s isulad " + DISTRO_FEATURES:append = " k3s-agent " + # DISTRO_FEATURES:append = " k3s-server " diff --git a/meta-openeuler/recipes-containers/k3s/k3s/0001-Finding-host-local-in-usr-libexec.patch b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/0001-Finding-host-local-in-usr-libexec.patch rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee new file mode 100644 index 00000000000..524ac2cb73f --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee @@ -0,0 +1,24 @@ +From 4faf68d68c97cfd10947e1152f711acc59f39647 Mon Sep 17 00:00:00 2001 +From: Erik Jansson +Date: Wed, 16 Oct 2019 15:07:48 +0200 +Subject: [PATCH] Finding host-local in /usr/libexec + +Upstream-status: Inappropriate [embedded specific] +Signed-off-by: +--- + pkg/agent/config/config.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +Index: import/pkg/agent/config/config.go +=================================================================== +--- import.orig/pkg/agent/config/config.go ++++ import/pkg/agent/config/config.go +@@ -445,7 +445,7 @@ + } + + if !nodeConfig.NoFlannel { +- hostLocal, err := exec.LookPath("host-local") ++ hostLocal, err := exec.LookPath("/usr/libexec/cni/host-local") + if err != nil { + return nil, errors.Wrapf(err, "failed to find host-local") + } diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md new file mode 100644 index 00000000000..7b07b746622 --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md @@ -0,0 +1,148 @@ +# k3s：轻量级 Kubernetes 简明指南 + +这个目录提供了在 openEuler Embedded 上构建和运行 [k3s](https://k3s.io/) 所需的 BitBake 配方、补丁和运行时脚本。k3s 是基于 Apache License 2.0 的精简版 Kubernetes，非常适合边缘和资源受限设备。 +我们重写了meta-virtualization的k3s配方，方便在 openEuler Yocto 构建体系里直接使用。 + +--- + +## 这套配方能够做什么 + +- **一次构建输出 server 与 agent**：`k3s` 多路复用二进制会同时提供 `kubectl`、`crictl` 和 `ctr`（在使用外部 containerd 时自动跳过 `ctr`）。 +- **按需切换容器运行时**：通过一个变量就能指定使用 isulad、外部 containerd，或者保留 k3s 自带的 bundle containerd。 +- **可控的依赖获取方式**：既支持在 `do_fetch` 阶段完成 go module 下载，也支持在 `do_compile` 阶段联网下载依赖。 +- **开箱即可的运行时脚本**：包括安装 agent、清理节点、停止服务等常用脚本，并提供 systemd 单元文件。 + +--- + +## 快速开始 + +1. 在 oebuild generate 中添加 k3s feature, 默认启用 k3s-agent +2. 可以在 `local.conf` 中添加 DISTRO_FEATURES:append = "k3s-server", 来构建 k3s server +3. 直接运行： + ```bash + bitbake k3s + ``` + 默认会构建带 bundle containerd 的版本（当前为 v1.27.15-rc2+k3s1）。 + +--- + +## 如何定制构建 + +### 选择容器运行时 + +用 `K3S_EXTERNAL_ENDPOINT` 指定运行时： + +```conf +# conf/local.conf +K3S_EXTERNAL_ENDPOINT ?= "isulad" # 也可以是 containerd +``` + +- 设为 `isulad` 或 `containerd`：选择对应版本并生成带 `--container-runtime-endpoint` 的 systemd 配置。 +- 留空：使用 k3s 自带的 bundle containerd。 +- 设置其他值：自动回退为 containerd 并打印警告。 + +### 控制依赖下载方式 + +构建时默认允许 `do_compile` 访问网络。如果需要在 `do_fetch` 阶段一次性拉全依赖，可提前运行： + +```bash +python3 oe-go-mod-autogen.py --repo https://github.com/k3s-io/k3s.git --rev +``` + +把生成的 `src_uri-*.inc` 和 `relocation-*.inc` 放到当前目录，BitBake 就会从镜像源而不是网络获取依赖。 + +### 其他常用选项 + +| 变量 | 作用 | 示例 | +| --- | --- | --- | +| `K3S_PREBUILD_BINARY` | 为 `1` 时跳过源码编译，改为下载官方预编译二进制 | `K3S_PREBUILD_BINARY = "1"` | +| `K3S_AGENT_BUILD_TAGS` | 追加自定义 Go build tag | `K3S_AGENT_BUILD_TAGS += "selinux"` | +| `IMAGE_INSTALL` | 往镜像里一次性加入 server/agent | `IMAGE_INSTALL:append = " k3s k3s-agent"` | + +--- + +## `k3s_%.bbappend` 做了哪些事情 + +1. **版本与依赖选择**：根据 `K3S_EXTERNAL_ENDPOINT` 设定不同分支、Go 依赖和 reloc 文件，保证 isulad/containerd/bundle-containerd 三套代码路径正确。 +2. **构建参数管理**：统一配置 Go 环境、编译标志、是否使用预编译二进制等选项。 +3. **安装多路复用二进制**：生成 `k3s` 主程序，并按需创建 `kubectl`、`crictl`、`ctr` 等符号链接。 +4. **systemd 集成**：安装 `k3s.service`、`k3s-agent.service`，复制一份 `.ori` 供调试，自动注入容器运行时依赖和 `--container-runtime-endpoint`。 +5. **运行时脚本**：把 `k3s-install-agent`、`k3s-killall.sh`、`k3s-clean` 等脚本装进镜像，方便初始化和清理节点。 + +--- + +## 运行时会安装哪些文件 + +- **k3s-killall.sh**：停止所有 k3s 相关进程、卸载挂载点、清理网络与 iptables，通常由 `k3s.service` 在停止时调用。 +- **k3s-kill-agent**：只针对 agent 节点的清理脚本。 +- **k3s-install-agent**：功能更完整的安装脚本，负责生成 agent 的 systemd drop-in、配置 isulad/containerd、导入离线镜像等。 +- **k3s-agent.sh**：较轻量的脚本，仅负责写入 token/server 参数并重启 agent。 +- **k3s.service / k3s-agent.service**：server 与 agent 的 systemd 单元，内置无限制重启策略并根据运行时自动附加 `Requires` / `After`。 + +--- + +## 配置并启动 k3s agent + +### 推荐：使用 `k3s-agent` 脚本 + +```bash +k3s-install-agent -t -s https://:6443 +``` + +常用参数： + +- `-t/--token`：server 上的 `/var/lib/rancher/k3s/server/node-token` +- `-s/--server`：server API 地址 +- `-e/--endpoint`：覆盖容器运行时（默认为构建时写入的值） +- `--airgap `：导入离线镜像。使用外部 containerd 时会调用 `containerd-ctr -n k8s.io images import `，使用 isulad 时会调用 `isula load -i ` +- `--skip-airgap`：跳过离线镜像导入 +- `--isula-setup`：只做 isulad 调整后退出 + +脚本执行流程概览： + +1. 写入 `/etc/systemd/system/k3s-agent.service.d/10-env.conf` +2. 视运行时决定是否修改 `/etc/isulad/daemon.json` 或导入 containerd 镜像 +3. 清理旧的 `/var/lib/rancher/k3s/agent` +4. `systemctl daemon-reload && systemctl restart --now k3s-agent` + +### 轻量方案：`k3s-agent.sh` + +若只想传 token 与 server，可执行： + +```bash +k3s-agent.sh -t -s https://:6443 +``` + +> 提示：在同一台机器上同时跑 server + agent 做测试时，请直接执行 `k3s agent ...`，不要运行安装脚本，以免它清理掉 Flannel 网络。 + +--- + +## 资源与调试建议 + +- **内存**：QEMU 默认 256 MB 内存不足以跑 k3s，建议至少 2 GB。示例：`runqemu ... qemuparams="-m 2048"`。 +- **磁盘**：core-image* 组合需要额外空间，常见做法是在镜像配方或 `local.conf` 增加 + `IMAGE_ROOTFS_EXTRA_SPACE = "2097152"`（增加约 2 GB）。 +- **查看日志**：`journalctl -xeu k3s`。 +- **常用引导命令**： + ```bash + runqemu qemu-aarch64 nographic kvm slirp qemuparams="-m 2048" + ``` + +--- + +## 关于依赖与 airgap 构建 + +如果需要完全离线的构建或部署： + +1. 使用 `oe-go-mod-autogen.py` 生成指定版本的 go module 镜像清单。 +2. 将生成的 `src_uri-*.inc` 与 `relocation-*.inc` 文件加入配方目录。 +3. 在目标设备上使用 `k3s-install-agent --airgap /path/to/k3s-airgap-images-.tar.gz` 导入镜像。脚本会根据运行时自动选择 `isula load` 或 `containerd-ctr images import`。 + +--- + +## 还想了解更多？ + +- k3s 官方文档： +- 网络选项（CNI、Flannel 后端等）： + +欢迎在本目录基础上继续扩展，例如增加新的运行时、引入更多 Go build tags，或编写自定义镜像配方。只需扩展对应的 `.inc` 文件和 `bbappend` 即可。 diff --git a/meta-openeuler/recipes-containers/k3s/k3s/cni-containerd-net.conf b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/cni-containerd-net.conf rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee new file mode 100644 index 00000000000..ca434d6fcdf --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee @@ -0,0 +1,24 @@ +{ + "cniVersion": "0.4.0", + "name": "containerd-net", + "plugins": [ + { + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "promiscMode": true, + "ipam": { + "type": "host-local", + "subnet": "10.88.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true} + } + ] +} diff --git a/meta-openeuler/recipes-containers/k3s/k3s/isulad-daemon-setting.patch b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/isulad-daemon-setting.patch similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/isulad-daemon-setting.patch rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/isulad-daemon-setting.patch diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s-agent rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service new file mode 100644 index 00000000000..feddac7342d --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service @@ -0,0 +1,24 @@ +# Derived from the k3s install.sh's create_systemd_service_file() function +[Unit] +Description=Lightweight Kubernetes Agent +Documentation=https://k3s.io + +[Install] +WantedBy=multi-user.target + +[Service] +Type=notify +KillMode=control-group +Delegate=yes +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s agent +ExecStopPost=/usr/local/bin/k3s-clean + diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-kill-agent b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-clean similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s-kill-agent rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-clean diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-install-agent b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-install-agent new file mode 100755 index 00000000000..6d192b7aac7 --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-install-agent @@ -0,0 +1,271 @@ +#!/bin/sh -eu +# Lightweight helper to configure a k3s agent on openEuler Embedded. +# Combines the simple environment templating flow from upstream k3s-agent +# with the isulad preparation logic we carried historically in k3s-install-agent. + +ENV_CONF=/etc/systemd/system/k3s-agent.service.d/10-env.conf +ISULAD_DAEMON_JSON=/etc/isulad/daemon.json +AIRGAP_IMAGES_DIR=${AIRGAP_IMAGES_DIR:-/etc/k3s/tools} +AIRGAP_IMAGES=${AIRGAP_IMAGES:-} +SKIP_ISULA_AIRGAP=${SKIP_ISULA_AIRGAP:-false} +SKIP_AIRGAP=${SKIP_AIRGAP:-$SKIP_ISULA_AIRGAP} +CONTAINER_RT_EP=${CONTAINER_RT_EP:-@default_container_engine@} +FORCE_ISULA_SETUP=false + +info() { printf '[INFO] %s\n' "$*"; } +warn() { printf '[WARN] %s\n' "$*" >&2; } +fatal() { printf '[ERROR] %s\n' "$*" >&2; exit 1; } + +bool_true() { + case "$1" in + 1|true|TRUE|yes|YES) return 0 ;; + esac + return 1 +} + +detect_arch() { + case "$(uname -m)" in + arm|armv7l) printf 'arm' ;; + aarch64|arm64) printf 'arm64' ;; + x86_64) printf 'amd64' ;; + s390x) printf 's390x' ;; + *) fatal "unsupported architecture $(uname -m)" ;; + esac +} + +resolve_airgap_path() { + if [ -n "$AIRGAP_IMAGES" ]; then + printf '%s' "$AIRGAP_IMAGES" + return + fi + arch=$(detect_arch) + printf '%s/k3s-airgap-images-%s.tar.gz' "$AIRGAP_IMAGES_DIR" "$arch" +} + +isulad_daemon_set() { + if [ ! -f "$ISULAD_DAEMON_JSON" ]; then + warn "${ISULAD_DAEMON_JSON} not found, skipping isulad daemon tweaks" + return 0 + fi + sed -i 's/"cni-bin-dir": "*",/"cni-bin-dir": "\/var\/lib\/rancher\/k3s\/data\/current\/bin",/' "$ISULAD_DAEMON_JSON" + sed -i 's/"cni-conf-dir": "*",/"cni-conf-dir": "\/var\/lib\/rancher\/k3s\/agent\/etc\/cni\/net.d",/' "$ISULAD_DAEMON_JSON" + sed -i 's/"pod-sandbox-image": "*",/"pod-sandbox-image": "docker.io\/rancher\/mirrored-pause:3.6",/' "$ISULAD_DAEMON_JSON" + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl restart isulad +} + +isulad_preload_images() { + bool_true "$SKIP_AIRGAP" && { info "skipping isula airgap preload"; return; } + image_tar=$(resolve_airgap_path) + if [ ! -f "$image_tar" ]; then + warn "airgap archive $image_tar not found; skipping isula preload" + return + fi + if ! command -v isula >/dev/null 2>&1; then + warn "isula command missing; cannot preload $image_tar" + return + fi + info "loading airgap images from $image_tar" + isula load -i "$image_tar" >/dev/null +} + +configure_isulad() { + info "configuring isulad runtime" + isulad_daemon_set + isulad_preload_images +} + +containerd_preload_images() { + bool_true "$SKIP_AIRGAP" && { info "skipping containerd airgap preload"; return; } + image_tar=$(resolve_airgap_path) + if [ ! -f "$image_tar" ]; then + warn "airgap archive $image_tar not found; skipping containerd preload" + return + fi + if ! command -v ctr >/dev/null 2>&1; then + warn "ctr command missing; cannot preload $image_tar" + return + fi + info "importing airgap images into containerd from $image_tar" + if ! ctr -n k8s.io images import "$image_tar" >/dev/null 2>&1; then + warn "ctr failed to import $image_tar" + fi +} + +is_containerd_endpoint() { + case "$1" in + containerd|unix:///run/containerd/containerd.sock|unix:///var/run/containerd/containerd.sock|/run/containerd/containerd.sock|/var/run/containerd/containerd.sock) + return 0 + ;; + esac + return 1 +} + +configure_runtime_endpoint() { + K3S_AGENT_SERVICE="/etc/systemd/system/k3s-agent.service" + if [ ! -f "$K3S_AGENT_SERVICE" ]; then + warn "k3s-agent.service not found, skipping runtime endpoint configuration" + return + fi + + if [ -z "$CONTAINER_RT_EP" ]; then + info "No container runtime endpoint specified, using default" + return + fi + + case "$CONTAINER_RT_EP" in + isulad) + NEW_ENDPOINT="unix:///var/run/isulad.sock" + ;; + containerd) + NEW_ENDPOINT="unix:///run/containerd/containerd.sock" + ;; + *) + if echo "$CONTAINER_RT_EP" | grep -q "^unix://"; then + NEW_ENDPOINT="$CONTAINER_RT_EP" + else + NEW_ENDPOINT="unix:///run/$CONTAINER_RT_EP/$CONTAINER_RT_EP.sock" + fi + ;; + esac + + if grep -q "ExecStart=.*--container-runtime-endpoint=${NEW_ENDPOINT}" "$K3S_AGENT_SERVICE" 2>/dev/null; then + info "k3s-agent.service already configured with correct endpoint: $NEW_ENDPOINT" + return + fi + + if grep -q "ExecStart=.*k3s agent" "$K3S_AGENT_SERVICE" 2>/dev/null; then + sed -i "s|ExecStart=\(.*\) --container-runtime-endpoint=[^ ]*|\1|g" "$K3S_AGENT_SERVICE" + sed -i "s|ExecStart=.*k3s agent.*|& --container-runtime-endpoint=${NEW_ENDPOINT}|" "$K3S_AGENT_SERVICE" + info "Updated k3s-agent.service with container runtime endpoint: $NEW_ENDPOINT" + fi +} + +usage() { +cat <<'USAGE' +Usage: k3s-install-agent [OPTIONS] + +Options: + -t, --token Cluster token + -s, --server k3s server URL + -e, --endpoint Container runtime endpoint (default: @default_container_engine@) + --token-file Read token from file + --node-name Override node name + --resolv-conf Custom resolv.conf for kubelet + --cluster-secret Legacy shared secret support + --isulad-setup Configure isulad and exit + --airgap Explicit airgap tarball for runtime preload + --skip-airgap Skip loading any airgap tarball + -h, --help Show this message +USAGE +} + +[ $# -gt 0 ] || { usage; exit 0; } +case $1 in + -*) :;; + *) usage; exit 1;; +esac + +rm -f "$ENV_CONF" +mkdir -p "${ENV_CONF%/*}" +printf '[Service]\n' > "$ENV_CONF" + +while getopts "t:s:e:-:hi" opt; do + case "$opt" in + h) + usage + exit 0 + ;; + i) + FORCE_ISULA_SETUP=true + ;; + t) + VAR_NAME=K3S_TOKEN + ;; + s) + VAR_NAME=K3S_URL + ;; + e) + CONTAINER_RT_EP=$OPTARG + continue + ;; + -) + [ $# -ge $OPTIND ] || { usage; exit 1; } + optname=$OPTARG + case "$optname" in + token) + VAR_NAME=K3S_TOKEN + ;; + token-file) + VAR_NAME=K3S_TOKEN_FILE + ;; + server) + VAR_NAME=K3S_URL + ;; + node-name) + VAR_NAME=K3S_NODE_NAME + ;; + resolv-conf) + VAR_NAME=K3S_RESOLV_CONF + ;; + cluster-secret) + VAR_NAME=K3S_CLUSTER_SECRET + ;; + endpoint) + eval OPTARG='$'$OPTIND + OPTIND=$((OPTIND + 1)) + CONTAINER_RT_EP=$OPTARG + continue + ;; + airgap) + eval OPTARG='$'$OPTIND + OPTIND=$((OPTIND + 1)) + AIRGAP_IMAGES=$OPTARG + continue + ;; + skip-airgap) + SKIP_AIRGAP=true + SKIP_ISULA_AIRGAP=true + continue + ;; + isulad-setup) + FORCE_ISULA_SETUP=true + continue + ;; + help) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac + eval OPTARG='$'$OPTIND + OPTIND=$((OPTIND + 1)) + ;; + *) + usage; exit 1;; + esac + printf 'Environment=%s=%s\n' "$VAR_NAME" "$OPTARG" >> "$ENV_CONF" +done + +chmod 0644 "$ENV_CONF" + +if bool_true $FORCE_ISULA_SETUP && [[ "$CONTAINER_RT_EP" = "isulad" ]]; then + configure_isulad + exit $? +fi + +if [ "$CONTAINER_RT_EP" = "isulad" ]; then + configure_isulad +elif is_containerd_endpoint "$CONTAINER_RT_EP"; then + containerd_preload_images +fi + +configure_runtime_endpoint + +rm -rf /var/lib/rancher/k3s/agent +systemctl daemon-reload +systemctl restart k3s-agent +systemctl enable k3s-agent.service diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-kill-agent b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-kill-agent new file mode 100755 index 00000000000..8eca918c051 --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-kill-agent @@ -0,0 +1,30 @@ +#!/bin/sh -eu +# +# Copyright (C) 2020 Axis Communications AB +# +# SPDX-License-Identifier: Apache-2.0 + +do_unmount() { + [ $# -eq 2 ] || return + local mounts= + while read ignore mount ignore; do + case $mount in + $1/*|$2/*) + mounts="$mount $mounts" + ;; + esac + done /dev/null | grep 'master cni0' | while read ignore iface ignore; do + iface=${iface%%@*} + [ -z "$iface" ] || ip link delete $iface +done + +ip link delete cni0 +ip link delete flannel.1 +rm -rf /var/lib/cni/ diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-killall.sh similarity index 100% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-killall.sh diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s.service b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s.service similarity index 90% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s.service rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s.service index 0917c3f2e36..407d788f3a4 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s.service +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s.service @@ -27,9 +27,9 @@ RestartSec=5s ExecStartPre=/bin/sh -xc '! systemctl is-enabled --quiet nm-cloud-setup.service' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/k3s server +ExecStart=/usr/local/bin/k3s server # Avoid any delay due to this service when the system is rebooting or shutting # down by using the k3s-killall.sh script to kill all of the running k3s # services and containers ExecStopPost=/bin/sh -c "if systemctl is-system-running | grep -i \ - 'stopping'; then /usr/bin/k3s-killall.sh; fi" + 'stopping'; then /usr/local/bin/k3s-killall.sh; fi" diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend new file mode 100644 index 00000000000..859dbd3ae8b --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend @@ -0,0 +1,328 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}:" +CNI_NETWORKING_FILES ?= "${WORKDIR}/cni-containerd-net.conf" +SRCREV_FORMAT = "k3s" + +SRC_URI = "" + +require k3s-config.inc +# used for downloading dependencies during do_fetch +K3S_DEP_SRC_URI_FILE ?= "" +K3S_DEP_RELOCATION_FILE ?= "" + +python () { + variants = get_k3s_variants(d) + + external_endpoint = (d.getVar('K3S_EXTERNAL_ENDPOINT') or '').strip() + + if external_endpoint: + selected_engine = external_endpoint + bb.note("K3S: External container engine is %s, selecting %s version" % (selected_engine, selected_engine)) + else: + selected_engine = "bundle-containerd" + + if selected_engine not in variants: + bb.warn('Unknown selected container engine "%s", falling back to containerd' % selected_engine) + selected_engine = "containerd" + + variant = variants[selected_engine] + d.setVar('K3S_BRANCH', variant['branch']) + d.setVar('PV', variant['pv'] + "+git" + variant['srcrev']) + d.setVar('SRCREV_k3s', variant['srcrev']) + d.setVar('K3S_SELECTED_ENGINE', selected_engine) + + # Select dependency files based on container engine and binary source + # When using prebuilt binary, these files are not needed (skip them) + if d.getVar('K3S_PREBUILD_BINARY') == "1": + bb.note("K3S: Using prebuilt binary, skipping dependency source/relocation files") + d.setVar('K3S_DEP_SRC_URI_FILE', '') + d.setVar('K3S_DEP_RELOCATION_FILE', '') + d.setVar('K3S_BUILD_TAGS', 'urfave_cli_no_docs static_build ctrd netcgo osusergo providerless') + elif selected_engine == "isulad": + d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-isulad.inc') + d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-isulad.inc') + d.setVar('K3S_BUILD_TAGS', 'no_btrfs ctrd netcgo osusergo providerless') + else: + d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-containerd.inc') + d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-containerd.inc') + d.setVar('K3S_BUILD_TAGS', 'urfave_cli_no_docs ctrd netcgo osusergo providerless') +} + +# k3s additional abilities: +apparmor="0" +selinux="0" +# hightly recommended to statically build +static_build="1" + +K3S_BUILD_TAGS:append = "\ + ${@bb.utils.contains('apparmor', '1', 'apparmor', '', d)} \ + ${@bb.utils.contains('selinux', '1', 'selinux', '', d)} \ + ${@bb.utils.contains('static_build', '1', ' static_build libsqlite3', '', d)} \ +" + +require ${K3S_DEP_SRC_URI_FILE} +require ${K3S_DEP_RELOCATION_FILE} + +SRC_URI += " \ + git://github.com/k3s-io/k3s.git;branch=${K3S_BRANCH};name=k3s;protocol=https \ + file://k3s.service \ + file://k3s-agent.service \ + file://k3s-install-agent \ + file://k3s-clean \ + file://cni-containerd-net.conf \ + file://0001-Finding-host-local-in-usr-libexec.patch;patchdir=src/import \ + file://k3s-killall.sh \ + file://modules.txt \ +" + +BIN_PREFIX = "${exec_prefix}" +GO_EXTRA_LDFLAGS += " \ + ${@bb.utils.contains('static_build', '1', '-static', '', d)} \ + -lm -ldl -lz -lpthread \ +" +GO_BUILD_LDFLAGS = "-X github.com/k3s-io/k3s/pkg/version.Version=${PV} \ + -X github.com/k3s-io/k3s/pkg/version.GitCommit=${@d.getVar('SRCREV_k3s', d, 1)[:8]} \ + -w -s \ + -extldflags '${GO_EXTRA_LDFLAGS}' \ + " + +K3S_AGENT_BUILD_TAGS ?= "${K3S_BUILD_TAGS}" + +# Use prebuilt k3s binary from k3s.io instead of building from source +# Set to "1" to enable using prebuilt binaries (much faster) +# Remain empty to build from source (default) +K3S_PREBUILD_BINARY ?= "" + +K3S_MIRROR_URL ?= "" + +upx_compress = "false" + +# Map Yocto arch to k3s arch naming +K3S_ARCH:x86-64 = "amd64" +K3S_ARCH:arm = "arm" +K3S_ARCH:aarch64 = "arm64" + + +do_download_prebuilt() { + if [ "${K3S_PREBUILD_BINARY}" != "1" ]; then + return 0 + fi + + install -d "${S}/src/import/dist/artifacts" + cd "${S}/src/import/dist/artifacts" + + K3S_BINARY_NAME="k3s" + if [ "${K3S_ARCH}" != "amd64" ]; then + K3S_BINARY_NAME="k3s-${K3S_ARCH}" + fi + + K3S_DOWNLOAD_URL="${K3S_MIRROR_URL:-https://github.com/k3s-io/k3s/releases/download/${PV}}" + + bbnote "Downloading k3s prebuilt binary from ${K3S_DOWNLOAD_URL}/${K3S_BINARY_NAME}" + + if command -v curl > /dev/null 2>&1; then + curl -sfL "${K3S_DOWNLOAD_URL}/${K3S_BINARY_NAME}" -o k3s + elif command -v wget > /dev/null 2>&1; then + wget -qO k3s "${K3S_DOWNLOAD_URL}/${K3S_BINARY_NAME}" + else + bberror "Neither curl nor wget is available for downloading k3s binary" + return 1 + fi + + if [ ! -f k3s ]; then + bberror "k3s binary download failed" + return 1 + fi + + chmod +x k3s + bbnote "Successfully downloaded k3s binary" + + if ! ./k3s --version > /dev/null 2>&1; then + bberror "Downloaded k3s binary is not valid or not executable" + return 1 + fi + +} + +do_download_prebuilt[network] = "1" +addtask download_prebuilt before do_compile after do_fetch + +inherit go-mod + +DEPENDS += "${@bb.utils.contains('static_build', '1', 'zlib-native zlib', 'zlib', d)}" + +do_compile[network] = "1" +# do_compile[cleandirs] += "${B}/bin ${B}/.mod" + +k3s_fix_gomodcache_perms() { + if [ -d "${B}/.mod" ]; then + chmod -R u+rwX,go+rwX "${B}/.mod" || bbfatal "Failed to fix permissions for ${B}/.mod" + fi + bbnote "set current gomocache dir: ${GOMODCACHE} permissions are: $(stat -c %a ${GOMODCACHE})" +} + +do_compile[postfuncs] += " k3s_fix_gomodcache_perms" +do_compile[prefuncs] += " k3s_fix_gomodcache_perms " +do_compile() { + if [ "${K3S_PREBUILD_BINARY}" = "1" ]; then + if [ -f "${S}/src/import/dist/artifacts/k3s" ]; then + bbnote "Skipping compilation, using prebuilt k3s binary" + return 0 + else + bberror "K3S_PREBUILD_BINARY is set but binary not found at ${S}/src/import/dist/artifacts/k3s" + return 1 + fi + fi + + export GOPATH=${S} + export GO111MODULE=on + export GOMODCACHE="${B}/.mod" + export CGO_ENABLED="1" + export GOPROXY="https://goproxy.cn,https://goproxy.io,https://mirrors.aliyun.com/goproxy/,direct" + export GOARCH=${TARGET_ARCH} + export GIT_SSL_NO_VERIFY=1 + git config --global http.sslVerify false + bbnote "GOPRIVATE=${GOPRIVATE},GOSUMDB=${GOSUMDB}" + cd ${S}/src/import + + build_target="./cmd/server/main.go" + build_output="./dist/artifacts/k3s" + build_tags="${K3S_BUILD_TAGS}" + if [ "${K3S_ROLE}" = "agent" ]; then + # temporary build k3s full binary instead of agentic only + # build_target="./cmd/agent/main.go" + build_tags="${K3S_AGENT_BUILD_TAGS}" + fi + + bbnote "GO_BUILD_TAGS=${K3S_BUILD_TAGS}" + bbnote "GO_LD_FLAGS=${GO_BUILD_LDFLAGS}" + + ${GO} build -tags "${build_tags}" -ldflags "${GO_BUILD_LDFLAGS} " -o ${build_output} ${build_target} + + bbnote "Successfully built k3s version ${PV}" +} + + +do_compile:append() { + if [ "${upx_compress}" = "true" ] && command -v upx > /dev/null; then + upx -9 ${build_output} + fi +} + +do_install() { + install -d "${D}${BIN_PREFIX}/bin" + install -d "${D}/etc/rancher/k3s" + k3s_bin="${S}/src/import/dist/artifacts/k3s" + + install -m 0755 "${k3s_bin}" "${D}${BIN_PREFIX}/bin/k3s" + ln -sr "${D}${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/kubectl" + ln -sr "${D}${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/crictl" + if [ "${K3S_EXTERNAL_ENDPOINT}" != "containerd" ]; then + ln -sr "${D}${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/ctr" + else + bbnote "use containerd as container engine, k3s multi-call ctr links are skipped" + fi + + install -m 0755 "${WORKDIR}/k3s-clean" "${D}${BIN_PREFIX}/bin" + install -m 0755 "${WORKDIR}/k3s-killall.sh" "${D}${BIN_PREFIX}/bin" + install -m 0755 "${WORKDIR}/k3s-install-agent" "${D}${BIN_PREFIX}/bin/k3s-agent" + + if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then + # install to k3s-service packages + install -D -m 0644 "${WORKDIR}/k3s.service" "${D}${systemd_system_unitdir}/k3s.service" + sed -i "s#/usr/local#${BIN_PREFIX}#g" "${D}${systemd_system_unitdir}/k3s.service" + cp "${D}${systemd_system_unitdir}/k3s.service" "${D}${systemd_system_unitdir}/k3s.service.ori" + + install -D -m 0644 "${WORKDIR}/k3s-agent.service" "${D}${systemd_system_unitdir}/k3s-agent.service" + sed -i "s#/usr/local#${BIN_PREFIX}#g" "${D}${systemd_system_unitdir}/k3s-agent.service" + cp "${D}${systemd_system_unitdir}/k3s-agent.service" "${D}${systemd_system_unitdir}/k3s-agent.service.ori" + + if [ "${K3S_SELECTED_ENGINE}" = "bundle-containerd" ]; then + sed -i 's/@default_container_engine@//g' "${D}${BIN_PREFIX}/bin/k3s-agent" + else + sed -i "s/@default_container_engine@/${K3S_SELECTED_ENGINE}/g" "${D}${BIN_PREFIX}/bin/k3s-agent" + fi + + if [ -n "${K3S_EXTERNAL_ENDPOINT}" ]; then + if [ "${K3S_EXTERNAL_ENDPOINT}" = "isulad" ]; then + SERVICE_NAME="isulad.service" + RUNTIME_ENDPOINT="unix:///var/run/isulad.sock" + elif [ "${K3S_EXTERNAL_ENDPOINT}" = "containerd" ]; then + SERVICE_NAME="containerd.service" + RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock" + fi + # no need to configure service for k3s.service 'cause k3s server should configure a lot + if [ -n "${SERVICE_NAME}" ]; then + bbnote "Configuring k3s agent to use external container runtime: ${K3S_EXTERNAL_ENDPOINT}" + if ! grep -q "^Requires=" "${D}${systemd_system_unitdir}/k3s-agent.service"; then + sed -i "/^\[Unit\]/a Requires=${SERVICE_NAME}" \ + "${D}${systemd_system_unitdir}/k3s-agent.service" + else + sed -i "s|^Requires=.*|Requires=${SERVICE_NAME}|" \ + "${D}${systemd_system_unitdir}/k3s-agent.service" + fi + if ! grep -q "^After=" "${D}${systemd_system_unitdir}/k3s-agent.service"; then + sed -i "/^\[Unit\]/a After=${SERVICE_NAME}" \ + "${D}${systemd_system_unitdir}/k3s-agent.service" + else + sed -i "s|^After=.*|After=network-online.target ${SERVICE_NAME}|" \ + "${D}${systemd_system_unitdir}/k3s-agent.service" + fi + sed -i "s|^ExecStart=.*k3s agent|ExecStart=${BIN_PREFIX}/bin/k3s agent --container-runtime-endpoint=${RUNTIME_ENDPOINT}|" \ + "${D}${systemd_system_unitdir}/k3s-agent.service" + fi + fi + else + bbwarn "systemd is highly recommended for k3s" + fi +} + +FILES:${k3s}-server += "${systemd_system_unitdir}/k3s.service.ori" +FILES:${k3s}-agent += "${systemd_system_unitdir}/k3s-agent.service.ori" + +# external container engine selection +python () { + engine_pkgs = get_container_engine_pkg(d) + endpoint = d.getVar('K3S') + d.setVar('engine_pkg', engine_pkgs.get('K3S_SELECTED_ENGINE', '')) +} + +RDEPENDS:${PN} += " \ + ${@bb.utils.contains('K3S_SELECTED_ENGINE','bundle-containerd','','${engine_pkg}',d)} \ + kernel-module-br-netfilter \ + kernel-module-bridge \ + kernel-module-iptable-mangle \ + kernel-module-ip6table-mangle \ + kernel-module-libcrc32c \ + kernel-module-nf-conntrack \ + kernel-module-nf-nat \ + kernel-module-nf-defrag-ipv4 \ + kernel-module-nf-defrag-ipv6 \ + kernel-module-stp \ + kernel-module-xt-addrtype \ + kernel-module-xt-comment \ + kernel-module-xt-nat \ + kernel-module-xt-tcpudp \ +" + +RRECOMMENDS:${PN} += " \ + kernel-module-nf-conntrack-netlink \ + kernel-module-nfnetlink \ + kernel-module-nfnetlink-log \ + kernel-module-nft-chain-nat \ + kernel-module-nft-compat \ + kernel-module-nft-counter \ + kernel-module-xt-connmark \ + kernel-module-xt-conntrack \ + kernel-module-xt-limit \ + kernel-module-xt-mark \ + kernel-module-xt-masquerade \ + kernel-module-xt-multiport \ + kernel-module-xt-nflog \ + kernel-module-xt-physdev \ + kernel-module-xt-statistic \ + kernel-module-vxlan \ + kernel-module-ip-vs \ + kernel-module-ip-vs-rr \ + kernel-module-ip-vs-sh \ + kernel-module-ip-vs-wrr \ +" diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-basic-containerd.bb b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-basic-containerd.bb index 7ebbac263a8..50fa48dfe22 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-basic-containerd.bb +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-basic-containerd.bb @@ -3,12 +3,12 @@ SUMMARY = "custom lightweight containerd-based toolkits" inherit packagegroup features_check REQUIRED_DISTRO_FEATURES += "systemd" -# it is not recommended to package this simple lightweight containerd packagegroup together with isulad -CONFLICT_DISTRO_FEATURES = "isulad" PACKAGES = " \ ${PN} \ " +BUILD_NERDCRTL = "" + # TODO: cni compatibility RDEPENDS:${PN} = " \ virtual-containerd \ @@ -16,21 +16,15 @@ RDEPENDS:${PN} = " \ oci-systemd-hook \ oci-runtime-tools \ oci-image-tools \ - nerdctl \ bridge-utils \ + ${@bb.utils.contains('BUILD_NERDCTL', '', '', 'nerdctl', d)} \ " +# due to network issue, it's recommended to install nerdctl via oebridge +# nerdctl RRECOMMENDS:${PN} = " \ cni \ kernel-module-veth \ kernel-module-bridge \ kernel-module-br-netfilter \ - kernel-module-ebtables \ - kernel-module-nf-nat \ - kernel-module-nf-conntrack-netlink \ - kernel-module-xt-comment \ - kernel-module-xt-statistic \ - kernel-module-xt-multiport \ - kernel-module-xt-addrtype \ - kernel-module-xt-masquerade \ -" + " diff --git a/meta-openeuler/recipes-containers/cni-plugins/cni_git.bb b/meta-openeuler/recipes-containers/cni-plugins/cni_git.bb index aa9902e356c..ebfb2de66bb 100644 --- a/meta-openeuler/recipes-containers/cni-plugins/cni_git.bb +++ b/meta-openeuler/recipes-containers/cni-plugins/cni_git.bb @@ -14,12 +14,12 @@ SRCREV_plugins = "b6a0e0bc96906f0d3bd6bfcaab0b5ae72292f46c" SRCREV_flannel_plugin = "6464faacf5c00e25321573225d74638455ef03a0" SRCREV_FORMAT = "cni_plugins" SRC_URI = "\ - git://github.com/containernetworking/cni.git;branch=main;name=cni;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX} \ + git://github.com/containernetworking/cni.git;branch=main;name=cni;protocol=https;destsuffix=${WORKDIR}/$[BP}/src/import \ file://modules.txt \ " -SRC_URI += "git://github.com/containernetworking/plugins.git;branch=main;destsuffix=${GO_SRCURI_DESTSUFFIX}/src/github.com/containernetworking/plugins;name=plugins;protocol=https" -SRC_URI += "git://github.com/flannel-io/cni-plugin;branch=main;name=flannel_plugin;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX}/src/github.com/containernetworking/plugins/plugins/meta/flannel" +SRC_URI += "git://github.com/containernetworking/plugins.git;branch=main;destsuffix=${WORKDIR}/$[BP}/src/import/src/github.com/containernetworking/plugins;name=plugins;protocol=https" +SRC_URI += "git://github.com/flannel-io/cni-plugin;branch=main;name=flannel_plugin;protocol=https;destsuffix=${WORKDIR}/$[BP}/src/import/src/github.com/containernetworking/plugins/plugins/meta/flannel" # generated via: # ./scripts/oe-go-mod-autogen.py --repo https://github.com/containernetworking/cni.git --rev diff --git a/meta-openeuler/recipes-containers/cri-tools/cri-tools_git.bb b/meta-openeuler/recipes-containers/cri-tools/cri-tools_git.bb index f9b60bbf9f2..21e514f16bc 100644 --- a/meta-openeuler/recipes-containers/cri-tools/cri-tools_git.bb +++ b/meta-openeuler/recipes-containers/cri-tools/cri-tools_git.bb @@ -18,7 +18,7 @@ What is not in scope for this project? \ SRCREV_cri-tools = "17b4dd65d660fec94d7a5a070e3e89ef640f1087" SRC_URI = "\ - git://github.com/kubernetes-sigs/cri-tools.git;branch=master;name=cri-tools;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX} \ + git://github.com/kubernetes-sigs/cri-tools.git;branch=master;name=cri-tools;protocol=https;destsuffix=${WORKDIR}/$[BP}/src/import \ file://0001-build-allow-environmental-CGO-settings-and-pass-dont.patch \ " diff --git a/meta-openeuler/recipes-containers/k3s/README.md b/meta-openeuler/recipes-containers/k3s/README.md deleted file mode 100644 index 8212e8de197..00000000000 --- a/meta-openeuler/recipes-containers/k3s/README.md +++ /dev/null @@ -1,355 +0,0 @@ -# issues - -## isulad 问题 - -启动isulad.service 后， 会发现isula找不到 engine 相关文件; - -## 版本问题 - -考虑到大文件托管不便，现在暂时移除 airgap images; 相关的脚本、设置也都已经作为yocto文件了，暂时将上游设置为k3s-io official -TODO src-openeuler 上游版本和 k3s-io有小出入，他们提供的是1.24+ 的 k3s, 当时QEMU试验部署没有很好工作，所以暂时用1.22.17 - - -## package - -目前的packagegroup-k3s其实是一个单包, 是用变量来控制 full k3s(server) 或者 agent, 这样不好，下一次PR时改成packagegroup的形式； - -## do patch - -k3s check config 脚本在oee镜像中是无用的；因为 oee 镜像没有 kernel cfg 列表 -待解决 - -## do compile - -### cni - -目前使用 oee `yocto-meta-openeuler/recipe-container/cni-plugin` 的cni -TODO: 移除 cni-plugin 配方的依赖，把 cni 也作为单独进程压缩到 multicall binary 中 - -### seccomp && selinux - -还未整合起来, 其中 seccomp 是安全上必需的 - -### containerd, runc, containerd shim - -目前 oee k3s 配方中提供了 containerd, isulad 两种 runtime endpoint 的部分支持，还有一些设置需要在runtime进行，已经整合到脚本/systemd service文件中； - -构建containerd以及相关组件的构建暂时从k3s配方中移除，分析后认为应该单独作为依赖配方构建;即暂时不可以使用 k3s containerd作为 runtime endpoint - -### go vendor & go modules mix compilation - - # ? 如果要构建非 isulad 作runtime的k3s,把 vendor和module混用起来比较好 - # 1. 不希望在do_compile时重复拉取依赖。 - # 2. 其他组件多数是vendor构建的，不保证编译结果差异 - -已经在本地尝试中，如果最终分析结果为没必要，那就不会提交; - -### multicall binary - -#### intro - -k3s实现 single binary , bunches of processes 的方式是压缩-解压, 在调用上，类似于busybox,将各组件做符号链接指向containerd, 识别command line argument来确定要启用的服务，并进行解压。 - -multicall k3s binary 在WORKDIR下一直编译不了,导致压缩二进制这一步无法进行，目前都是用无压缩地二进制测试 - -#### undefined data.Asset - # ? FIXME: - # cmd/k3s/main.go:181:16: undefined: data.AssetNames - # cmd/k3s/main.go:218:23: undefined: data.Asset - # under ${S}, pkg/data/data.go is empty indeed - # 位于${S}下编译会出现上面的错误 - # 源码是从 FROM=yocto-meta-openeuler/../k3s 获取,复制到${S} - # 在FROM用完全相同地编译指令，编译，则没有问题, go env 除了pwd地差别，其他完全没有差异； - # 使用sysroot-native的go, host的go,都一样 - # 从${FROM}前往${S}, 失败，从${S}返回到${FROM}, 成功 - # 将${S}复制到外部，依然失败,${FROM}复制到相同区域，依然成功 - # tree 差别为 ${S} 总是多一个 pkg/deploy/zz_generated... 文件，但将此文件mv到FROM, - # S依然失败，FROM依然成功；build 前总是 go clean -cache - # 编译指令如下： - # `CGO_ENABLED=0 ${GO} build -tags "urfave_cli_no_docs" -ldflags "${K3S_LDFLAGS} ${STATIC}"` - # -v -o "${PN}-${ARCH}" ./cmd/k3s/main.go - # 手动加打印也确认了用的是pkg/data/data.go - -解决中 - -### airgap images - -#### Why airgap images? - -airgap images使得agent node在仅联通 server node时即可加入k3s集群; -配方在调试时总是通过k3s-io官方prebuild的airgap image。 -**如果** 在配方中实现了 airgap images 的构建，那么会方便许多,而且不依赖外部托管数百MB的airgap images了 - - -airgap images 将k3s运行依赖地容器镜像组件通过docker build 打包，目前只能在host上打包，本地尝试中, 如果airgap并不比较，则不提交 -初步分析 没有发现比较容易地在无容器引擎环境中build 容器镜像; oebuild 容器环境内部目前也没有提供 containerd/docker/podman等。 - - -# k3s: Lightweight Kubernetes - -Rancher's [k3s](https://k3s.io/), available under -[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0), provides -lightweight Kubernetes suitable for small/edge devices. There are use cases -where the -[installation procedures provided by Rancher](https://rancher.com/docs/k3s/latest/en/installation/) -are not ideal but a bitbake-built version is what is needed. And only a few -mods to the [k3s source code](https://github.com/rancher/k3s) is needed to -accomplish that. - -## CNI - -By default, K3s will run with flannel as the CNI, using VXLAN as the default -backend. It is both possible to change the flannel backend and to change from -flannel to another CNI. - -Please see -for further k3s networking details. - -## Configure and run a k3s agent - -The convenience script `k3s-agent` can be used to set up a k3s agent (service): - -```shell -k3s-agent -t -s https://:6443 -``` -(Here `` is found in `/var/lib/rancher/k3s/server/node-token` at the -k3s master.) - -Example: -```shell -k3s-agent -t /var/lib/rancher/k3s/server/node-token -s https://localhost:6443 -``` - -If you are running an all in one node (both the server and agent) for testing -purposes, do not run the above script. It will perform cleanup and break flannel -networking on your host. - -Instead, run the following (note the space between 'k3s' and 'agent'): - -```shell -k3s agent -t /var/lib/rancher/k3s/server/token --server http://localhost:6443/ -``` - -## Notes: - -Memory: - - if running under qemu, the default of 256M of memory is not enough, k3s will - OOM and exit. - - Boot with qemuparams="-m 2048" to boot with 2G of memory (or choose the - appropriate amount for your configuration) - -Disk: - - if using qemu and core-image* you'll need to add extra space in your disks - to ensure containers can start. The following in your image recipe, or - local.conf would add 2G of extra space to the rootfs: - -```shell -IMAGE_ROOTFS_EXTRA_SPACE = "2097152" -``` - -## Example qemux86-64 boot line: - -```shell -runqemu qemux86-64 nographic kvm slirp qemuparams="-m 2048" -``` - -k3s logs can be seen via: - - -```shell -% journalctl -u k3s -``` - -or - -```shell -% journalctl -xe -``` - -## Example output from qemux86-64 running k3s server: - -```shell -root@qemux86-64:~# kubectl get nodes -NAME STATUS ROLES AGE VERSION -qemux86-64 Ready master 46s v1.18.9-k3s1 - -root@qemux86-64:~# kubectl get pods -n kube-system -NAME READY STATUS RESTARTS AGE -local-path-provisioner-6d59f47c7-h7lxk 1/1 Running 0 2m32s -metrics-server-7566d596c8-mwntr 1/1 Running 0 2m32s -helm-install-traefik-229v7 0/1 Completed 0 2m32s -coredns-7944c66d8d-9rfj7 1/1 Running 0 2m32s -svclb-traefik-pb5j4 2/2 Running 0 89s -traefik-758cd5fc85-lxpr8 1/1 Running 0 89s - -root@qemux86-64:~# kubectl describe pods -n kube-system - -root@qemux86-64:~# ip a s -1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 - link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 - inet 127.0.0.1/8 scope host lo - valid_lft forever preferred_lft forever - inet6 ::1/128 scope host - valid_lft forever preferred_lft forever -2: eth0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 - link/ether 52:54:00:12:35:02 brd ff:ff:ff:ff:ff:ff - inet 10.0.2.15/24 brd 10.0.2.255 scope global eth0 - valid_lft forever preferred_lft forever - inet6 fec0::5054:ff:fe12:3502/64 scope site dynamic mngtmpaddr - valid_lft 86239sec preferred_lft 14239sec - inet6 fe80::5054:ff:fe12:3502/64 scope link - valid_lft forever preferred_lft forever -3: sit0@NONE: mtu 1480 qdisc noop state DOWN group default qlen 1000 - link/sit 0.0.0.0 brd 0.0.0.0 -4: flannel.1: mtu 1450 qdisc noqueue state UNKNOWN group default - link/ether e2:aa:04:89:e6:0a brd ff:ff:ff:ff:ff:ff - inet 10.42.0.0/32 brd 10.42.0.0 scope global flannel.1 - valid_lft forever preferred_lft forever - inet6 fe80::e0aa:4ff:fe89:e60a/64 scope link - valid_lft forever preferred_lft forever -5: docker0: mtu 1500 qdisc noqueue state DOWN group default - link/ether 02:42:be:3e:25:e7 brd ff:ff:ff:ff:ff:ff - inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0 - valid_lft forever preferred_lft forever -6: cni0: mtu 1450 qdisc noqueue state UP group default qlen 1000 - link/ether 82:8e:b4:f8:06:e7 brd ff:ff:ff:ff:ff:ff - inet 10.42.0.1/24 brd 10.42.0.255 scope global cni0 - valid_lft forever preferred_lft forever - inet6 fe80::808e:b4ff:fef8:6e7/64 scope link - valid_lft forever preferred_lft forever -7: veth82ac482e@if4: mtu 1450 qdisc noqueue master cni0 state UP group default - link/ether ea:9d:14:c1:00:70 brd ff:ff:ff:ff:ff:ff link-netns cni-c52e6e09-f6e0-a47b-aea3-d6c47d3e2d01 - inet6 fe80::e89d:14ff:fec1:70/64 scope link - valid_lft forever preferred_lft forever -8: vethb94745ed@if4: mtu 1450 qdisc noqueue master cni0 state UP group default - link/ether 1e:7f:7e:d3:ca:e8 brd ff:ff:ff:ff:ff:ff link-netns cni-86958efe-2462-016f-292d-81dbccc16a83 - inet6 fe80::8046:3cff:fe23:ced1/64 scope link - valid_lft forever preferred_lft forever -9: veth81ffb276@if4: mtu 1450 qdisc noqueue master cni0 state UP group default - link/ether 2a:1d:48:54:76:50 brd ff:ff:ff:ff:ff:ff link-netns cni-5d77238e-6452-4fa3-40d2-91d48386080b - inet6 fe80::acf4:7fff:fe11:b6f2/64 scope link - valid_lft forever preferred_lft forever -10: vethce261f6a@if4: mtu 1450 qdisc noqueue master cni0 state UP group default - link/ether 72:a3:90:4a:c5:12 brd ff:ff:ff:ff:ff:ff link-netns cni-55675948-77f2-a952-31ce-615f2bdb0093 - inet6 fe80::4d5:1bff:fe5d:db3a/64 scope link - valid_lft forever preferred_lft forever -11: vethee199cf4@if4: mtu 1450 qdisc noqueue master cni0 state UP group default - link/ether e6:90:a4:a3:bc:a1 brd ff:ff:ff:ff:ff:ff link-netns cni-4aeccd16-2976-8a78-b2c4-e028da3bb1ea - inet6 fe80::c85a:8bff:fe0b:aea0/64 scope link - valid_lft forever preferred_lft forever - - -root@qemux86-64:~# kubectl describe nodes - -Name: qemux86-64 -Roles: master -Labels: beta.kubernetes.io/arch=amd64 - beta.kubernetes.io/instance-type=k3s - beta.kubernetes.io/os=linux - k3s.io/hostname=qemux86-64 - k3s.io/internal-ip=10.0.2.15 - kubernetes.io/arch=amd64 - kubernetes.io/hostname=qemux86-64 - kubernetes.io/os=linux - node-role.kubernetes.io/master=true - node.kubernetes.io/instance-type=k3s -Annotations: flannel.alpha.coreos.com/backend-data: {"VtepMAC":"2e:52:6a:1b:76:d4"} - flannel.alpha.coreos.com/backend-type: vxlan - flannel.alpha.coreos.com/kube-subnet-manager: true - flannel.alpha.coreos.com/public-ip: 10.0.2.15 - k3s.io/node-args: ["server"] - k3s.io/node-config-hash: MLFMUCBMRVINLJJKSG32TOUFWB4CN55GMSNY25AZPESQXZCYRN2A==== - k3s.io/node-env: {} - node.alpha.kubernetes.io/ttl: 0 - volumes.kubernetes.io/controller-managed-attach-detach: true -CreationTimestamp: Tue, 10 Nov 2020 14:01:28 +0000 -Taints: -Unschedulable: false -Lease: - HolderIdentity: qemux86-64 - AcquireTime: - RenewTime: Tue, 10 Nov 2020 14:56:27 +0000 -Conditions: - Type Status LastHeartbeatTime LastTransitionTime Reason Message - ---- ------ ----------------- ------------------ ------ ------- - NetworkUnavailable False Tue, 10 Nov 2020 14:43:46 +0000 Tue, 10 Nov 2020 14:43:46 +0000 FlannelIsUp Flannel is running on this node - MemoryPressure False Tue, 10 Nov 2020 14:51:48 +0000 Tue, 10 Nov 2020 14:45:46 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available - DiskPressure False Tue, 10 Nov 2020 14:51:48 +0000 Tue, 10 Nov 2020 14:45:46 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure - PIDPressure False Tue, 10 Nov 2020 14:51:48 +0000 Tue, 10 Nov 2020 14:45:46 +0000 KubeletHasSufficientPID kubelet has sufficient PID available - Ready True Tue, 10 Nov 2020 14:51:48 +0000 Tue, 10 Nov 2020 14:45:46 +0000 KubeletReady kubelet is posting ready status -Addresses: - InternalIP: 10.0.2.15 - Hostname: qemux86-64 -Capacity: - cpu: 1 - ephemeral-storage: 39748144Ki - memory: 2040164Ki - pods: 110 -Allocatable: - cpu: 1 - ephemeral-storage: 38666994453 - memory: 2040164Ki - pods: 110 -System Info: - Machine ID: 6a4abfacbf83457e9a0cbb5777457c5d - System UUID: 6a4abfacbf83457e9a0cbb5777457c5d - Boot ID: f5ddf6c8-1abf-4aef-9e29-106488e3c337 - Kernel Version: 5.8.13-yocto-standard - OS Image: Poky (Yocto Project Reference Distro) 3.2+snapshot-20201105 (master) - Operating System: linux - Architecture: amd64 - Container Runtime Version: containerd://1.4.1-4-ge44e8ebea.m - Kubelet Version: v1.18.9-k3s1 - Kube-Proxy Version: v1.18.9-k3s1 -PodCIDR: 10.42.0.0/24 -PodCIDRs: 10.42.0.0/24 -ProviderID: k3s://qemux86-64 -Non-terminated Pods: (5 in total) - Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE - --------- ---- ------------ ---------- --------------- ------------- --- - kube-system svclb-traefik-jpmnd 0 (0%) 0 (0%) 0 (0%) 0 (0%) 54m - kube-system metrics-server-7566d596c8-wh29d 0 (0%) 0 (0%) 0 (0%) 0 (0%) 56m - kube-system local-path-provisioner-6d59f47c7-npn4d 0 (0%) 0 (0%) 0 (0%) 0 (0%) 56m - kube-system coredns-7944c66d8d-md8hr 100m (10%) 0 (0%) 70Mi (3%) 170Mi (8%) 56m - kube-system traefik-758cd5fc85-phjr2 0 (0%) 0 (0%) 0 (0%) 0 (0%) 54m -Allocated resources: - (Total limits may be over 100 percent, i.e., overcommitted.) - Resource Requests Limits - -------- -------- ------ - cpu 100m (10%) 0 (0%) - memory 70Mi (3%) 170Mi (8%) - ephemeral-storage 0 (0%) 0 (0%) -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Starting 56m kube-proxy Starting kube-proxy. - Normal Starting 55m kubelet Starting kubelet. - Warning InvalidDiskCapacity 55m kubelet invalid capacity 0 on image filesystem - Normal NodeHasSufficientPID 55m (x2 over 55m) kubelet Node qemux86-64 status is now: NodeHasSufficientPID - Normal NodeHasSufficientMemory 55m (x2 over 55m) kubelet Node qemux86-64 status is now: NodeHasSufficientMemory - Normal NodeHasNoDiskPressure 55m (x2 over 55m) kubelet Node qemux86-64 status is now: NodeHasNoDiskPressure - Normal NodeAllocatableEnforced 55m kubelet Updated Node Allocatable limit across pods - Normal NodeReady 54m kubelet Node qemux86-64 status is now: NodeReady - Normal Starting 52m kube-proxy Starting kube-proxy. - Normal NodeReady 50m kubelet Node qemux86-64 status is now: NodeReady - Normal NodeAllocatableEnforced 50m kubelet Updated Node Allocatable limit across pods - Warning Rebooted 50m kubelet Node qemux86-64 has been rebooted, boot id: a4e4d2d8-ddb4-49b8-b0a9-e81d12707113 - Normal NodeHasSufficientMemory 50m (x2 over 50m) kubelet Node qemux86-64 status is now: NodeHasSufficientMemory - Normal Starting 50m kubelet Starting kubelet. - Normal NodeHasSufficientPID 50m (x2 over 50m) kubelet Node qemux86-64 status is now: NodeHasSufficientPID - Normal NodeHasNoDiskPressure 50m (x2 over 50m) kubelet Node qemux86-64 status is now: NodeHasNoDiskPressure - Normal NodeNotReady 17m kubelet Node qemux86-64 status is now: NodeNotReady - Warning InvalidDiskCapacity 15m (x2 over 50m) kubelet invalid capacity 0 on image filesystem - Normal Starting 12m kube-proxy Starting kube-proxy. - Normal Starting 10m kubelet Starting kubelet. - Warning InvalidDiskCapacity 10m kubelet invalid capacity 0 on image filesystem - Normal NodeAllocatableEnforced 10m kubelet Updated Node Allocatable limit across pods - Warning Rebooted 10m kubelet Node qemux86-64 has been rebooted, boot id: f5ddf6c8-1abf-4aef-9e29-106488e3c337 - Normal NodeHasSufficientMemory 10m (x2 over 10m) kubelet Node qemux86-64 status is now: NodeHasSufficientMemory - Normal NodeHasNoDiskPressure 10m (x2 over 10m) kubelet Node qemux86-64 status is now: NodeHasNoDiskPressure - Normal NodeHasSufficientPID 10m (x2 over 10m) kubelet Node qemux86-64 status is now: NodeHasSufficientPID - Normal NodeReady 10m kubelet Node qemux86-64 status is now: NodeReady -``` diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent b/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent deleted file mode 100644 index b5c60e628dc..00000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent +++ /dev/null @@ -1,512 +0,0 @@ -#!/bin/sh -set -e - -# OpenEuler Embedded K3s Agent Deploy Script -# Modified from k3s-io install.sh -# -# Usage: -# curl ... | ENV_VAR=... sh - -# or -# ENV_VAR=... ./install.sh -# -# **Only if you know what you are doing, you can use this script to install k3s agent on OpenEuler Embedded. -# -# What is hardcoded: -# INSTALL_K3S_CHANNEL=stable -# INSTALL_K3S_SKIP_SELINUX_RPM=true -# INSTALL_K3S_TYPE=exec (not notify for server) -# INSTALL_K3S_EXEC="agent " -# Example: -# Installing an agent with airgap images: -# AIRGAP_IMAGES_DIR=/path/to/images K3S_URL=https://server-url:6443 K3S_TOKEN=xxx k3s-install -# Installing an agent with custom container runtime: -# K3S_URL=https://server-url:6443 K3S_TOKEN=xxx CONTAINER_RT_EP=unix:///custom/path.sock k3s-install -# -# Environment variables: -# - K3S_URL (REQUIRED) -# The URL of the k3s server to connect to in agent mode. Must use HTTPS protocol. -# -# - K3S_TOKEN/K3S_TOKEN_FILE (REQUIRED) -# Cluster secret token for agent authentication. At least one must be set. -# CLUSTER_SECRETE is deprecated -# -# - AIRGAP_IMAGES_DIR -# Directory containing k3s-airgap-images-ARCH.tar.gz for offline installation -# -# - AIRGAP_IMAGES -# Directly specify k3s-airgap images for offline installation -# -# - ARGS -# ARGS will be after `k3s agent`, e.g.: -# ARGS="--disable=traefik" => "k3s agent --disable=traefik" -# -# - CONTAINER_RT_EP -# Container runtime endpoint type -# default CONTAINER_RT_EP=isulad => --container-runtime-endpoint unix:///var/run/isulad.sock instead of containerd.sock) -# if CONTAINER_RT_EP=embedded => use embedded k3s containerd instead. -# otherwise => --container-runtime=endpint "${CONTAINER_RT_EP}" -# -# - INSTALL_K3S_BIN_DIR -# Directory to install k3s binary and scripts (default: /usr/bin) -# -# - INSTALL_K3S_SYSTEMD_DIR -# Directory to install systemd service files (default: /etc/systemd/system) -# -# - INSTALL_K3S_BIN_DIR_READ_ONLY -# If set to true, prevents writing to INSTALL_K3S_BIN_DIR -# -# - SKIP_LOAD_AIRGAP -# If set to true, prevents loading airgap images to isula -# -# - ONLY_SET_ISULAD -# If set to true, only isulad configuration will take place -# -# Notes: -# 1. This version ONLY supports agent mode and requires: -# - systemd as process supervisor (SysV is in progress) -# - isulad as container runtime endpoint -# - Preloaded pause image (docker.io/rancher/mirrored-pause:3.6) -# 2. Automatically configures isulad: -# - Configures cni-conf-dir and cni-bin-dir for k3s -# 3. Airgap installation requires: -# - k3s-airgap-images-ARCH.tar.gz in AIRGAP_IMAGES_DIR -# - Properly configured container registry -# -# For detailed documentation about k3s, see: https://docs.k3s.io/ - -# --- helper functions for logs --- -info() { - echo '[INFO] ' "$@" -} -warn() -{ - echo '[WARN] ' "$@" >&2 -} -fatal() { - echo '[ERROR] ' "$@" >&2 - exit 1 -} - -# --- fatal if not system (future: or SysV) --- -verify_system() { - if [ -x /sbin/systemd ]; then - HAS_SYSTEMD=true - return - fi - fatal 'Can not find systemd to use as a process supervisor for k3s' -} - -# --- add quotes to command arguments --- -quote() { - for arg in "$@"; do - printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" - done -} - -# --- add indentation and trailing slash to quoted args --- -quote_indent() { - printf ' \\\n' - for arg in "$@"; do - printf '\t%s \\\n' "$(quote "$arg")" - done -} - -# --- escape most punctuation characters, except quotes, forward slash, and space --- -escape() { - printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;' -} - -# --- escape double quotes --- -escape_dq() { - printf '%s' "$@" | sed -e 's/"/\\"/g' -} - -# --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise --- -verify_k3s_url() { - case "${K3S_URL}" in - "") - fatal "missing K3S_URL" - ;; - https://*) - ;; - *) - fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})" - ;; - esac -} - -# --- setup basic environment --- -setup_env() { - - # always set OEE as agent mode - AIRGAP_DIR="${AIRGAP_IMAGES_DIR:-/etc/k3s/tools}" - rawARCH="$(uname -m)" - case "$rawARCH" in - arm) - ARCH=arm;; - aarch64|arm64) - ARCH=arm64 ;; - x86_64) - ARCH=amd64 ;; - s390x) - ARCH=s390x ;; - *) - fatal "unsupported architecture ${rawARCH}" - ;; - esac - - info "got arch=${ARCH}" - - if [ ! -n "${AIRGAP_IMAGES}" ]; then - AIRGAP_IMAGES="${AIRGAP_DIR}/k3s-airgap-images-${ARCH}.tar.gz" - fi - - - - if [ "$ONLY_SET_ISULAD" = "true" ]; then - config_isulad - exit 0 - fi - verify_k3s_url - - CONTAINER_RT_EP=${CONTAINER_RT_EP:-"isulad"} - case $CONTAINER_RT_EP in - "isulad") - CONTAINER_RUNTIME_ENDPOINT="--container-runtime-endpoint=unix:///var/run/isulad.sock" - ;; - "embedded") - CONTAINER_RUNTIME_ENDPOINT="" - ;; - *) - CONTAINER_RUNTIME_ENDPOINT="--container-runtime-endpoint=${CONTAINER_RT_EP}" - ;; - esac - - CMD_K3S=agent - CMD_K3S_EXEC="${CMD_K3S}$(quote_indent ""$@""${CONTAINER_RUNTIME_ENDPOINT}"")" - SYSTEM_NAME=k3s-${CMD_K3S} - - SUDO=sudo - if [ $(id -u) -eq 0 ]; then - SUDO= - fi - - SYSTEM_TYPE=exec - - # --- set binary and service directories - BIN_DIR=${INSTALL_K3S_BIN_DIR:-/usr/bin} - if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then - if [ -d /opt/bin ]; then - BIN_DIR=/opt/bin - fi - fi - - # --- use systemd directory if defined or create default --- - if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then - SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}" - else - SYSTEMD_DIR=/etc/systemd/system - fi - - # --- set service and environment file paths - SERVICE_K3S=${SYSTEM_NAME}.service - UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} - KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} - AGENT_KILLER=${KILL_AGENT:-${BIN_DIR}/k3s-kill-agent} - - FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} - FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env - -} - -# --- create killall script --- -create_agent_killer() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating killall script ${AGENT_KILLER}" - $SUDO tee ${AGENT_KILLER} >/dev/null << \EOF -#!/bin/sh -eu -# -# Copyright (C) 2020 Axis Communications AB -# -# SPDX-License-Identifier: Apache-2.0 - -do_unmount() { - [ $# -eq 2 ] || return - local mounts= - while read ignore mount ignore; do - case $mount in - $1/*|$2/*) - mounts="$mount $mounts" - ;; - esac - done /dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done - -ip link delete cni0 -ip link delete flannel.1 -rm -rf /var/lib/cni/ -EOF - $SUDO chmod 755 ${AGENT_KILLER} - $SUDO chown root:root ${AGENT_KILLER} -} - - -# --- create killall script --- -create_killall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating k3s agent cleaner ${KILLALL_K3S_SH}" - $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF -#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for bin in /var/lib/rancher/k3s/data/**/bin/; do - [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux -done - -set -x - -for service in /etc/systemd/system/k3s*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done - -for service in /etc/init.d/k3s*; do - [ -x $service ] && $service stop -done - -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} - -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} - -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} - -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 -} - -killtree $({ set +x; } 2>/dev/null; getshims; set -x) - -do_unmount_and_remove() { - awk -v path="$1" '$2 ~ ("^" path) { print $2 }' /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' -} - -do_unmount_and_remove '/run/k3s' -do_unmount_and_remove '/var/lib/rancher/k3s' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/run/netns/cni-' - -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete - -# Delete network interface(s) that match 'master cni0' -ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -rm -rf /var/lib/cni/ -iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore -EOF - $SUDO chmod 755 ${KILLALL_K3S_SH} - $SUDO chown root:root ${KILLALL_K3S_SH} -} - -# --- create uninstall script --- -create_uninstall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating uninstall script ${UNINSTALL_K3S_SH}" - $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF -#!/bin/sh -set -x -[ \$(id -u) -eq 0 ] || exec sudo \$0 \$@ - -${KILLALL_K3S_SH} - -if which systemctl; then - systemctl disable ${SYSTEM_NAME} - systemctl reset-failed ${SYSTEM_NAME} - systemctl daemon-reload -fi -if which rc-update; then - rc-update delete ${SYSTEM_NAME} default -fi - -rm -f ${FILE_K3S_SERVICE} -rm -f ${FILE_K3S_ENV} - -remove_uninstall() { - rm -f ${UNINSTALL_K3S_SH} -} -trap remove_uninstall EXIT - -if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then - set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L ${BIN_DIR}/\$cmd ]; then - rm -f ${BIN_DIR}/\$cmd - fi -done - -rm -rf /etc/rancher/k3s -rm -rf /run/k3s -rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s -rm -rf /var/lib/kubelet -rm -f ${BIN_DIR}/k3s -rm -f ${KILLALL_K3S_SH} - -if type yum >/dev/null 2>&1; then - yum remove -y k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -fi -EOF - $SUDO chmod 755 ${UNINSTALL_K3S_SH} - $SUDO chown root:root ${UNINSTALL_K3S_SH} -} - -create_scripts() { - create_agent_killer - create_uninstall -} - -systemd_disable() { - $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true -} - -# --- create environment file --- -create_env_file() { - info "Creating environment file ${FILE_K3S_ENV}" - UMASK=$(umask) - umask 0377 - if [ -z "${K3S_TOKEN}" ]; then - fatal "missing K3S_TOKEN." - exit -1 - fi - env | grep '^K3S_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null - env | grep -E '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null - umask $UMASK -} - -# --- verify an executable k3s binary is installed --- -verify_k3s_is_executable() { - if [ ! -x ${BIN_DIR}/k3s ]; then - fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" - fi -} - -# --- create systemd service file --- -create_systemd_service_file() { - info "Creating service file ${FILE_K3S_SERVICE}" - $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF -[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=exec -EnvironmentFile=${FILE_K3S_ENV} -KillMode=process -Delegate=yes -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStart=${BIN_DIR}/k3s \\ - ${CMD_K3S_EXEC} - -EOF -} - - -# --- enable and start systemd service --- -systemd_enable_and_start() { - info "Enabling ${SYSTEM_NAME} unit" - $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null - $SUDO systemctl daemon-reload >/dev/null - - info "Starting ${SYSTEM_NAME}" - $SUDO systemctl restart ${SYSTEM_NAME} -} - - -isulad_daemon_set() { - sed -i 's/"cni-bin-dir": "*",/"cni-bin-dir": "\/var\/lib\/rancher\/k3s\/data\/current\/bin",/' /etc/isulad/daemon.json - sed -i 's/"cni-conf-dir": "*",/"cni-conf-dir": "\/var\/lib\/rancher\/k3s\/agent\/etc\/cni\/net.d",/' /etc/isulad/daemon.json - sed -i 's/"pod-sandbox-image": "*",/"pod-sandbox-image": "docker.io\/rancher\/mirrored-pause:3.6",/' /etc/isulad/daemon.json - systemctl daemon-reload - systemctl restart isulad -} - -isulad_preload_images() { - if [ "${SKIP_LOAD_AIRGAP}" = true ]; then - info "skipped airgap images loading" - return 0 - fi - info "searching for : ${AIRGAP_IMAGES}" - if ! test -f "${AIRGAP_IMAGES}"; then - fatal "Unable to find ${AIRGAP_IMAGES} tarball" - fi - isula load -i "${AIRGAP_IMAGES}" > /dev/null -} - -config_isulad() { - isulad_daemon_set - isulad_preload_images -} - -eval set -- $(escape "${ARGS}") $(quote "${@}") - -{ - setup_env "$@" - create_scripts - create_env_file - if [ "${CONTAINER_RT_EP}" = "isulad" ]; then - config_isulad - elif [ "${CONTAINER_RT_EP}" = "embedded" ]; then - create_systemd_service_file - else - echo "customize k3s-agent.service manually for endpoint ${CONTAINER_RT_EP}" - fi - systemd_enable_and_start -} - diff --git a/meta-openeuler/recipes-containers/k3s/k3s/modules.txt b/meta-openeuler/recipes-containers/k3s/k3s/modules.txt deleted file mode 100644 index ef53706779d..00000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/modules.txt +++ /dev/null @@ -1,202 +0,0 @@ -# github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.20 -# github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7 -# github.com/containerd/aufs => github.com/containerd/aufs v1.0.0 -# github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0 -# github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1 -# github.com/containerd/console => github.com/containerd/console v1.0.2 -# github.com/containerd/containerd => github.com/k3s-io/containerd v1.5.9-k3s1 -# github.com/containerd/continuity => github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1 -# github.com/containerd/fifo => github.com/containerd/fifo v1.0.0 -# github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0 -# github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2 -# github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2 -# github.com/containerd/zfs => github.com/containerd/zfs v1.0.0 -# github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e -# github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible -# github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible -# github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 -# github.com/golang/protobuf => github.com/golang/protobuf v1.5.2 -# github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.0.5 -# github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f -# github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1 -# github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009 -# github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.3 -# github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b -# go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.1-k3s1 -# golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 -# golang.org/x/net => golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 -# golang.org/x/sys => golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e -# google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 -# google.golang.org/grpc => google.golang.org/grpc v1.38.0 -# gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 -# k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.22.6-k3s1 -# k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.22.6-k3s1 -# k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.22.6-k3s1 -# k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.22.6-k3s1 -# k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.22.6-k3s1 -# k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.22.6-k3s1 -# k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.22.6-k3s1 -# k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.22.6-k3s1 -# k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.22.6-k3s1 -# k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.22.6-k3s1 -# k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.22.6-k3s1 -# k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.6-k3s1 -# k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.6-k3s1 -# k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.6-k3s1 -# k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s2 -# k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s2 -# k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.6-k3s1 -# k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.6-k3s1 -# k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.6-k3s1 -# k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.22.6-k3s1 -# k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.22.6-k3s1 -# k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.22.6-k3s1 -# k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.22.6-k3s1 -# k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.22.6-k3s1 -# k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.22.6-k3s1 -# k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.22.6-k3s1 -# k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.22.6-k3s1 -# k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.22.6-k3s1 -# k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.22.6-k3s1 -# k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.22.6-k3s1 -# k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.22.6-k3s1 -# mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 -# github.com/Microsoft/hcsshim v0.9.2 -## explicit -# github.com/cloudnativelabs/kube-router v1.3.2 -## explicit -# github.com/containerd/cgroups v1.0.1 -## explicit -# github.com/containerd/containerd v1.6.0-beta.2.0.20211117185425-a776a27af54a -## explicit -# github.com/containerd/fuse-overlayfs-snapshotter v1.0.4 -## explicit -# github.com/containerd/stargz-snapshotter v0.10.1 -## explicit -# github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f -## explicit -# github.com/docker/docker v20.10.10+incompatible -## explicit -# github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 -## explicit -# github.com/flannel-io/flannel v0.16.3 -## explicit -# github.com/go-bindata/go-bindata v3.1.2+incompatible -## explicit -# github.com/go-sql-driver/mysql v1.6.0 -## explicit -# github.com/golangplus/testing v1.0.0 -## explicit -# github.com/google/cadvisor v0.39.3 -## explicit -# github.com/google/uuid v1.3.0 -## explicit -# github.com/gorilla/mux v1.8.0 -## explicit -# github.com/gorilla/websocket v1.4.2 -## explicit -# github.com/k3s-io/helm-controller v0.11.7 -## explicit -# github.com/k3s-io/kine v0.8.1 -## explicit -# github.com/klauspost/compress v1.13.6 -## explicit -# github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 -## explicit -# github.com/lib/pq v1.10.2 -## explicit -# github.com/mattn/go-sqlite3 v1.14.8 -## explicit -# github.com/minio/minio-go/v7 v7.0.7 -## explicit -# github.com/natefinch/lumberjack v2.0.0+incompatible -## explicit -# github.com/onsi/ginkgo v1.16.4 -## explicit -# github.com/onsi/gomega v1.11.0 -## explicit -# github.com/opencontainers/runc v1.0.3 -## explicit -# github.com/opencontainers/selinux v1.8.2 -## explicit -# github.com/otiai10/copy v1.6.0 -## explicit -# github.com/pkg/errors v0.9.1 -## explicit -# github.com/rancher/dynamiclistener v0.3.1 -## explicit -# github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 -## explicit -# github.com/rancher/remotedialer v0.2.0 -## explicit -# github.com/rancher/wharfie v0.5.1 -## explicit -# github.com/rancher/wrangler v0.8.10 -## explicit -# github.com/robfig/cron/v3 v3.0.1 -## explicit -# github.com/rootless-containers/rootlesskit v0.14.5 -## explicit -# github.com/sirupsen/logrus v1.8.1 -## explicit -# github.com/spf13/pflag v1.0.5 -## explicit -# github.com/stretchr/testify v1.7.0 -## explicit -# github.com/tchap/go-patricia v2.3.0+incompatible -## explicit -# github.com/urfave/cli v1.22.4 -## explicit -# github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 -## explicit -# go.etcd.io/etcd/api/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/client/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/etcdutl/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/server/v3 v3.5.1 -## explicit -# golang.org/x/crypto v0.0.0-20211202192323-5770296d904e -## explicit -# golang.org/x/net v0.0.0-20211209124913-491a49abca63 -## explicit -# golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e -## explicit -# google.golang.org/grpc v1.42.0 -## explicit -# gopkg.in/yaml.v2 v2.4.0 -## explicit -# inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 -## explicit -# k8s.io/api v0.22.6 -## explicit -# k8s.io/apimachinery v0.22.6 -## explicit -# k8s.io/apiserver v0.22.6 -## explicit -# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible -## explicit -# k8s.io/cloud-provider v0.22.6 -## explicit -# k8s.io/code-generator v0.22.6 -## explicit -# k8s.io/component-base v0.22.6 -## explicit -# k8s.io/controller-manager v0.22.6 -## explicit -# k8s.io/cri-api v0.23.0-alpha.4 -## explicit -# k8s.io/klog v1.0.0 -## explicit -# k8s.io/kubectl v0.22.6 -## explicit -# k8s.io/kubernetes v1.22.6 -## explicit -# k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a -## explicit -# sigs.k8s.io/yaml v1.2.0 -## explicit diff --git a/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend deleted file mode 100644 index 75cc276e4bb..00000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend +++ /dev/null @@ -1,30 +0,0 @@ -# enable cgroup v2 -# isulad, embedded, ... -container_runtime_endpoint = "embedded" -PV = "1.22.17-k3s1" -#full_k3s = "false" -MAX_K3S_BINARY_SIZE ?= "61000000" -GO_BUILD_LDFLAGS:append = "" - -# overwrite install_other_endpoint() to use other container_runtime_points -install_other_endpoint() { - # customize install: - # 1. install airgap images - # 2. modifiy k3s-install-agent - # 3. modifiy k3s-agent.service and k3s.service - bbplain "customize your endpoint install script!" - bbwarn "rtest" -} - -do_sizecheck() { - bbplain "checking k3s binary size at WORKDIR, modified from ./scripts/binary_size_check.sh" - SAMPLE="${build_bindir}/k3s" - BIN_SIZE=$(stat -c '%s' ${SAMPLE}) - if [ ${BIN_SIZE} -gt ${MAX_K3S_BINARY_SIZE} ]; then - bbwarn "k3s binary ${SAMPLE} size ${BIN_SIZE} exceeds max accetable size of ${MAX_K3S_BINARY_SIZE} bytes" - else - bbplain "k3s binary ${SAMPLE} size ${BIN_SIZE} is less than max acceptaable size of ${MAX_K3S_BINARY_SIZE} bytes" - fi -} - -addtask do_sizecheck after do_install diff --git a/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb b/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb deleted file mode 100644 index 2ce51e556b8..00000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb +++ /dev/null @@ -1,439 +0,0 @@ -SUMMARY = "Production-Grade Container Scheduling and Management" -DESCRIPTION = "Lightweight Kubernetes, intended to be a fully compliant Kubernetes." -HOMEPAGE = "https://k3s.io/" -LICENSE = "Apache-2.0" -S = "${WORKDIR}/${BP}" -LIC_FILES_CHKSUM = "file://${S}/LICENSE;md5=2ee41112a44fe7014dce33e26468ba93" -APPEND += " cgroup_no_v1=all" - -PV = "v1.22.17-k3s1" - -python() { - if d.getVar('TUNE_PKGARCH') == 'aarch64': - d.setVar('ARCH', 'arm64') - elif d.getVar('TUNE_PKGARCH') == 'x86_64': - d.setVar('ARCH', 'amd64') - else: - d.setVar('ARCH', d.getVar('TUNE_PKGARCH')) -} - -#file://0001-Finding-host-local-in-usr-libexec.patch;patchdir=src/import - -do_compile[network] = "1" -SRC_URI = "\ - file://oee_archive/${BP}.tar.gz \ - file://k3s-agent.service \ - file://k3s.service \ - file://k3s-kill-agent \ - file://k3s-killall.sh \ - file://modules.txt \ - file://k3s-install-agent \ - file://install.sh \ - file://k3s-rootless.service \ -" - -#include src_uri.inc - -inherit go -inherit goarch -inherit systemd -inherit features_check - -PACKAGECONFIG = "" -GO_IMPORT = "import" -full_k3s ?= "true" -# runtime_endpoint: isulad => isulad, embedded => k3s-embedded containerd, others -container_runtime_endpoint ?= "isulad" -PKG = "github.com/k3s-io/k3s" -PKG_CONTAINERD="github.com/containerd/containerd" -PKG_K3S_CONTAINERD="github.com/k3s-io/containerd" -PKG_CRICTL="github.com/kubernetes-sigs/cri-tools/pkg" -PKG_K8S_BASE="k8s.io/component-base" -PKG_K8S_CLIENT="k8s.io/client-go/pkg" -PKG_CNI_PLUGINS="github.com/containernetworking/plugins" - -# PKG = "github.com/k3s-io/k3s" -# PKG_CONTAINERD = "github.com/containerd/containerd" -# PKG_K3S_CONTAINERD = "github.com/k3s-io/containerd" -# PKG_CRICTL = "github.com/kubernetes-sigs/cri-tools/pkg" -# PKG_K8S_BASE = "k8s.io/component-base" -# PKG_K8S_CLIENT = "k8s.io/client-go/pkg" -# PKG_CNI_PLUGINS = "github.com/containernetworking/plugins" -COMMIT = "3ed243d" -#COMMIT = "3ed243d" -#COMMIT = "3d82902b" from k3s-io upstream - -# because of the limits of variable expansion in shell-style function, -# set k3s-building flags inside `do_compile` is troublesome -VERSION_FLAGS="\ - -X ${PKG}/pkg/version.Version=${PV} \ - -X ${PKG}/pkg/version.GitCommit=${COMMIT} \ -" - -STATIC = " -extldflags '-static' " -# `-lz` : adding zlib to sysroot -STATIC_SQLITE = " -extldflags '-static -lm -lz -ldl -lpthread' " -K3S_LDFLAGS = " ${VERSION_FLAGS} -w -s" - -# tags -# BASIC = "ctrd,seccomp,no_btrfs,netcgo,osusergo,providerless" -BASIC = "no_btrfs,ctrd,netcgo,osusergo,providerless" -SELINUX = "selinux" -APPARMOR = "apparmor" -STATIC_TAG = "static_build" -SECCOMP = "seccomp" -SQLITE_TAG = "${STATIC_TAG},libsqlite3" -TAGS = "${BASIC}" - -# TODO fix compilation abortion when applied sqlite to k3s server compilation -SERVER_LDFLAGS = "${K3S_LDFLAGS} ${STATIC_SQLITE}" -AGENT_LDFLAGS = "${K3S_LDFLAGS} ${STATIC}" -SERVER_TAGS = "${TAGS}" -AGENT_TAGS = "${TAGS}" - -AGENT_TAGS .= ",${STATIC_TAG},${APPARMOR}" - -k3s_bindir = "${exec_prefix}/bin" -build_dir = "${S}/build" -build_bindir = "${build_dir}/bin/${BPN}" -build_srcdir = "${build_dir}/pkg/mod" -vendor_path = "${S}/vendor" -modules_path = "${S}/modules" -REQUIRED_DISTRO_FEATURES = "seccomp" -DEPENDS += "\ - rsync-native \ - coreutils-native \ - go-native \ - zlib \ -" -export GOPROXY="https://goproxy.cn,direct" - - -do_compile() { - - #build K3S binary - # go module will cache module@ver under the dir: vendor_gopath/pkg/mod - # I use vendor dir under the module mod for convinence, but it is a confussing action - # TODO: module is module, vendor is vendor. - # download path: - # go mod vendor => ${S}/vendor/sites/packages - # GO111MODULE=on => ${S}/vendor/pkg/mod/sites/packages - export CGO_ENABLED="1" - export GO111MODULE=on - export _GOPATH=${GOPATH} - export GOPATH="${modules_path}:${vendor_path}:${STAGING_DIR_TARGET}/${prefix}/lib64/go" - export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" - export CGO_LDFLAGS="${LDFLAGS} --sysroot=${STAGING_DIR_TARGET}" - export CFLAGS="" - export LDFLAGS=" -w -s" - export CC="${CC}" - export LD="${LD}" - export OEE_YOCTO_VERSION="${PV}-oee+yocto" - - cd "${S}" - # mkdir -p "${vendor_path}" - mkdir -p "${modules_path}" - mkdir -p "${build_bindir}" - mkdir -p "${WORKDIR}/build/pkg/mod" - - # if [ ! -e "${vendor_path}" ]; then - # ${GO} mod vendor 2> ${WORKDIR}/temp/k3s_vendor_info - # fi - - # useless building scripts - rm -f "Dockerfile.*" - # GO will find same package in both recipes-sysroot and vendor dir. - # pointing modules/pkg/mod/src to vendor dir - rm -f "${build_bindir}/k3s-agent" - rm -f "${build_bindir}/k3s-server" - rm -f "${build_bindir}/k3s-etcd-snapshot" - rm -f "${build_bindir}/k3s-secrets-encrypt" - rm -f "${build_bindir}/k3s-certificate" - rm -f "${build_bindir}/k3s-completion" - rm -f "${build_bindir}/kubectl" - rm -f "${build_bindir}/crictl" - rm -f "${build_bindir}/containerd" - rm -f "${build_bindir}/containerd-shim" - rm -f "${build_bindir}/containerd-shim-runc-v2" - rm -f "${build_bindir}/runc" - rm -rf "${builds}/data" - - # TODO: fixing linking warnings - # * warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking - # * warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking - # NOTICE In k3s version 1.22 and later, k3s-io has removed the building for the - # standalone k3s-agent binary,resulting in an increase of a little in the binary size. - # both two products are stored in /var/lib/rancher/k3s/data/SomeHash/bin - if [ "${full_k3s}" = "true" ]; then - ${GO} build -mod=readonly -tags "${SERVER_TAGS}" -ldflags "${SERVER_LDFLAGS}" \ - -trimpath -v -o ${build_bindir}/containerd ./cmd/server/main.go \ - 2> ${WORKDIR}/temp/k3s_build_info - else - ${GO} build -mod=readonly -tags "${AGENT_TAGS}" -ldflags "${AGENT_LDFLAGS}" \ - -trimpath -v -o ${build_bindir}/k3s-agent ./cmd/agent/main.go \ - 2> ${WORKDIR}/temp/k3s_build_info - fi - - # make sure cached package tree modificable without 'Permission Denided' - chmod 777 -R "${modules_path}/pkg/mod" - chmod 777 -R "${WORKDIR}/build/pkg/mod" - - # TODO move them into containerd recipe instead. - # build_runc - # build_containerd_shim_runc_v2 - - # ? 如果要构建非 isulad 作runtime的k3s,就需要把 vendor和module混用起来。 - # 1. 不希望在do_compile时重复拉取依赖。 - # 2. 其他组件多数是vendor构建的。 - # bbfatal "stop" - # ${GO} generate - export GO111MODULE=off - # FATAL: - # cmd/k3s/main.go:181:16: undefined: data.AssetNames - # cmd/k3s/main.go:218:23: undefined: data.Asset - # under ${S}, pkg/data/data.go is empty indeed - # ?? 位于${S}下，编译会出现上面的错误 - # 源码是从 FROM=yocto-meta-openeuler/../k3s 获取,复制到${S} - # 在FROM编译，则没有问题； - # 使用sysroot-native的go, host的go,都一样 - # 从${FROM}前往${S}, 失败，从${S}返回到${FROM}, 成功, - # CGO_ENABLED=0 ${GO} build -tags "urfave_cli_no_docs" -ldflags "${K3S_LDFLAGS} ${STATIC}"\ - # -v -o "${PN}-${ARCH}" ./cmd/k3s/main.go 2> ${WORKDIR}/temp/cli_build_info - # 手动加打印也确认了用的是pkg/data/data.go -} - -build_runc() { - RUNC_TAGS="${APPARMOR},${SECCOMP}" - RUNC_STATIC="static" - # go vendor build. - bbplain "[INFO] Building runc" - CGO_ENABLE=0 GO111MODULE=off make GOPATH=${build_dir} EXTRA_LDFLAGS=" -w -s" BUILDTAGS="$RUNC_TAGS" \ - -C "${vendor_path}/github.com/opencontainers/runc" ${RUNC_STATIC} - cp -f "${vendor_path}/github.com/opencontainers/runc/runc" ${build_bindir}/runc -} - -build_containerd_shim() { - bbplain "Building containerd-shim" - rm -r "${build_bindir}/containerd-shim" - cd "${vendor_path}" - CGO_ENABLE=0 GO111MODULE=off make -C ./github.com/containerd/containerd bin/containerd-shim - cp -f ./github.com/containerd/containerd/bin/containerd-shim ${build_bindir}/containerd-shim -} - -build_containerd_shim_runc_v2() { - bbplain "Building containerd-shim-runc-v2" - rm -r "${build_bindir}/containerd-shim-runc-v2" - cd "${modules_path}" - # CGO_ENABLE=0 GO111MODULE=off make -C ./github.com/containerd/containerd bin/containerd-shim-runc-v2 - CGO_ENABLE=0 GO111MODULE=off make -C "./github.com/k3s-io/containerd@*"/cmd/containerd-shim-runc-v2 bin/containerd-shim-runc-v2 - cp -f ./github.com/containerd/containerd/bin/containerd-shim-runc-v2 ${build_bindir}/containerd-shim-runc-v2 -} - -# compress and make a multicall binary (a single binary contains bunches of subprocess) -do_multicall() { - # mkdir -p "${build_dir}/data ${build_dir}/out" - # mkdir -p "${S}/dist/artifacts" - - # tar cvf "${build_dir}/out/data.tar" "${build_dir}" - # zstd --no-progress -T0 -16 -f --long=25 --rm "${build_dir}/out/data.tar.zst" - # HASH=$(sha256sum "${build_dir}/out/data.tar.zst" | awk '{print $1}') - # cp "${build_dir}/out/data.tar.zst" "${build_dir}/data/${HASH}.tar.zst" - - # build multicall k3s binary, this is what will be installed into usr/bin - bbplain "[INFO] multicall built" -} - -addtask do_multicall after do_compile before do_install - - -do_airgap() { - bbfatal "it is super troublesome to build airgap images in the oebuild docker environment, \ - if you wanna build k3s airgap image manually, you should run \ - `bitbake k3s-airgap` in the **native host** outside oebuild docker environment" -} - -# addtask do_airgap after do_multicall before do_install - -do_install() { - - install -d "${D}${k3s_bindir}" - install -d "${D}${sysconfdir}" - install -d "${D}${sysconfdir}/k3s" - install -d "${D}${sysconfdir}/k3s/config" - # airgap images for isulad will installed here - install -d "${D}${sysconfdir}/k3s/tools" - install -d "${D}${sysconfdir}/k3s/tools" - # install dir for airgap images when use embedded containerd as endpoint - install -d "${D}${localstatedir}/lib/rancher/k3s/agent/images" - - - bbplain "container runtime endpoint = ${container_runtime_endpoint}" - bbplain "full_k3s = ${full_k3s}" - - # install binaries - # only if `-k` option is passed, the search for talternatives works after the failure of do_compile - # you should add k3s prebuild binary to SRC_URI - if [ ! -f "${build_bindir}/containerd" ] && \ - [ ! -f "${build_bindir}/k3s" ] && \ - [ -f "${WORKDIR}/k3s"]; then - echo "in ${build_bindir} products does not exist, use a prebuild standard binary as alternative." - if [ -f "${WORKDIR}/k3s-${ARCH}-${PV}" ]; then - mv "${WORKDIR}/k3s-${ARCH}-${PV}" "${build_bindir}/k3s" - else - echo "missing prebuild binary" - exit -1 - fi - fi - - install -m 755 "${WORKDIR}/k3s-kill-agent" "${D}${k3s_bindir}" - # check-config, which is temporaliy useless for oee because currently oee image - # is production-level, does not contain a config file in rootfs - install -m 755 "${S}/contrib/util/check-config.sh" "${D}${k3s_bindir}/check-config" - - # next commit : remove full_k3s, support k3s agent only and make binary smaller - if [ "${full_k3s}" = "true" ]; then - install -m 755 "${build_bindir}/containerd" "${D}${k3s_bindir}" - #TODO crictl conflicts - #ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/crictl" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/ctr" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/kubectl" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-agent" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-server" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-completion" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-certificate" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-secrets-encrpyt" - # etcd does not well work in some k3s at lower version - #ln -sr "${D}${k3s_bindir}/k3s" "${D}${k3s_bindir}/k3s-etcd-snapshot" - install -m 755 "${WORKDIR}/k3s-killall.sh" "${D}${k3s_bindir}" - else - install -m 755 "${build_bindir}/k3s-agent" "${D}${k3s_bindir}" - fi - install -D -m 755 "${WORKDIR}/install.sh" "${D}${k3s_bindir}/k3s-official-install" - - # install script for tests - install -D -m 755 "${WORKDIR}/k3s-install-agent" "${D}${k3s_bindir}/k3s-install-agent" - - - # install systemd services unit files - systemd_launched=${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)} - if [ "$systemd_launched" = "true" ]; then - unitfile_destdir="${D}${sysconfdir}/systemd/system" - install -d "${unitfile_destdir}" - if [ "${full_k3s}" = "true" ]; then - install -D -m 0644 "${WORKDIR}/k3s.service" "${unitfile_destdir}/k3s.service" - sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_destdir}/k3s.service" - fi - - install -D -m 0644 "${WORKDIR}/k3s-agent.service" "${unitfile_destdir}/k3s-agent.service" - sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_destdir}/k3s-agent.service" - - if [ "${container_runtime_endpoint}" = "isulad" ]; then - sed -i "s#^ExecStart=\(.*\)#ExecStart=\1\n\t --container-runtime-endpoint unix:///var/run/isulad.sock #" "${unitfile_destdir}/k3s-agent.service" - sed -i "s/^Documentation=.*$/&\nRequires=isulad.service /" "${unitfile_destdir}/k3s-agent.service" - sed -i "s/^Requires=isulad.service/&\nAfter=isulad.service/" "${unitfile_destdir}/k3s-agent.service" - elif [ "${container_runtime_endpoint}" != "embedded" ]; then - # customize your endpoint setup functions in k3s_%.bbappend - install_other_endpoint - fi - - else - echo "Systemd-free k3s hasn't implemented!" - install -d "${D}${sysconfdir}/init.d" - install -d "${D}${sysconfdir}/rcS.d" - exit -1 - fi -} - - - -install_other_endpoint() { - bbwarn " \ - Using ${container_runtime_endpoint} as the container runtime endpoint \ - You should overwrite install_more_endpoint function in k3s_%.bbappend \ - to customize your endpoint setup functions \ - " -} - -compress_binaries() { - bbplain "compressing binaries" -} - -FULL_K3S_FILES = "\ - ${k3s_bindir}/crictl \ - ${k3s_bindir}/kubectl \ - ${k3s_bindir}/ctr \ - ${k3s_bindir}/k3s-secrets-encrypt \ - ${k3s_bindir}/k3s-certificate \ - ${k3s_bindir}/k3s-server \ - ${k3s_bindir}/k3s-killall.sh \ - ${unitfile_destdir}/k3s.service \ -" - -FILES:${PN} += "\ - ${k3s_bindir}/k3s-install-agent \ - ${k3s_bindir}/k3s-kill-agent \ - ${unitfile_destdir}/k3s-agent.service \ - ${bb.utils.contains('full_k3s', 'true', '${FULL_K3S_FILES}', '', d)} \ -" - -#${sysconfdir}/systemd/system/k3s-agent.service, -#${bb.utils.contains('full_k3s', 'true', 'k3s.service', '', d)} -SYSTEMD_SERVICE:k3s = " k3s-agent.service \ - k3s.service\ -" - -USE_PREBUILD_SHIM_V2 = "${@bb.utils.contains_any('ARCH', 'arm riscv64', '0', '1', d)}" - -RDEPENDS:${PN} = "\ - conntrack-tools \ - coreutils \ - findutils \ - iproute2 \ - iptables \ - ${@bb.utils.contains('USE_PREBUILD_SHIM_V2', '1', 'lib-shim-v2-bin', 'lib-shim-v2', d)} \ -" - -RDEPENDS:${PN} += "\ - kernel-module-br-netfilter \ - kernel-module-bridge \ - kernel-module-iptable-mangle \ - kernel-module-ip6table-mangle \ - kernel-module-libcrc32c \ - kernel-module-nf-conntrack \ - kernel-module-nf-nat \ - kernel-module-nf-defrag-ipv4 \ - kernel-module-nf-defrag-ipv6 \ - kernel-module-stp \ - kernel-module-xt-addrtype \ - kernel-module-xt-comment \ - kernel-module-xt-nat \ - kernel-module-xt-tcpudp \ -" - -RRECOMMENDS:${PN} = "\ - kernel-module-nf-conntrack-netlink \ - kernel-module-nfnetlink-log \ - kernel-module-nfnetlink \ - kernel-module-nft-chain-nat \ - kernel-module-nft-compat \ - kernel-module-nft-counter \ - kernel-module-xt-addrtype \ - kernel-module-xt-comment \ - kernel-module-xt-connmark \ - kernel-module-xt-conntrack \ - kernel-module-xt-limit \ - kernel-module-xt-mark \ - kernel-module-xt-masquerade \ - kernel-module-xt-multiport \ - kernel-module-xt-nflog \ - kernel-module-xt-physdev \ - kernel-module-xt-nat \ - kernel-module-xt-statistic \ - kernel-module-xt-conntrack \ - kernel-module-xt-statistic \ - kernel-module-xt-physdev \ - kernel-module-vxlan \ -" - -INHIBIT_PACKAGE_STRIP = "1" -INSANE_SKIP:${PN} += "ldflags already-stripped" diff --git a/meta-openeuler/recipes-core/images/openeuler-image.bb b/meta-openeuler/recipes-core/images/openeuler-image.bb index 9a5f69b3aac..9690260805e 100644 --- a/meta-openeuler/recipes-core/images/openeuler-image.bb +++ b/meta-openeuler/recipes-core/images/openeuler-image.bb @@ -25,7 +25,7 @@ ${@bb.utils.contains("DISTRO_FEATURES", "mcs", "packagegroup-mcs", "",d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "ros", "packagegroup-ros", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "hmi", "packagegroup-hmi", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "kubeedge isulad", "packagegroup-kubeedge", "", d)} \ -${@bb.utils.contains("DISTRO_FEATURES", "k3s isulad", "k3s packagegroup-k3s", "", d)} \ +${@bb.utils.contains_any("DISTRO_FEATURES", "k3s-agent k3s-server", "packagegroup-k3s", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "isulad", "packagegroup-isulad", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "preempt-rt", "packagegroup-preempt-rt", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "dsoftbus", "packagegroup-dsoftbus", "", d)} \ diff --git a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb index 80f7b06b6e4..1388f06b174 100644 --- a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb +++ b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb @@ -5,16 +5,27 @@ PACKAGE_ARCH = "${MACHINE_ARCH}" PR = "r1" inherit packagegroup +inherit cni_networking REQUIRED_DISTRO_FEATURES ?= "seccomp" -PACKAGES = " \ - ${PN} \ + +PACKAGES = "\ + ${PN}-server \ + ${PN}-agent \ + " + +RPROVIDES:${PN} = " \ + ${PN}-server \ + ${PN}-agent \ " -RDEPENDS:${PN} = " \ -packagegroup-isulad \ -cni \ -cri-tools \ +RDEPENDS:${PN}-server = " \ + packagegroup-oci \ + k3s-server \ " +RDEPENDS:${PN}-agent = " \ + packagegroup-oci \ + k3s-agent \ +" diff --git a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.scc b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.scc index 59949546ad9..b262a3220c2 100644 --- a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.scc +++ b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.scc @@ -1,4 +1,4 @@ define KFEATURE_DESCRIPTION "Enable k3s runtime related configs" define KFEATURE_COMPATIBILITY all - +include net/ip_vs.scc kconf non-hardware k3s.cfg diff --git a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.cfg b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.cfg new file mode 100644 index 00000000000..8c7090da6e1 --- /dev/null +++ b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.cfg @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: MIT +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_TAB_BITS=12 + +# +# IPVS transport protocol load balancing support +# +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_AH_ESP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y + +# +# IPVS scheduler +# +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_LC=m +CONFIG_IP_VS_WLC=m +CONFIG_IP_VS_LBLC=m +CONFIG_IP_VS_LBLCR=m +CONFIG_IP_VS_DH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_SED=m +CONFIG_IP_VS_NQ=m + +# +# IPVS SH scheduler +# +CONFIG_IP_VS_SH_TAB_BITS=8 + +# +# IPVS application helper +# diff --git a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.scc b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.scc new file mode 100644 index 00000000000..bb03308802a --- /dev/null +++ b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/net/ip_vs.scc @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: MIT +define KFEATURE_DESCRIPTION "Enable IP virtual server" +define KFEATURE_COMPATIBILITY all + +kconf non-hardware ip_vs.cfg -- Gitee From c4674e2e34625df564d94a219279c1d14703c39c Mon Sep 17 00:00:00 2001 From: egg12138 Date: Wed, 19 Nov 2025 17:15:09 +0800 Subject: [PATCH 3/7] k3s: refacor recipes structure This update centralizes k3s configuration and variant mappings into a shared k3s-config.inc file. Key improvements: - Centralized version and container engine mapping logic - Flexible dependency file and SRC_URI items selection based on k3s variants - Updated README with simple instructions - Fixed agent binary build target selection - Fix typos Signed-off-by: egg12138 --- .../recipes-containers/k3s/README.md | 20 ++--- .../recipes-containers/k3s/k3s-config.inc | 79 +++++++++++++++++++ .../recipes-containers/k3s/k3s_%.bbappend | 63 ++++++--------- .../packagegroups/packagegroup-k3s.bb | 3 +- 4 files changed, 116 insertions(+), 49 deletions(-) create mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md index 7b07b746622..fa42ec36e35 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/README.md @@ -1,28 +1,28 @@ -# k3s：轻量级 Kubernetes 简明指南 +# yocto k3s 这个目录提供了在 openEuler Embedded 上构建和运行 [k3s](https://k3s.io/) 所需的 BitBake 配方、补丁和运行时脚本。k3s 是基于 Apache License 2.0 的精简版 Kubernetes，非常适合边缘和资源受限设备。 + 我们重写了meta-virtualization的k3s配方，方便在 openEuler Yocto 构建体系里直接使用。 --- ## 这套配方能够做什么 -- **一次构建输出 server 与 agent**：`k3s` 多路复用二进制会同时提供 `kubectl`、`crictl` 和 `ctr`（在使用外部 containerd 时自动跳过 `ctr`）。 -- **按需切换容器运行时**：通过一个变量就能指定使用 isulad、外部 containerd，或者保留 k3s 自带的 bundle containerd。 -- **可控的依赖获取方式**：既支持在 `do_fetch` 阶段完成 go module 下载，也支持在 `do_compile` 阶段联网下载依赖。 -- **开箱即可的运行时脚本**：包括安装 agent、清理节点、停止服务等常用脚本，并提供 systemd 单元文件。 +- **一次构建输出 server 或 agent**：`k3s` 多路复用二进制会同时提供 `kubectl`、`crictl` 和 `ctr`（在使用外部 containerd 时自动跳过 `ctr`）。 +- **按需切换容器运行时**：可以设定 isulad、外部 containerd 作为 k3s external endpoint，或者默认使用 k3s 自带的 bundle containerd。 +- **可控的依赖获取方式**：考虑不同的网络情况，支持在 `do_fetch` 阶段完成 go module 下载，也支持在 `do_compile` 阶段联网下载依赖。 --- ## 快速开始 1. 在 oebuild generate 中添加 k3s feature, 默认启用 k3s-agent -2. 可以在 `local.conf` 中添加 DISTRO_FEATURES:append = "k3s-server", 来构建 k3s server -3. 直接运行： +2. 可以在 `local.conf` 中将 DISTRO_FEATURES:append = "k3s-agent" 改为 "k3s-server", 来构建 完整的k3s server二进制，默认静态链接分发 +3. 运行： ```bash bitbake k3s ``` - 默认会构建带 bundle containerd 的版本（当前为 v1.27.15-rc2+k3s1）。 + 默认会构建带 bundle containerd 的版本（当前为 v1.27.15-rc2+k3s1）, isulad作为外部endpoint时k3s的版本为v1.22.6 --- @@ -34,7 +34,7 @@ ```conf # conf/local.conf -K3S_EXTERNAL_ENDPOINT ?= "isulad" # 也可以是 containerd +K3S_EXTERNAL_ENDPOINT ?= "containerd" # "containerd", "isulad" or "bundle-containerd"(默认) ``` - 设为 `isulad` 或 `containerd`：选择对应版本并生成带 `--container-runtime-endpoint` 的 systemd 配置。 @@ -140,7 +140,7 @@ k3s-agent.sh -t -s https://:6443 --- -## 还想了解更多？ +## 更多 - k3s 官方文档： - 网络选项（CNI、Flannel 后端等）： diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc new file mode 100644 index 00000000000..11beac81773 --- /dev/null +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc @@ -0,0 +1,79 @@ +# Common k3s configuration shared between packagegroup-k3s and k3s recipe +# +# This file defines the mapping between container engines and their +# corresponding k3s versions and packagegroups. + + +# External container engine flag +# Set to "isulad" or "containerd" to use external container runtime. +# Set to "" (empty) to use k3s-bundled containerd (default). +# When set, k3s will be configured with --container-runtime-endpoint. +K3S_EXTERNAL_ENDPOINT ?= "" +K3S_ROLE ?= "agent" +# used for downloading dependencies during do_fetch +K3S_DEP_SRC_URI_FILE ?= "" +K3S_DEP_RELOCATION_FILE ?= "" +K3S_DEP_MODULES_TXT ?= "" + +# building options: +# k3s additional abilities: +apparmor="0" +selinux="0" +# hightly recommended to statically build +static_build="1" + +# Use prebuilt k3s binary from k3s.io instead of building from source +# Set to "1" to enable using prebuilt binaries (much faster) +# Remain empty to build from source (default) +K3S_PREBUILD_BINARY ?= "" +K3S_MIRROR_URL ?= "" +upx_compress = "false" +K3S_ARCH:x86-64 = "amd64" +K3S_ARCH:arm = "arm" +K3S_ARCH:aarch64 = "arm64" + + + +python () { + endpoint = (d.getVar('K3S_EXTERNAL_ENDPOINT') or '').strip() + d.setVar('K3S_EXTERNAL_ENDPOINT', endpoint) + d.setVar('K3S_EFFECTIVE_EXTERNAL_ENDPOINT', endpoint) +} + + +# Mapping of container engines to k3s versions and packagegroups +# These dictionaries are used by both recipes to ensure consistency +def get_k3s_variants(d): + """Return k3s version mapping for different container engines""" + return { + "isulad": { + "branch": "release-1.22", + "pv": "v1.22.6+k3s1", + "srcrev": "4262c6b91a43ef8411870f72ff8b8715949f90e2", + "baisc_build_tags": "no_btrfs ctrd netcgo osusergo providerless", + # "pv": "v1.22.17-k3s1", + # "srcrev": "3ed243df453edc27d20b8d5efa74eb0cbba4548b", + }, + # version for both external containerd and bundled containerd + "containerd": { + "branch": "release-1.27", + "pv": "v1.27.15-rc2+k3s1", + "srcrev": "cb36c910a6d1b111d13a9e39b9158703bf8f3274", + "basic_build_tags": "urfave_cli_no_docs static_build ctrd netcgo osusergo providerless", + }, + "bundle-containerd": { + "branch": "release-1.27", + "pv": "v1.27.15-rc2+k3s1", + "srcrev": "cb36c910a6d1b111d13a9e39b9158703bf8f3274", + "basic_build_tags": "urfave_cli_no_docs static_build ctrd netcgo osusergo providerless", + } + } + +# TODO: add more like cri-o, podman, ... +# key name == container engine name +def get_container_engine_pkg(d): + ce_pkgs = { + "isulad": "packagegroup-isulad", + "containerd": "packagegroup-basic-containerd", + } + return ce_pkgs diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend index 859dbd3ae8b..7ed1f3874ed 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend @@ -5,9 +5,6 @@ SRCREV_FORMAT = "k3s" SRC_URI = "" require k3s-config.inc -# used for downloading dependencies during do_fetch -K3S_DEP_SRC_URI_FILE ?= "" -K3S_DEP_RELOCATION_FILE ?= "" python () { variants = get_k3s_variants(d) @@ -17,7 +14,7 @@ python () { if external_endpoint: selected_engine = external_endpoint bb.note("K3S: External container engine is %s, selecting %s version" % (selected_engine, selected_engine)) - else: + else: # fallback/default to bundle-containerd selected_engine = "bundle-containerd" if selected_engine not in variants: @@ -25,6 +22,8 @@ python () { selected_engine = "containerd" variant = variants[selected_engine] + pv = variant['pv'] + srcrev = variant['srcrev'] d.setVar('K3S_BRANCH', variant['branch']) d.setVar('PV', variant['pv'] + "+git" + variant['srcrev']) d.setVar('SRCREV_k3s', variant['srcrev']) @@ -32,26 +31,17 @@ python () { # Select dependency files based on container engine and binary source # When using prebuilt binary, these files are not needed (skip them) - if d.getVar('K3S_PREBUILD_BINARY') == "1": + d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-' + pv + '.inc') + d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-' + pv + '.inc') + d.setVar('K3S_DEP_MODULES_TXT', 'modules-' + pv + '.txt') + d.setVar('K3S_BUILD_TAGS', variant.get('basic_build_tags', '')) + if selected_engine == "bundle-containerd": bb.note("K3S: Using prebuilt binary, skipping dependency source/relocation files") d.setVar('K3S_DEP_SRC_URI_FILE', '') d.setVar('K3S_DEP_RELOCATION_FILE', '') - d.setVar('K3S_BUILD_TAGS', 'urfave_cli_no_docs static_build ctrd netcgo osusergo providerless') - elif selected_engine == "isulad": - d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-isulad.inc') - d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-isulad.inc') - d.setVar('K3S_BUILD_TAGS', 'no_btrfs ctrd netcgo osusergo providerless') - else: - d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-containerd.inc') - d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-containerd.inc') - d.setVar('K3S_BUILD_TAGS', 'urfave_cli_no_docs ctrd netcgo osusergo providerless') + d.setVar('K3S_DEP_MODULES_TXT', '') } -# k3s additional abilities: -apparmor="0" -selinux="0" -# hightly recommended to statically build -static_build="1" K3S_BUILD_TAGS:append = "\ ${@bb.utils.contains('apparmor', '1', 'apparmor', '', d)} \ @@ -61,6 +51,9 @@ K3S_BUILD_TAGS:append = "\ require ${K3S_DEP_SRC_URI_FILE} require ${K3S_DEP_RELOCATION_FILE} +SRC_URI_MODULES = "\ + ${@'file://${K3S_DEP_MODULES_TXT}' if d.getVar('K3S_DEP_MODULES_TXT') else ''} \ +" SRC_URI += " \ git://github.com/k3s-io/k3s.git;branch=${K3S_BRANCH};name=k3s;protocol=https \ @@ -71,7 +64,7 @@ SRC_URI += " \ file://cni-containerd-net.conf \ file://0001-Finding-host-local-in-usr-libexec.patch;patchdir=src/import \ file://k3s-killall.sh \ - file://modules.txt \ + ${SRC_URI_MODULES} \ " BIN_PREFIX = "${exec_prefix}" @@ -87,20 +80,6 @@ GO_BUILD_LDFLAGS = "-X github.com/k3s-io/k3s/pkg/version.Version=${PV} \ K3S_AGENT_BUILD_TAGS ?= "${K3S_BUILD_TAGS}" -# Use prebuilt k3s binary from k3s.io instead of building from source -# Set to "1" to enable using prebuilt binaries (much faster) -# Remain empty to build from source (default) -K3S_PREBUILD_BINARY ?= "" - -K3S_MIRROR_URL ?= "" - -upx_compress = "false" - -# Map Yocto arch to k3s arch naming -K3S_ARCH:x86-64 = "amd64" -K3S_ARCH:arm = "arm" -K3S_ARCH:aarch64 = "arm64" - do_download_prebuilt() { if [ "${K3S_PREBUILD_BINARY}" != "1" ]; then @@ -162,6 +141,15 @@ k3s_fix_gomodcache_perms() { do_compile[postfuncs] += " k3s_fix_gomodcache_perms" do_compile[prefuncs] += " k3s_fix_gomodcache_perms " +# Wanna fetch k3s dependencies dureing do_fetch(), just rewrite do_compile(): +# * we have prepared src_uri-${PV}.inc, relocation-${PV}.inc and modules-${PV}.txt files, +# hence you can keep do_fetch unchanged +# * do_compile[network] = "0" +# * comment do_compile[postfuncs] += " k3s_fix_gomodcache_perms" +# * comment do_compile[prefuncs] += " k3s_fix_gomodcache_perms" +# * mapping dependencies cache to correct location, according to the guide [yocto-meta-openeuler/scripts/oe-go-mod-autogen.py] +# or read [meta-virtualization/recipes-containers/k3s/k3s_git.bb as reference] +# * change build mode to go vendor do_compile() { if [ "${K3S_PREBUILD_BINARY}" = "1" ]; then if [ -f "${S}/src/import/dist/artifacts/k3s" ]; then @@ -188,8 +176,7 @@ do_compile() { build_output="./dist/artifacts/k3s" build_tags="${K3S_BUILD_TAGS}" if [ "${K3S_ROLE}" = "agent" ]; then - # temporary build k3s full binary instead of agentic only - # build_target="./cmd/agent/main.go" + build_target="./cmd/agent/main.go" build_tags="${K3S_AGENT_BUILD_TAGS}" fi @@ -276,8 +263,8 @@ do_install() { fi } -FILES:${k3s}-server += "${systemd_system_unitdir}/k3s.service.ori" -FILES:${k3s}-agent += "${systemd_system_unitdir}/k3s-agent.service.ori" +FILES:${PN}-server += "${systemd_system_unitdir}/k3s.service.ori" +FILES:${PN}-agent += "${systemd_system_unitdir}/k3s-agent.service.ori" # external container engine selection python () { diff --git a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb index 1388f06b174..2263e17e871 100644 --- a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb +++ b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb @@ -4,13 +4,14 @@ PACKAGE_ARCH = "${MACHINE_ARCH}" PR = "r1" -inherit packagegroup +inherit packagegroup features_check inherit cni_networking REQUIRED_DISTRO_FEATURES ?= "seccomp" PACKAGES = "\ + ${PN} \ ${PN}-server \ ${PN}-agent \ " -- Gitee From 330c21d74dc1334a4373bf019399e15faf980cd8 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Wed, 19 Nov 2025 17:41:46 +0800 Subject: [PATCH 4/7] k3s: update k3s-config.inc - migrate variable selection python func to k3s-config.inc Signed-off-by: egg12138 --- .../recipes-containers/k3s/k3s-config.inc | 36 +++++++++++++++++++ .../recipes-containers/k3s/k3s_%.bbappend | 35 ------------------ 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc index 11beac81773..434c9cf0d7c 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-config.inc @@ -77,3 +77,39 @@ def get_container_engine_pkg(d): "containerd": "packagegroup-basic-containerd", } return ce_pkgs + +python () { + variants = get_k3s_variants(d) + + external_endpoint = (d.getVar('K3S_EXTERNAL_ENDPOINT') or '').strip() + + if external_endpoint: + selected_engine = external_endpoint + bb.note("K3S: External container engine is %s, selecting %s version" % (selected_engine, selected_engine)) + else: # fallback/default to bundle-containerd + selected_engine = "bundle-containerd" + + if selected_engine not in variants: + bb.warn('Unknown selected container engine "%s", falling back to containerd' % selected_engine) + selected_engine = "containerd" + + variant = variants[selected_engine] + pv = variant['pv'] + srcrev = variant['srcrev'] + d.setVar('K3S_BRANCH', variant['branch']) + d.setVar('PV', variant['pv'] + "+git" + variant['srcrev']) + d.setVar('SRCREV_k3s', variant['srcrev']) + d.setVar('K3S_SELECTED_ENGINE', selected_engine) + + # Select dependency files based on container engine and binary source + # When using prebuilt binary, these files are not needed (skip them) + d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-' + pv + '.inc') + d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-' + pv + '.inc') + d.setVar('K3S_DEP_MODULES_TXT', 'modules-' + pv + '.txt') + d.setVar('K3S_BUILD_TAGS', variant.get('basic_build_tags', '')) + if selected_engine == "bundle-containerd": + bb.note("K3S: Using prebuilt binary, skipping dependency source/relocation files") + d.setVar('K3S_DEP_SRC_URI_FILE', '') + d.setVar('K3S_DEP_RELOCATION_FILE', '') + d.setVar('K3S_DEP_MODULES_TXT', '') +} \ No newline at end of file diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend index 7ed1f3874ed..3f5224a1019 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend @@ -6,41 +6,6 @@ SRC_URI = "" require k3s-config.inc -python () { - variants = get_k3s_variants(d) - - external_endpoint = (d.getVar('K3S_EXTERNAL_ENDPOINT') or '').strip() - - if external_endpoint: - selected_engine = external_endpoint - bb.note("K3S: External container engine is %s, selecting %s version" % (selected_engine, selected_engine)) - else: # fallback/default to bundle-containerd - selected_engine = "bundle-containerd" - - if selected_engine not in variants: - bb.warn('Unknown selected container engine "%s", falling back to containerd' % selected_engine) - selected_engine = "containerd" - - variant = variants[selected_engine] - pv = variant['pv'] - srcrev = variant['srcrev'] - d.setVar('K3S_BRANCH', variant['branch']) - d.setVar('PV', variant['pv'] + "+git" + variant['srcrev']) - d.setVar('SRCREV_k3s', variant['srcrev']) - d.setVar('K3S_SELECTED_ENGINE', selected_engine) - - # Select dependency files based on container engine and binary source - # When using prebuilt binary, these files are not needed (skip them) - d.setVar('K3S_DEP_SRC_URI_FILE', 'src_uri-' + pv + '.inc') - d.setVar('K3S_DEP_RELOCATION_FILE', 'relocation-' + pv + '.inc') - d.setVar('K3S_DEP_MODULES_TXT', 'modules-' + pv + '.txt') - d.setVar('K3S_BUILD_TAGS', variant.get('basic_build_tags', '')) - if selected_engine == "bundle-containerd": - bb.note("K3S: Using prebuilt binary, skipping dependency source/relocation files") - d.setVar('K3S_DEP_SRC_URI_FILE', '') - d.setVar('K3S_DEP_RELOCATION_FILE', '') - d.setVar('K3S_DEP_MODULES_TXT', '') -} K3S_BUILD_TAGS:append = "\ -- Gitee From 2f3b9f7dc45d14f7373c59f2c837e45b0dc49402 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Wed, 19 Nov 2025 17:48:31 +0800 Subject: [PATCH 5/7] k3s: remove useless files - remove some duplicated configuration files Signed-off-by: egg12138 --- ...inding-host-local-in-usr-libexec.patch.oee | 24 --- .../k3s/cni-containerd-net.conf.oee | 24 --- .../recipes-containers/k3s/k3s-agent.oee | 137 ------------------ .../k3s/k3s-agent.service.oee | 24 --- 4 files changed, 209 deletions(-) delete mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee delete mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee delete mode 100755 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee delete mode 100644 meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee deleted file mode 100644 index 524ac2cb73f..00000000000 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/0001-Finding-host-local-in-usr-libexec.patch.oee +++ /dev/null @@ -1,24 +0,0 @@ -From 4faf68d68c97cfd10947e1152f711acc59f39647 Mon Sep 17 00:00:00 2001 -From: Erik Jansson -Date: Wed, 16 Oct 2019 15:07:48 +0200 -Subject: [PATCH] Finding host-local in /usr/libexec - -Upstream-status: Inappropriate [embedded specific] -Signed-off-by: ---- - pkg/agent/config/config.go | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -Index: import/pkg/agent/config/config.go -=================================================================== ---- import.orig/pkg/agent/config/config.go -+++ import/pkg/agent/config/config.go -@@ -445,7 +445,7 @@ - } - - if !nodeConfig.NoFlannel { -- hostLocal, err := exec.LookPath("host-local") -+ hostLocal, err := exec.LookPath("/usr/libexec/cni/host-local") - if err != nil { - return nil, errors.Wrapf(err, "failed to find host-local") - } diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee deleted file mode 100644 index ca434d6fcdf..00000000000 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/cni-containerd-net.conf.oee +++ /dev/null @@ -1,24 +0,0 @@ -{ - "cniVersion": "0.4.0", - "name": "containerd-net", - "plugins": [ - { - "type": "bridge", - "bridge": "cni0", - "isGateway": true, - "ipMasq": true, - "promiscMode": true, - "ipam": { - "type": "host-local", - "subnet": "10.88.0.0/16", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true} - } - ] -} diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee deleted file mode 100755 index ca32b409fab..00000000000 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.oee +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/sh -eu -# -# Copyright (C) 2020 Axis Communications AB -# -# SPDX-License-Identifier: Apache-2.0 - -# -# Simplified from k3s-install.sh - -set -x - -ENV_CONF=/etc/systemd/system/k3s-agent.service.d/10-env.conf - -usage() { - echo " -USAGE: - ${0##*/} [OPTIONS] -OPTIONS: - --token value, -t value Token to use for authentication [\$K3S_TOKEN] - --token-file value Token file to use for authentication [\$K3S_TOKEN_FILE] - --server value, -s value Server to connect to [\$K3S_URL] - --node-name value Node name [\$K3S_NODE_NAME] - --resolv-conf value Kubelet resolv.conf file [\$K3S_RESOLV_CONF] - --cluster-secret value Shared secret used to bootstrap a cluster [\$K3S_CLUSTER_SECRET] - --isula-setup, -i Setup container runtime endpoint as iSulad - -h print this -" -} - -isulad_daemon_set() { - sed -i 's/"cni-bin-dir": "*",/"cni-bin-dir": "\/var\/lib\/rancher\/k3s\/data\/current\/bin",/' /etc/isulad/daemon.json - sed -i 's/"cni-conf-dir": "*",/"cni-conf-dir": "\/var\/lib\/rancher\/k3s\/agent\/etc\/cni\/net.d",/' /etc/isulad/daemon.json - sed -i 's/"pod-sandbox-image": "*",/"pod-sandbox-image": "docker.io\/rancher\/mirrored-pause:3.6",/' /etc/isulad/daemon.json -} - -isulad_preload_images() { - if [ ! -e "/etc/k3s/tools/k3s-airgap-images-arm64.tar.gz" ]; then - echo "failed to find k3s-airgap-images tarball!" - exit -1 - fi - isula load -i /etc/k3s/tools/k3s-airgap-images-arm64.tar.gz -} - -isula_set() { - isulad_preload_images - isulad_daemon_set -} - -[ $# -gt 0 ] || { - usage - exit 1 -} - -rm -f $ENV_CONF -mkdir -p ${ENV_CONF%/*} -echo [Service] > $ENV_CONF - -while getopts "t:s:ih-:" opt; do - case $opt in - h) - usage - exit - ;; - i) - isulad_daemon_set - exit - ;; - t) - VAR_NAME=K3S_TOKEN - ;; - s) - VAR_NAME=K3S_URL - ;; - -) - [ $# -ge $OPTIND ] || { - usage - exit 1 - } - opt=$OPTARG - case $opt in - token) - VAR_NAME=K3S_TOKEN - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - token-file) - VAR_NAME=K3S_TOKEN_FILE - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - server) - VAR_NAME=K3S_URL - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - node-name) - VAR_NAME=K3S_NODE_NAME - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - resolv-conf) - VAR_NAME=K3S_RESOLV_CONF - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - cluster-secret) - VAR_NAME=K3S_CLUSTER_SECRET - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - isula-setup) - isula_set - exit - ;; - help) - usage - exit - ;; - *) - usage - exit 1 - ;; - esac - ;; - *) - usage - exit 1 - ;; - esac -echo Environment=$VAR_NAME=$OPTARG >> $ENV_CONF -done - -chmod 0644 $ENV_CONF -rm -rf /var/lib/rancher/k3s/agent -systemctl daemon-reload -systemctl restart k3s-agent -systemctl enable k3s-agent.service diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee deleted file mode 100644 index a613de2311c..00000000000 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s-agent.service.oee +++ /dev/null @@ -1,24 +0,0 @@ -# Derived from the k3s install.sh's create_systemd_service_file() function -[Unit] -Description=Lightweight Kubernetes Agent -Documentation=https://k3s.io - -[Install] -WantedBy=multi-user.target - -[Service] -Type=notify -KillMode=control-group -Delegate=yes -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/k3s agent -ExecStopPost=/usr/bin/k3s-kill-agent - -- Gitee From 4c0ae1caaa64ecb636adb878d5d552d7949a1b53 Mon Sep 17 00:00:00 2001 From: egg12138 Date: Thu, 20 Nov 2025 11:07:39 +0800 Subject: [PATCH 6/7] packagegroup-k3s: move to dynamic layer - remove cni-networking inheritance - move to dynamic layer/virtualization-layer Signed-off-by: egg12138 --- .../recipes-core/packagegroups/packagegroup-k3s.bb | 1 - 1 file changed, 1 deletion(-) rename meta-openeuler/{ => dynamic-layers/virtualization-layer}/recipes-core/packagegroups/packagegroup-k3s.bb (94%) diff --git a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-k3s.bb similarity index 94% rename from meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb rename to meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-k3s.bb index 2263e17e871..435ca061ed1 100644 --- a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-core/packagegroups/packagegroup-k3s.bb @@ -5,7 +5,6 @@ PACKAGE_ARCH = "${MACHINE_ARCH}" PR = "r1" inherit packagegroup features_check -inherit cni_networking REQUIRED_DISTRO_FEATURES ?= "seccomp" -- Gitee From 7d87553269953925b40e59a976dd1aee2e1656db Mon Sep 17 00:00:00 2001 From: egg12138 Date: Thu, 20 Nov 2025 11:08:33 +0800 Subject: [PATCH 7/7] k3s: inherit cni - add cni_networking inheritance in k3s, instead of packagegroup-k3s Signed-off-by: egg12318 --- .../virtualization-layer/recipes-containers/k3s/k3s_%.bbappend | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend index 3f5224a1019..a91d5b0c9b0 100644 --- a/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend +++ b/meta-openeuler/dynamic-layers/virtualization-layer/recipes-containers/k3s/k3s_%.bbappend @@ -5,6 +5,7 @@ SRCREV_FORMAT = "k3s" SRC_URI = "" require k3s-config.inc +inherit cni_networking -- Gitee