diff --git a/Jenkinsfile b/Jenkinsfile index 3f4609d..6aaa412 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,6 +10,10 @@ def normalizeDockerTag(String value) { return value.replaceAll(/[^A-Za-z0-9_.-]/, '-') } +def shellQuote(String value) { + return "'" + (value ?: '').replace("'", "'\"'\"'") + "'" +} + def resolveSourceName(String branchName, String changeBranch, String changeId, String tagName) { if (changeId) { return changeBranch ?: "PR-${changeId}" @@ -32,11 +36,12 @@ pipeline { } parameters { + choice(name: 'DEPLOY_TARGET', choices: ['k8s', 'docker', 'none'], description: '发布目标:k8s 为标准发布链路,docker 为旧容器替换链路,none 只做 CI 和镜像构建') booleanParam(name: 'BUILD_DOCKER_IMAGE', defaultValue: true, description: '是否在非 PR 分支使用项目现有 dockerfile 构建镜像') - booleanParam(name: 'PUSH_DOCKER_IMAGE', defaultValue: false, description: '是否执行 docker push;仅发布分支生效,需要 Agent 已提前完成 docker login') - booleanParam(name: 'RUN_DOCKER_CONTAINER', defaultValue: true, description: 'Docker 镜像构建成功后是否重启业务容器;仅发布分支生效') + booleanParam(name: 'PUSH_DOCKER_IMAGE', defaultValue: true, description: '是否执行 docker push;K8s 发布会强制推送到本地 registry') + booleanParam(name: 'RUN_DOCKER_CONTAINER', defaultValue: false, description: '旧 Docker 发布链路:镜像构建成功后是否重启业务容器;仅 DEPLOY_TARGET=docker 生效') string(name: 'PUBLISH_BRANCH_PATTERN', defaultValue: '^(main|master|release/.+)$', description: '允许推送镜像的分支正则') - string(name: 'DOCKER_REGISTRY', defaultValue: '', description: '镜像仓库地址,为空时只生成本地镜像') + string(name: 'DOCKER_REGISTRY', defaultValue: 'k3d-kt-registry.localhost:5000', description: '镜像仓库地址;K8s 发布默认使用 fnOS NAS 上的 k3d 本地 registry') string(name: 'IMAGE_NAME', defaultValue: 'kt-template-online-api', description: 'Docker 镜像名称') string(name: 'IMAGE_TAG', defaultValue: '', description: '镜像标签,为空时使用 分支名-BUILD_NUMBER;PR 使用源分支名') string(name: 'CONTAINER_NAME', defaultValue: 'kt-template-online-api', description: '业务容器名称') @@ -44,6 +49,13 @@ pipeline { string(name: 'CONTAINER_ENV_FILE', defaultValue: '/home/jenkins/agent/env/kt-template-online-api/.env.production', description: 'Agent workdir 内可读取的业务 env 文件路径') string(name: 'CONTAINER_NETWORK', defaultValue: 'bridge', description: '业务容器加入的 Docker 网络,默认使用 Docker bridge') string(name: 'CONTAINER_EXTRA_ARGS', defaultValue: '', description: 'docker run 额外参数,例如 -v /host/data:/app/data') + string(name: 'KUBE_CONFIG_FILE', defaultValue: '/home/jenkins/agent/kubeconfig/kt-nas.jenkins.yaml', description: 'Agent 容器内可读取的 kubeconfig 文件路径') + string(name: 'K8S_MANIFEST_FILE', defaultValue: 'k8s/prod/api.yaml', description: 'K8s manifest 文件路径') + string(name: 'K8S_NAMESPACE', defaultValue: 'kt-prod', description: 'K8s 命名空间') + string(name: 'K8S_DEPLOYMENT', defaultValue: 'kt-template-online-api', description: 'K8s Deployment 名称') + string(name: 'K8S_CONTAINER', defaultValue: 'api', description: 'Deployment 内业务容器名称') + string(name: 'K8S_ENV_SECRET', defaultValue: 'kt-template-online-api-env', description: '由 .env.production 生成的 K8s Secret 名称') + string(name: 'K8S_ROLLOUT_TIMEOUT', defaultValue: '180s', description: 'kubectl rollout status 超时时间') } environment { @@ -71,7 +83,11 @@ pipeline { def publishPattern = params.PUBLISH_BRANCH_PATTERN?.trim() ?: '^(main|master|release/.+)$' env.IS_PUBLISH_BRANCH = (!env.CHANGE_ID && isPublishBranch(env.BRANCH_NAME ?: '', publishPattern)) ? 'true' : 'false' def registry = params.DOCKER_REGISTRY?.trim() + if (params.DEPLOY_TARGET == 'k8s' && !registry) { + error('DOCKER_REGISTRY is required when DEPLOY_TARGET=k8s.') + } env.DOCKER_IMAGE = registry ? "${registry}/${params.IMAGE_NAME}:${env.IMAGE_TAG_FINAL}" : "${params.IMAGE_NAME}:${env.IMAGE_TAG_FINAL}" + env.DOCKER_IMAGE_LATEST = registry ? "${registry}/${params.IMAGE_NAME}:latest" : "${params.IMAGE_NAME}:latest" // Agent 由 NAS 侧预先创建;这里仅确认 CI 所需的 Node/pnpm 环境可用。 if (isUnix()) { @@ -88,7 +104,20 @@ pipeline { fi pnpm --version """.stripIndent()) + + if (params.DEPLOY_TARGET == 'k8s') { + runCmd(""" + if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl is required on the Jenkins Agent when DEPLOY_TARGET=k8s." + exit 1 + fi + kubectl version --client=true + """.stripIndent()) + } } else { + if (params.DEPLOY_TARGET == 'k8s') { + error('K8s deploy requires a Linux/NAS Jenkins Agent.') + } runCmd('', """ node --version where pnpm >nul 2>nul @@ -107,6 +136,8 @@ pipeline { Change request: ${env.CHANGE_ID ?: '-'} Tag: ${env.TAG_NAME ?: '-'} Docker image: ${env.DOCKER_IMAGE} + Docker latest: ${env.DOCKER_IMAGE_LATEST} + Deploy target: ${params.DEPLOY_TARGET} Publish branch: ${env.IS_PUBLISH_BRANCH} Run container: ${params.RUN_DOCKER_CONTAINER} """.stripIndent() @@ -152,6 +183,7 @@ pipeline { allOf { expression { return params.BUILD_DOCKER_IMAGE } expression { return env.IS_CHANGE_REQUEST != 'true' } + expression { return params.DEPLOY_TARGET != 'none' } } } steps { @@ -160,11 +192,15 @@ pipeline { runCmd(""" test -f dist/main.js docker build -f dockerfile -t ${env.DOCKER_IMAGE} . + if [ '${env.DOCKER_IMAGE}' != '${env.DOCKER_IMAGE_LATEST}' ]; then + docker tag ${env.DOCKER_IMAGE} ${env.DOCKER_IMAGE_LATEST} + fi """.stripIndent()) } else { runCmd('', """ if not exist dist\\main.js exit /b 1 docker build -f dockerfile -t ${env.DOCKER_IMAGE} . + if not "${env.DOCKER_IMAGE}"=="${env.DOCKER_IMAGE_LATEST}" docker tag ${env.DOCKER_IMAGE} ${env.DOCKER_IMAGE_LATEST} """.stripIndent()) } } @@ -174,13 +210,88 @@ pipeline { stage('Docker Push') { when { allOf { - expression { return params.BUILD_DOCKER_IMAGE && params.PUSH_DOCKER_IMAGE } + expression { return params.BUILD_DOCKER_IMAGE && (params.PUSH_DOCKER_IMAGE || params.DEPLOY_TARGET == 'k8s') } + expression { return env.IS_PUBLISH_BRANCH == 'true' } + expression { return params.DEPLOY_TARGET != 'none' } + } + } + steps { + script { + if (params.DOCKER_REGISTRY?.trim()) { + runCmd(""" + docker push ${env.DOCKER_IMAGE} + docker push ${env.DOCKER_IMAGE_LATEST} + """.stripIndent()) + } else { + runCmd("docker push ${env.DOCKER_IMAGE}") + } + } + } + } + + stage('K8s Deploy') { + when { + allOf { + expression { return params.BUILD_DOCKER_IMAGE } + expression { return params.DEPLOY_TARGET == 'k8s' } + expression { return env.IS_CHANGE_REQUEST != 'true' } expression { return env.IS_PUBLISH_BRANCH == 'true' } } } steps { script { - runCmd("docker push ${env.DOCKER_IMAGE}") + if (!isUnix()) { + error('K8s Deploy stage requires a Linux/NAS Jenkins Agent.') + } + + def kubeConfigFile = params.KUBE_CONFIG_FILE?.trim() + def manifestFile = params.K8S_MANIFEST_FILE?.trim() ?: 'k8s/prod/api.yaml' + def namespace = params.K8S_NAMESPACE?.trim() ?: 'kt-prod' + def deploymentName = params.K8S_DEPLOYMENT?.trim() ?: 'kt-template-online-api' + def containerName = params.K8S_CONTAINER?.trim() ?: 'api' + def envSecret = params.K8S_ENV_SECRET?.trim() ?: 'kt-template-online-api-env' + def rolloutTimeout = params.K8S_ROLLOUT_TIMEOUT?.trim() ?: '180s' + def containerEnvFile = params.CONTAINER_ENV_FILE?.trim() + + if (!kubeConfigFile) { + error('KUBE_CONFIG_FILE is required when DEPLOY_TARGET=k8s.') + } + if (!containerEnvFile) { + error('CONTAINER_ENV_FILE is required when DEPLOY_TARGET=k8s.') + } + + def kubeConfigArg = "--kubeconfig ${shellQuote(kubeConfigFile)}" + def namespaceArg = "-n ${shellQuote(namespace)}" + def changeCause = "Jenkins ${env.JOB_NAME} #${env.BUILD_NUMBER} ${env.GIT_COMMIT ?: 'unknown'}" + + // 每次发布都从 Agent 私有 env 文件重建 Secret,避免真实配置进入 Git。 + runCmd(""" + set -e + if [ ! -f ${shellQuote(kubeConfigFile)} ]; then + echo "Kubeconfig file not found: ${kubeConfigFile}" + exit 1 + fi + if [ ! -f ${shellQuote(containerEnvFile)} ]; then + echo "Container env file not found: ${containerEnvFile}" + exit 1 + fi + if [ ! -f ${shellQuote(manifestFile)} ]; then + echo "K8s manifest file not found: ${manifestFile}" + exit 1 + fi + + kubectl ${kubeConfigArg} get namespace ${shellQuote(namespace)} >/dev/null + kubectl ${kubeConfigArg} ${namespaceArg} create secret generic ${shellQuote(envSecret)} \\ + --from-env-file=${shellQuote(containerEnvFile)} \\ + --dry-run=client -o yaml | kubectl ${kubeConfigArg} apply -f - + + kubectl ${kubeConfigArg} apply -f ${shellQuote(manifestFile)} + kubectl ${kubeConfigArg} ${namespaceArg} set image ${shellQuote("deployment/${deploymentName}")} ${shellQuote("${containerName}=${env.DOCKER_IMAGE}")} + kubectl ${kubeConfigArg} ${namespaceArg} annotate ${shellQuote("deployment/${deploymentName}")} \\ + ${shellQuote("kubernetes.io/change-cause=${changeCause}")} --overwrite + kubectl ${kubeConfigArg} ${namespaceArg} rollout status ${shellQuote("deployment/${deploymentName}")} --timeout=${shellQuote(rolloutTimeout)} + kubectl ${kubeConfigArg} ${namespaceArg} get pod,svc -l app=${shellQuote(deploymentName)} + """.stripIndent()) } } } @@ -188,6 +299,7 @@ pipeline { stage('Docker Run') { when { allOf { + expression { return params.DEPLOY_TARGET == 'docker' } expression { return params.BUILD_DOCKER_IMAGE && params.RUN_DOCKER_CONTAINER } expression { return env.IS_CHANGE_REQUEST != 'true' } expression { return env.IS_PUBLISH_BRANCH == 'true' } @@ -239,7 +351,7 @@ pipeline { post { success { - archiveArtifacts artifacts: 'dist/**,package.json,pnpm-lock.yaml,dockerfile', fingerprint: true, allowEmptyArchive: true + archiveArtifacts artifacts: 'dist/**,package.json,pnpm-lock.yaml,dockerfile,k8s/**,ci/fnos-k8s/**', fingerprint: true, allowEmptyArchive: true } } } diff --git a/ci/fnos-k8s/bootstrap.sh b/ci/fnos-k8s/bootstrap.sh new file mode 100644 index 0000000..3649734 --- /dev/null +++ b/ci/fnos-k8s/bootstrap.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +KT_K8S_ROOT="${KT_K8S_ROOT:-/vol1/docker/kt-k8s}" +CLUSTER_NAME="${CLUSTER_NAME:-kt-nas}" +K8S_NAMESPACE="${K8S_NAMESPACE:-kt-prod}" +REGISTRY_NAME="${REGISTRY_NAME:-kt-registry.localhost}" +REGISTRY_PORT="${REGISTRY_PORT:-5000}" +API_HOST_PORT="${API_HOST_PORT:-48085}" +API_NODE_PORT="${API_NODE_PORT:-30085}" +AGENT_CONTAINER="${AGENT_CONTAINER:-kt-node-agent}" +AGENT_KUBECONFIG="${AGENT_KUBECONFIG:-/home/jenkins/agent/kubeconfig/${CLUSTER_NAME}.jenkins.yaml}" +API_ENV_FILE_ON_AGENT="${API_ENV_FILE_ON_AGENT:-/home/jenkins/agent/env/kt-template-online-api/.env.production}" +API_ENV_SECRET="${API_ENV_SECRET:-kt-template-online-api-env}" +PAUSE_IMAGE="${PAUSE_IMAGE:-rancher/mirrored-pause:3.6}" +OLD_API_CONTAINER="${OLD_API_CONTAINER:-kt-template-online-api}" +STOP_OLD_API_CONTAINER="${STOP_OLD_API_CONTAINER:-false}" + +REGISTRY_CONTAINER="k3d-${REGISTRY_NAME}" +K3D_NETWORK="k3d-${CLUSTER_NAME}" +HOST_KUBECONFIG="${KT_K8S_ROOT}/kubeconfig/${CLUSTER_NAME}.host.yaml" +JENKINS_KUBECONFIG="${KT_K8S_ROOT}/kubeconfig/${CLUSTER_NAME}.jenkins.yaml" + +log() { + printf '\n[%s] %s\n' "$(date '+%F %T')" "$*" +} + +warn() { + printf '\n[%s] WARN: %s\n' "$(date '+%F %T')" "$*" >&2 +} + +die() { + printf '\n[%s] ERROR: %s\n' "$(date '+%F %T')" "$*" >&2 + exit 1 +} + +require_root() { + [ "$(id -u)" -eq 0 ] || die "Please run as root." +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Required command not found: $1" +} + +cluster_exists() { + docker inspect "k3d-${CLUSTER_NAME}-serverlb" "k3d-${CLUSTER_NAME}-server-0" >/dev/null 2>&1 +} + +install_k3d() { + if command -v k3d >/dev/null 2>&1; then + log "k3d already installed: $(k3d version | head -n 1)" + return + fi + + require_command curl + log "Installing k3d from official installer" + curl -fsSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + k3d version +} + +kubectl_arch() { + case "$(uname -m)" in + x86_64|amd64) echo amd64 ;; + aarch64|arm64) echo arm64 ;; + armv7l|armhf) echo arm ;; + *) die "Unsupported kubectl architecture: $(uname -m)" ;; + esac +} + +install_kubectl() { + if command -v kubectl >/dev/null 2>&1; then + log "kubectl already installed: $(kubectl version --client=true --short 2>/dev/null || kubectl version --client=true)" + return + fi + + require_command curl + local version arch + version="$(curl -fsSL https://dl.k8s.io/release/stable.txt)" + arch="$(kubectl_arch)" + log "Installing kubectl ${version} for linux/${arch}" + curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${version}/bin/linux/${arch}/kubectl" + chmod +x /usr/local/bin/kubectl + kubectl version --client=true +} + +prepare_dirs() { + log "Preparing ${KT_K8S_ROOT}" + mkdir -p \ + "${KT_K8S_ROOT}/registry" \ + "${KT_K8S_ROOT}/kubeconfig" \ + "${KT_K8S_ROOT}/secrets" \ + "${KT_K8S_ROOT}/manifests" \ + "${KT_K8S_ROOT}/backups" +} + +ensure_registry_host_entry() { + if ! grep -Eq "[[:space:]]${REGISTRY_CONTAINER}([[:space:]]|$)" /etc/hosts; then + log "Adding ${REGISTRY_CONTAINER} to /etc/hosts" + printf '127.0.0.1 %s\n' "$REGISTRY_CONTAINER" >> /etc/hosts + fi +} + +ensure_registry() { + if docker inspect "$REGISTRY_CONTAINER" >/dev/null 2>&1; then + log "k3d registry already exists: ${REGISTRY_CONTAINER}" + return + fi + + log "Creating local k3d registry: ${REGISTRY_CONTAINER}:${REGISTRY_PORT}" + k3d registry create "$REGISTRY_NAME" \ + --port "$REGISTRY_PORT" \ + -v "${KT_K8S_ROOT}/registry:/var/lib/registry" +} + +assert_api_port_available_for_new_cluster() { + cluster_exists && return + + local docker_owner + docker_owner="$(docker ps --format '{{.Names}} {{.Ports}}' | grep -E "(:|0\.0\.0\.0:|:::|127\.0\.0\.1:)${API_HOST_PORT}->" || true)" + if [ -z "$docker_owner" ] && command -v ss >/dev/null 2>&1; then + ss -ltn "( sport = :${API_HOST_PORT} )" | grep -q ":${API_HOST_PORT}" && docker_owner="non-docker-process" + fi + [ -z "$docker_owner" ] && return + + if [ "$STOP_OLD_API_CONTAINER" = "true" ]; then + log "Host port ${API_HOST_PORT} is in use. Stopping old API container: ${OLD_API_CONTAINER}" + docker rm -f "$OLD_API_CONTAINER" >/dev/null 2>&1 || true + return + fi + + die "Host port ${API_HOST_PORT} is in use: ${docker_owner}. Re-run with STOP_OLD_API_CONTAINER=true when you are ready to cut over." +} + +ensure_cluster() { + if cluster_exists; then + log "k3d cluster already exists: ${CLUSTER_NAME}" + else + assert_api_port_available_for_new_cluster + log "Creating k3d cluster: ${CLUSTER_NAME}" + k3d cluster create "$CLUSTER_NAME" \ + --servers 1 \ + --agents 1 \ + --registry-use "${REGISTRY_CONTAINER}:${REGISTRY_PORT}" \ + -p "${API_HOST_PORT}:${API_NODE_PORT}@loadbalancer" \ + --kubeconfig-update-default=false \ + --kubeconfig-switch-context=false + fi + + docker network connect "$K3D_NETWORK" "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true +} + +ensure_pause_image() { + log "Ensuring K3s sandbox image in cluster: ${PAUSE_IMAGE}" + if ! docker image inspect "$PAUSE_IMAGE" >/dev/null 2>&1; then + docker pull "$PAUSE_IMAGE" + fi + k3d image import "$PAUSE_IMAGE" -c "$CLUSTER_NAME" +} + +export_kubeconfigs() { + log "Exporting kubeconfigs" + k3d kubeconfig get "$CLUSTER_NAME" > "$HOST_KUBECONFIG" + cp "$HOST_KUBECONFIG" "$JENKINS_KUBECONFIG" + + # Jenkins Agent runs inside Docker, so it reaches the API server through the k3d Docker network. + kubectl config set-cluster "k3d-${CLUSTER_NAME}" \ + --server="https://k3d-${CLUSTER_NAME}-serverlb:6443" \ + --kubeconfig "$JENKINS_KUBECONFIG" >/dev/null + + chmod 600 "$HOST_KUBECONFIG" "$JENKINS_KUBECONFIG" + kubectl --kubeconfig "$HOST_KUBECONFIG" get nodes +} + +ensure_namespace() { + log "Ensuring namespace: ${K8S_NAMESPACE}" + kubectl --kubeconfig "$HOST_KUBECONFIG" create namespace "$K8S_NAMESPACE" \ + --dry-run=client -o yaml | kubectl --kubeconfig "$HOST_KUBECONFIG" apply -f - +} + +sync_agent_kubeconfig() { + if ! docker inspect "$AGENT_CONTAINER" >/dev/null 2>&1; then + warn "Jenkins Agent container not found: ${AGENT_CONTAINER}. Copy ${JENKINS_KUBECONFIG} into ${AGENT_KUBECONFIG} after Agent is created." + return + fi + + if [ "$(docker inspect -f '{{.State.Running}}' "$AGENT_CONTAINER")" != "true" ]; then + warn "Jenkins Agent container exists but is not running: ${AGENT_CONTAINER}. Start it before syncing kubeconfig." + return + fi + + log "Connecting Jenkins Agent to ${K3D_NETWORK}" + if ! docker inspect -f '{{json .NetworkSettings.Networks}}' "$AGENT_CONTAINER" | grep -q "\"${K3D_NETWORK}\""; then + docker network connect "$K3D_NETWORK" "$AGENT_CONTAINER" + fi + + log "Copying kubeconfig into Jenkins Agent: ${AGENT_KUBECONFIG}" + docker exec "$AGENT_CONTAINER" sh -lc "mkdir -p '$(dirname "$AGENT_KUBECONFIG")'" + docker cp "$JENKINS_KUBECONFIG" "${AGENT_CONTAINER}:${AGENT_KUBECONFIG}" + docker exec "$AGENT_CONTAINER" sh -lc "chmod 600 '${AGENT_KUBECONFIG}' && kubectl --kubeconfig '${AGENT_KUBECONFIG}' get namespace '${K8S_NAMESPACE}'" +} + +sync_api_secret_if_present() { + if ! docker inspect "$AGENT_CONTAINER" >/dev/null 2>&1; then + return + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$AGENT_CONTAINER")" != "true" ]; then + return + fi + if ! docker exec "$AGENT_CONTAINER" sh -lc "test -f '${API_ENV_FILE_ON_AGENT}'"; then + warn "API env file not found in Agent: ${API_ENV_FILE_ON_AGENT}. Jenkins will fail K8s deploy until this file exists." + return + fi + + log "Creating/updating API env Secret from Agent env file" + docker exec "$AGENT_CONTAINER" sh -lc "kubectl --kubeconfig '${AGENT_KUBECONFIG}' -n '${K8S_NAMESPACE}' create secret generic '${API_ENV_SECRET}' --from-env-file='${API_ENV_FILE_ON_AGENT}' --dry-run=client -o yaml | kubectl --kubeconfig '${AGENT_KUBECONFIG}' apply -f -" +} + +main() { + require_root + require_command docker + docker info >/dev/null + + install_k3d + install_kubectl + prepare_dirs + ensure_registry_host_entry + ensure_registry + ensure_cluster + ensure_pause_image + export_kubeconfigs + ensure_namespace + sync_agent_kubeconfig + sync_api_secret_if_present + + log "Bootstrap completed" + cat < k3d NodePort ${API_NODE_PORT} + +Jenkins defaults: + DEPLOY_TARGET=k8s + DOCKER_REGISTRY=${REGISTRY_CONTAINER}:${REGISTRY_PORT} + KUBE_CONFIG_FILE=${AGENT_KUBECONFIG} + CONTAINER_ENV_FILE=${API_ENV_FILE_ON_AGENT} + +EOF +} + +main "$@" diff --git a/ci/fnos-k8s/run-remote-bootstrap.ps1 b/ci/fnos-k8s/run-remote-bootstrap.ps1 new file mode 100644 index 0000000..2a9934e --- /dev/null +++ b/ci/fnos-k8s/run-remote-bootstrap.ps1 @@ -0,0 +1,26 @@ +param( + [string]$SshTarget = "root@yd.frp-bag.com", + [int]$SshPort = 45122, + [switch]$Cutover +) + +$ErrorActionPreference = "Stop" + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$localScript = Join-Path $scriptDir "bootstrap.sh" +$remoteScript = "/tmp/kt-fnos-k8s-bootstrap.sh" +$sshOptions = @("-o", "StrictHostKeyChecking=accept-new") +$sshArgs = $sshOptions + @("-p", $SshPort.ToString(), $SshTarget) +$scpArgs = $sshOptions + @("-P", $SshPort.ToString(), $localScript, "${SshTarget}:$remoteScript") +$remoteEnv = "" + +if ($Cutover) { + # Cutover allows the bootstrap script to stop the old Docker API container if 48085 is occupied. + $remoteEnv = "STOP_OLD_API_CONTAINER=true " +} + +Write-Host "Uploading $localScript to ${SshTarget}:$remoteScript" +& scp @scpArgs + +Write-Host "Running fnOS k3d bootstrap on $SshTarget" +& ssh @sshArgs "chmod +x '$remoteScript' && ${remoteEnv}bash '$remoteScript'" diff --git a/ci/jenkins-agent/Dockerfile b/ci/jenkins-agent/Dockerfile index 3e1037d..527ffc9 100644 --- a/ci/jenkins-agent/Dockerfile +++ b/ci/jenkins-agent/Dockerfile @@ -4,6 +4,7 @@ USER root ARG NODE_MAJOR=22 ARG PNPM_VERSION=9 +ARG KUBECTL_VERSION=stable ARG GIT_SSH_HOST=github.com RUN apt-get update \ @@ -19,10 +20,17 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends nodejs docker-ce-cli docker-buildx-plugin docker-compose-plugin \ && corepack enable \ && corepack prepare pnpm@${PNPM_VERSION} --activate \ + && ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in amd64|arm64) ;; armhf) ARCH=arm ;; *) echo "Unsupported kubectl architecture: $ARCH" >&2; exit 1 ;; esac \ + && KUBECTL_VERSION_RESOLVED="${KUBECTL_VERSION}" \ + && if [ "${KUBECTL_VERSION}" = "stable" ]; then KUBECTL_VERSION_RESOLVED="$(curl -fsSL https://dl.k8s.io/release/stable.txt)"; fi \ + && curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION_RESOLVED}/bin/linux/${ARCH}/kubectl" \ + && chmod +x /usr/local/bin/kubectl \ && node --version \ && pnpm --version \ && docker --version \ && docker compose version \ + && kubectl version --client=true \ && mkdir -p /root/.ssh /home/jenkins/.ssh /etc/ssh \ && ssh-keyscan -t rsa,ecdsa,ed25519 ${GIT_SSH_HOST} | tee /etc/ssh/ssh_known_hosts /root/.ssh/known_hosts /home/jenkins/.ssh/known_hosts >/dev/null \ && chmod 700 /root/.ssh /home/jenkins/.ssh \ diff --git a/ci/jenkins-agent/README.md b/ci/jenkins-agent/README.md index d04fc90..f31b5a1 100644 --- a/ci/jenkins-agent/README.md +++ b/ci/jenkins-agent/README.md @@ -9,6 +9,7 @@ Agent 镜像内置: - Node.js 22 - pnpm 9 - Docker CLI / Buildx / Compose plugin +- kubectl - `github.com` SSH known_hosts 项目业务镜像仍然使用仓库根目录的 `dockerfile`。本目录的 Dockerfile 是给 Jenkins Agent 用的,不是后端服务运行镜像。 @@ -122,6 +123,32 @@ CONTAINER_ENV_FILE=/home/jenkins/agent/env/kt-template-online-api/.env.productio 如果业务容器需要加入某个 Docker 网络,在 Jenkins 参数 `CONTAINER_NETWORK` 填网络名;如果需要挂载上传目录、日志目录等,在 `CONTAINER_EXTRA_ARGS` 填额外的 `docker run` 参数。 +## K8s 发布 kubeconfig + +标准 K8s 发布链路使用 `ci/fnos-k8s/bootstrap.sh` 在 NAS 上创建 k3d 集群,并把 Jenkins Agent 专用 kubeconfig 放入: + +```text +/home/jenkins/agent/kubeconfig/kt-nas.jenkins.yaml +``` + +这个 kubeconfig 的 API Server 地址是 k3d Docker 网络内的: + +```text +https://k3d-kt-nas-serverlb:6443 +``` + +因此 Agent 容器需要同时加入 Jenkins 网络和 k3d 网络。初始化脚本会自动执行: + +```bash +docker network connect k3d-kt-nas kt-node-agent +``` + +如果重建了 Agent 容器,重新执行一次下面命令即可恢复 kubeconfig 和网络连接: + +```powershell +.\ci\fnos-k8s\run-remote-bootstrap.ps1 +``` + ## 验证 查看 Agent 日志: diff --git a/docs/fnos-docker-jenkins-k8s-standard.md b/docs/fnos-docker-jenkins-k8s-standard.md new file mode 100644 index 0000000..4b2c01e --- /dev/null +++ b/docs/fnos-docker-jenkins-k8s-standard.md @@ -0,0 +1,161 @@ +# fnOS Docker + Jenkins + k3d/K8s 标准发布流程 + +这套流程把飞牛 NAS 上的 Docker 保留为基础控制面,把业务运行逐步迁到 k3d/K3s: + +- Jenkins Controller、Jenkins Agent、本地 Registry 仍由 Docker 管理。 +- 后端 API 进入 k3d/K8s,由 Jenkins 构建镜像、推送本地 Registry、滚动更新 Deployment。 +- Web 和 Playground 继续走现有 Nginx 静态发布,等后端链路稳定后再决定是否容器化。 + +## 固定命名 + +| 对象 | 名称 | +| --- | --- | +| Jenkins Agent | `kt-node-agent` | +| k3d 集群 | `kt-nas` | +| K8s namespace | `kt-prod` | +| 本地 Registry | `k3d-kt-registry.localhost:5000` | +| API Deployment | `kt-template-online-api` | +| API Service | `kt-template-online-api` | +| API K8s 容器名 | `api` | +| API 容器端口 | `48085` | +| API NodePort | `30085` | +| NAS 对外端口 | `48085` | + +## 一次性初始化 + +先确保本机 SSH key 已授权到 NAS 的 root 用户,然后从仓库根目录执行: + +```powershell +.\ci\fnos-k8s\run-remote-bootstrap.ps1 +``` + +如果 NAS 上旧的 Docker API 容器已经占用 `48085`,第一次真正切换时再执行: + +```powershell +.\ci\fnos-k8s\run-remote-bootstrap.ps1 -Cutover +``` + +`-Cutover` 会允许脚本停止旧的 `kt-template-online-api` Docker 容器,把 `48085` 交给 k3d loadbalancer 映射到 K8s `NodePort 30085`。 + +脚本会在 NAS 上完成: + +- 创建 `/vol1/docker/kt-k8s/{registry,kubeconfig,secrets,manifests,backups}`。 +- 安装缺失的 `k3d` 和 `kubectl`。 +- 创建本地 Registry。 +- 创建 `kt-nas` 集群。 +- 拉取并导入 `rancher/mirrored-pause:3.6`,避免 K3s 节点因 Docker Hub 超时卡在 `ContainerCreating`。 +- 导出 host kubeconfig 和 Jenkins Agent kubeconfig。 +- 创建 `kt-prod` namespace。 +- 将 `kt-node-agent` 接入 k3d Docker 网络。 +- 将 kubeconfig 复制到 Agent 内的 `/home/jenkins/agent/kubeconfig/kt-nas.jenkins.yaml`。 +- 如果 Agent 内已有 `/home/jenkins/agent/env/kt-template-online-api/.env.production`,同步创建 `kt-template-online-api-env` Secret。 + +## Jenkins Agent 镜像 + +Agent 镜像位于: + +```text +ci/jenkins-agent/Dockerfile +``` + +镜像内置: + +- Node.js 22 +- pnpm 9 +- Docker CLI / Buildx / Compose +- kubectl +- Git / OpenSSH + +NAS 上重新构建并重启 Agent: + +```bash +docker build -t kt-jenkins-agent:node22 -f ci/jenkins-agent/Dockerfile ci/jenkins-agent +docker rm -f kt-node-agent +``` + +然后按 `ci/jenkins-agent/README.md` 中的 `docker run` 命令重新启动。Agent 启动后再跑一次: + +```powershell +.\ci\fnos-k8s\run-remote-bootstrap.ps1 +``` + +这样脚本会把 kubeconfig 重新复制进 Agent,并把 Agent 接到 `k3d-kt-nas` 网络。 + +## Jenkins 发布参数 + +后端 Jenkinsfile 的标准参数: + +```text +DEPLOY_TARGET=k8s +BUILD_DOCKER_IMAGE=true +PUSH_DOCKER_IMAGE=true +DOCKER_REGISTRY=k3d-kt-registry.localhost:5000 +IMAGE_NAME=kt-template-online-api +CONTAINER_ENV_FILE=/home/jenkins/agent/env/kt-template-online-api/.env.production +KUBE_CONFIG_FILE=/home/jenkins/agent/kubeconfig/kt-nas.jenkins.yaml +K8S_MANIFEST_FILE=k8s/prod/api.yaml +K8S_NAMESPACE=kt-prod +K8S_DEPLOYMENT=kt-template-online-api +K8S_CONTAINER=api +K8S_ENV_SECRET=kt-template-online-api-env +``` + +发布阶段会做四件事: + +1. 构建后端 `dist`。 +2. 用仓库根目录 `dockerfile` 构建业务镜像。 +3. 推送到 NAS 本地 Registry,同时更新 `latest` 标签。 +4. 从 Agent 私有 `.env.production` 重建 K8s Secret,并滚动更新 Deployment 镜像。 + +## 验证 + +NAS 上验证集群: + +```bash +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml get nodes +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml -n kt-prod get pod,svc +``` + +Agent 内验证: + +```bash +docker exec kt-node-agent sh -lc 'kubectl --kubeconfig /home/jenkins/agent/kubeconfig/kt-nas.jenkins.yaml -n kt-prod get pod,svc' +``` + +API 验证: + +```bash +curl -I http://127.0.0.1:48085 +``` + +如果公网入口仍由腾讯云 WireGuard/Caddy 转发到 NAS `10.66.66.2:48085`,切换到 K8s 后公网侧不需要改端口。 + +## 回滚 + +查看发布历史: + +```bash +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml -n kt-prod rollout history deployment/kt-template-online-api +``` + +回滚上一个版本: + +```bash +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml -n kt-prod rollout undo deployment/kt-template-online-api +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml -n kt-prod rollout status deployment/kt-template-online-api --timeout=180s +``` + +查看日志: + +```bash +kubectl --kubeconfig /vol1/docker/kt-k8s/kubeconfig/kt-nas.host.yaml -n kt-prod logs -l app=kt-template-online-api --tail=200 +``` + +如果需要临时退回旧 Docker 容器,先删除或停止 k3d loadbalancer 对 `48085` 的占用,再按旧 Jenkins Docker 参数重启 `kt-template-online-api` 容器。 + +## 参考 + +- k3d: +- k3d Registry: +- kubectl Linux 安装: +- Kubernetes Deployment: diff --git a/k8s/prod/api.yaml b/k8s/prod/api.yaml new file mode 100644 index 0000000..600d70a --- /dev/null +++ b/k8s/prod/api.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kt-template-online-api + namespace: kt-prod + labels: + app: kt-template-online-api +spec: + replicas: 1 + revisionHistoryLimit: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: kt-template-online-api + template: + metadata: + labels: + app: kt-template-online-api + spec: + containers: + - name: api + image: k3d-kt-registry.localhost:5000/kt-template-online-api:latest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 48085 + env: + - name: NODE_ENV + value: production + # Jenkins 每次发布会从 Agent 私有 .env.production 重建这个 Secret。 + envFrom: + - secretRef: + name: kt-template-online-api-env + readinessProbe: + tcpSocket: + port: 48085 + initialDelaySeconds: 8 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + tcpSocket: + port: 48085 + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1000m + memory: 768Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: kt-template-online-api + namespace: kt-prod + labels: + app: kt-template-online-api +spec: + type: NodePort + selector: + app: kt-template-online-api + ports: + - name: http + port: 48085 + targetPort: 48085 + nodePort: 30085