758 lines
25 KiB
TypeScript
758 lines
25 KiB
TypeScript
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
import type {
|
|
DeployObservationAssertion,
|
|
DeployObservationCommand,
|
|
DeployObservationDeploymentEvidence,
|
|
DeployObservationEvidence,
|
|
DeployObservationInput,
|
|
DeployObservationJenkinsEvidence,
|
|
DeployObservationPodEvidence,
|
|
DeployObservationResult,
|
|
DeployObservationRuntimeHealthEvidence,
|
|
DeployObservationStatus,
|
|
} from '../types.js';
|
|
import { tryPowerShell } from '../core/exec.js';
|
|
import {
|
|
formatDateInShanghai,
|
|
resolveInsideRoot,
|
|
workspaceRoot,
|
|
} from '../core/workspace.js';
|
|
|
|
const defaultArtifactRoot = '.kt-workspace/test-artifacts/deploy-observation';
|
|
const unsafeHereStringPattern = /[\r\n]|'@|"@/;
|
|
|
|
interface NormalizedInput
|
|
extends Required<
|
|
Pick<
|
|
DeployObservationInput,
|
|
| 'artifactRoot'
|
|
| 'container'
|
|
| 'deployment'
|
|
| 'healthUrl'
|
|
| 'jenkinsHome'
|
|
| 'jobName'
|
|
| 'kubeconfigPath'
|
|
| 'namespace'
|
|
| 'project'
|
|
| 'selector'
|
|
| 'sshTarget'
|
|
>
|
|
> {
|
|
buildNumber?: string;
|
|
execute: boolean;
|
|
expectedCommit?: string;
|
|
imageTag?: string;
|
|
smoke?: string;
|
|
sshPort?: number;
|
|
}
|
|
|
|
interface NormalizeEvidenceInput {
|
|
checkedAt?: Date;
|
|
input: DeployObservationInput;
|
|
sectionMap: Map<string, string>;
|
|
}
|
|
|
|
function normalizeArtifactRoot(value?: string): string {
|
|
const artifactRoot = value?.trim() || defaultArtifactRoot;
|
|
assertSafeStructuredInput('artifactRoot', artifactRoot);
|
|
const base = resolveInsideRoot(defaultArtifactRoot);
|
|
const target = resolveInsideRoot(artifactRoot);
|
|
const relative = path.relative(base, target);
|
|
|
|
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
throw new Error(
|
|
`Deploy observation artifacts must stay under ${defaultArtifactRoot}`,
|
|
);
|
|
}
|
|
|
|
return artifactRoot;
|
|
}
|
|
|
|
function assertSafeStructuredInput(field: string, value?: string): void {
|
|
if (!value) return;
|
|
if (unsafeHereStringPattern.test(value)) {
|
|
throw new Error(
|
|
`Unsafe deploy observation input "${field}": CR/LF and PowerShell here-string terminators are not allowed.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function cleanStructuredInput(
|
|
field: string,
|
|
value: string | undefined,
|
|
): string | undefined {
|
|
const trimmed = value?.trim();
|
|
assertSafeStructuredInput(field, trimmed);
|
|
return trimmed || undefined;
|
|
}
|
|
|
|
function normalizeSshTarget(value?: string, execute = false): string {
|
|
const target = cleanStructuredInput('sshTarget', value) || 'nas';
|
|
if (target.startsWith('-')) {
|
|
throw new Error(
|
|
'Unsafe deploy observation input "sshTarget": SSH option injection is not allowed.',
|
|
);
|
|
}
|
|
if (execute && target !== 'nas') {
|
|
throw new Error(
|
|
'Unsafe deploy observation input "sshTarget": execute mode only allows the stabilized "nas" SSH target.',
|
|
);
|
|
}
|
|
return target;
|
|
}
|
|
|
|
function normalizeHttpUrl(field: string, value: string): string {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
throw new Error(
|
|
`Unsupported deploy observation ${field}: value must be a valid http(s) URL.`,
|
|
);
|
|
}
|
|
|
|
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
throw new Error(
|
|
`Unsupported deploy observation ${field}: URL must use http or https.`,
|
|
);
|
|
}
|
|
if (parsed.username || parsed.password) {
|
|
throw new Error(
|
|
`Unsupported deploy observation ${field}: URL credentials are not allowed.`,
|
|
);
|
|
}
|
|
|
|
return parsed.toString();
|
|
}
|
|
|
|
function normalizeSmokeCommand(value?: string): string | undefined {
|
|
const smoke = value?.trim();
|
|
if (!smoke) return undefined;
|
|
assertSafeStructuredInput('smoke', smoke);
|
|
|
|
const match = smoke.match(
|
|
/^curl\s+-fsS\s+--max-time\s+([1-9]\d?)\s+(['"]?)(https?:\/\/[^\s'"`;$|&<>\\]+)\2$/,
|
|
);
|
|
if (!match) {
|
|
throw new Error(
|
|
'Unsupported deploy observation smoke command: only bounded curl -fsS --max-time <seconds> <http-url> GET smoke is allowed.',
|
|
);
|
|
}
|
|
|
|
const timeout = Number(match[1]);
|
|
const url = match[3];
|
|
if (!Number.isInteger(timeout) || timeout < 1 || timeout > 30) {
|
|
throw new Error(
|
|
'Unsupported deploy observation smoke command: --max-time must be between 1 and 30 seconds.',
|
|
);
|
|
}
|
|
|
|
const parsedUrl = normalizeHttpUrl('smoke command', url);
|
|
|
|
return `curl -fsS --max-time ${timeout} ${shellSingleQuote(parsedUrl)}`;
|
|
}
|
|
|
|
function normalizeInput(input: DeployObservationInput = {}): NormalizedInput {
|
|
const deployment =
|
|
cleanStructuredInput('deployment', input.deployment) ||
|
|
'kt-template-online-api';
|
|
const healthUrl =
|
|
cleanStructuredInput('healthUrl', input.healthUrl) ||
|
|
'http://127.0.0.1:48085/health/runtime';
|
|
const execute = input.execute === true;
|
|
|
|
return {
|
|
artifactRoot: normalizeArtifactRoot(input.artifactRoot),
|
|
buildNumber: cleanStructuredInput('buildNumber', input.buildNumber),
|
|
container: cleanStructuredInput('container', input.container) || 'api',
|
|
deployment,
|
|
execute,
|
|
expectedCommit: cleanStructuredInput(
|
|
'expectedCommit',
|
|
input.expectedCommit,
|
|
),
|
|
healthUrl: normalizeHttpUrl('healthUrl', healthUrl),
|
|
imageTag: cleanStructuredInput('imageTag', input.imageTag),
|
|
jenkinsHome:
|
|
cleanStructuredInput('jenkinsHome', input.jenkinsHome) ||
|
|
'/vol1/docker/jenkins/jenkins_home',
|
|
jobName:
|
|
cleanStructuredInput('jobName', input.jobName) ||
|
|
'KT-Template/KT-Template-API/main',
|
|
kubeconfigPath:
|
|
cleanStructuredInput('kubeconfigPath', input.kubeconfigPath) ||
|
|
'/vol1/docker/kt-k8s/kubeconfig/kt-nas.jenkins.yaml',
|
|
namespace: cleanStructuredInput('namespace', input.namespace) || 'kt-prod',
|
|
project: cleanStructuredInput('project', input.project) || 'api',
|
|
selector: cleanStructuredInput('selector', input.selector) || `app=${deployment}`,
|
|
smoke: normalizeSmokeCommand(input.smoke),
|
|
sshPort:
|
|
typeof input.sshPort === 'number' && Number.isFinite(input.sshPort)
|
|
? input.sshPort
|
|
: undefined,
|
|
sshTarget: normalizeSshTarget(input.sshTarget, execute),
|
|
};
|
|
}
|
|
|
|
function shellSingleQuote(value: string): string {
|
|
return `'${value.replaceAll("'", "'\"'\"'")}'`;
|
|
}
|
|
|
|
function powerShellArgument(value: string): string {
|
|
if (/^[A-Za-z0-9._@:/-]+$/.test(value)) return value;
|
|
return `'${value.replaceAll("'", "''")}'`;
|
|
}
|
|
|
|
function numberOrNull(value: unknown): number | null {
|
|
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === 'string' && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function parseJsonObject(value: string): Record<string, unknown> | null {
|
|
try {
|
|
const parsed = JSON.parse(value);
|
|
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
? (parsed as Record<string, unknown>)
|
|
: null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function getPath(source: unknown, keys: string[]): unknown {
|
|
return keys.reduce<unknown>((current, key) => {
|
|
if (!current || typeof current !== 'object') return undefined;
|
|
return (current as Record<string, unknown>)[key];
|
|
}, source);
|
|
}
|
|
|
|
function getArray(source: unknown, keys: string[]): unknown[] {
|
|
const value = getPath(source, keys);
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function objectAt(value: unknown): Record<string, unknown> | null {
|
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: null;
|
|
}
|
|
|
|
function findNamedObject(
|
|
values: unknown[],
|
|
name: string,
|
|
): Record<string, unknown> | null {
|
|
const objects = values
|
|
.map(objectAt)
|
|
.filter((item): item is Record<string, unknown> => Boolean(item));
|
|
return objects.find((item) => stringOrNull(item.name) === name) || null;
|
|
}
|
|
|
|
function readKeyValueSection(value: string): Record<string, string> {
|
|
return Object.fromEntries(
|
|
value
|
|
.split(/\r?\n/)
|
|
.map((line) => {
|
|
const index = line.indexOf('=');
|
|
return index >= 0
|
|
? [line.slice(0, index).trim(), line.slice(index + 1).trim()]
|
|
: null;
|
|
})
|
|
.filter((item): item is [string, string] => Boolean(item?.[0])),
|
|
);
|
|
}
|
|
|
|
function buildRemoteScript(input: NormalizedInput): string {
|
|
const smokeBlock = input.smoke
|
|
? [
|
|
'section smoke_text',
|
|
'set +e',
|
|
`${input.smoke} 2>&1`,
|
|
'SMOKE_EXIT=$?',
|
|
'set -u',
|
|
'section smoke_status',
|
|
'printf "exitCode=%s\\n" "$SMOKE_EXIT"',
|
|
].join('\n')
|
|
: [
|
|
'section smoke_text',
|
|
"echo 'no task smoke command provided'",
|
|
'section smoke_status',
|
|
'printf "exitCode=\\n"',
|
|
].join('\n');
|
|
|
|
return [
|
|
'set -u',
|
|
'section() { printf "\\n__KT_SECTION:%s__\\n" "$1"; }',
|
|
`JOB_NAME=${shellSingleQuote(input.jobName)}`,
|
|
`BUILD_NUMBER=${shellSingleQuote(input.buildNumber || '')}`,
|
|
`JENKINS_HOME=${shellSingleQuote(input.jenkinsHome)}`,
|
|
`KUBECONFIG_PATH=${shellSingleQuote(input.kubeconfigPath)}`,
|
|
`NAMESPACE=${shellSingleQuote(input.namespace)}`,
|
|
`DEPLOYMENT=${shellSingleQuote(input.deployment)}`,
|
|
`SELECTOR=${shellSingleQuote(input.selector)}`,
|
|
`HEALTH_URL=${shellSingleQuote(input.healthUrl)}`,
|
|
'KUBECONFIG_ACTIVE="$KUBECONFIG_PATH"',
|
|
'KUBECONFIG_TEMP=""',
|
|
'cleanup_kubeconfig_temp() { if [ -n "$KUBECONFIG_TEMP" ]; then rm -f "$KUBECONFIG_TEMP"; fi; }',
|
|
'trap cleanup_kubeconfig_temp EXIT',
|
|
'if [ -f "$KUBECONFIG_PATH" ] && grep -q "https://k3d-kt-nas-serverlb:6443" "$KUBECONFIG_PATH"; then',
|
|
' K3D_API_PORT="$(docker port k3d-kt-nas-serverlb 6443/tcp 2>/dev/null | tail -1 | rev | cut -d: -f1 | rev || true)"',
|
|
' if [ -n "$K3D_API_PORT" ]; then',
|
|
' KUBECONFIG_TEMP="/tmp/kt-deploy-observation-kubeconfig-$$.yaml"',
|
|
' sed "s#https://k3d-kt-nas-serverlb:6443#https://127.0.0.1:${K3D_API_PORT}#g" "$KUBECONFIG_PATH" > "$KUBECONFIG_TEMP"',
|
|
' KUBECONFIG_ACTIVE="$KUBECONFIG_TEMP"',
|
|
' fi',
|
|
'fi',
|
|
'JOB_DIR="$JENKINS_HOME/jobs/$JOB_NAME/builds"',
|
|
'if [ ! -d "$JOB_DIR" ]; then',
|
|
' JOB_SLASH_COUNT="$(printf "%s" "$JOB_NAME" | awk -F/ \'{print NF-1}\')"',
|
|
' if [ "$JOB_SLASH_COUNT" = "2" ]; then',
|
|
' JOB_FOLDER="${JOB_NAME%%/*}"',
|
|
' JOB_BRANCH="${JOB_NAME##*/}"',
|
|
' JOB_PIPELINE="${JOB_NAME#*/}"',
|
|
' JOB_PIPELINE="${JOB_PIPELINE%/*}"',
|
|
' MULTIBRANCH_JOB_DIR="$JENKINS_HOME/jobs/$JOB_FOLDER/jobs/$JOB_PIPELINE/branches/$JOB_BRANCH/builds"',
|
|
' if [ -d "$MULTIBRANCH_JOB_DIR" ]; then',
|
|
' JOB_DIR="$MULTIBRANCH_JOB_DIR"',
|
|
' else',
|
|
' FOUND_JOB_DIR="$(find "$JENKINS_HOME/jobs" -path "*/jobs/$JOB_PIPELINE/branches/$JOB_BRANCH/builds" -type d -print -quit 2>/dev/null || true)"',
|
|
' if [ -z "$FOUND_JOB_DIR" ]; then FOUND_JOB_DIR="$(find "$JENKINS_HOME/jobs" -path "*/jobs/$JOB_PIPELINE/branches/main/builds" -type d -print -quit 2>/dev/null || true)"; fi',
|
|
' if [ -n "$FOUND_JOB_DIR" ]; then JOB_DIR="$FOUND_JOB_DIR"; fi',
|
|
' fi',
|
|
' fi',
|
|
'fi',
|
|
'if [ -z "$BUILD_NUMBER" ] && [ -d "$JOB_DIR" ]; then BUILD_NUMBER="$(find "$JOB_DIR" -maxdepth 1 -type d -printf "%f\\n" 2>/dev/null | grep -E "^[0-9]+$" | sort -n | tail -1 || true)"; fi',
|
|
'BUILD_DIR="$JOB_DIR/$BUILD_NUMBER"',
|
|
'section jenkins_meta',
|
|
'printf "jobName=%s\\n" "$JOB_NAME"',
|
|
'printf "buildNumber=%s\\n" "$BUILD_NUMBER"',
|
|
'if [ -f "$BUILD_DIR/build.xml" ]; then RESULT="$(grep -Eo "<result>[^<]*</result>" "$BUILD_DIR/build.xml" 2>/dev/null | sed -E "s#</?result>##g" | tail -1 || true)"; else RESULT=""; fi',
|
|
'printf "result=%s\\n" "$RESULT"',
|
|
'section jenkins_build_xml',
|
|
'if [ -f "$BUILD_DIR/build.xml" ]; then grep -E "<(hash|sha1)>[0-9a-fA-F]{7,40}</(hash|sha1)>" "$BUILD_DIR/build.xml" 2>/dev/null | head -n 20 || true; else echo "jenkins build.xml not found: $BUILD_DIR/build.xml"; fi',
|
|
'section jenkins_log_tail',
|
|
'if [ -f "$BUILD_DIR/log" ]; then tail -n 160 "$BUILD_DIR/log"; else echo "jenkins log not found: $BUILD_DIR/log"; fi',
|
|
'section deployment_json',
|
|
'kubectl --kubeconfig "$KUBECONFIG_ACTIVE" --request-timeout=8s -n "$NAMESPACE" get deployment "$DEPLOYMENT" -o json 2>&1 || true',
|
|
'section pods_json',
|
|
'kubectl --kubeconfig "$KUBECONFIG_ACTIVE" --request-timeout=8s -n "$NAMESPACE" get pods -l "$SELECTOR" -o json 2>&1 || true',
|
|
'section events_tail',
|
|
'kubectl --kubeconfig "$KUBECONFIG_ACTIVE" --request-timeout=8s -n "$NAMESPACE" get events --sort-by=.lastTimestamp 2>&1 | tail -n 80 || true',
|
|
'section health_json',
|
|
'curl -fsS --max-time 8 "$HEALTH_URL" 2>&1 || true',
|
|
smokeBlock,
|
|
].join('\n');
|
|
}
|
|
|
|
export function buildDeployObservationCommands(
|
|
input: DeployObservationInput = {},
|
|
): DeployObservationCommand[] {
|
|
const normalized = normalizeInput(input);
|
|
const sshParts = [
|
|
'ssh',
|
|
...(normalized.sshPort ? ['-p', String(normalized.sshPort)] : []),
|
|
powerShellArgument(normalized.sshTarget),
|
|
'"tr -d \'\\015\' | bash -s"',
|
|
];
|
|
const command = [
|
|
"$script = @'",
|
|
buildRemoteScript(normalized),
|
|
"'@",
|
|
`$script | ${sshParts.join(' ')}`,
|
|
].join('\n');
|
|
|
|
return [
|
|
{
|
|
command,
|
|
name: 'nas-deploy-observation',
|
|
},
|
|
];
|
|
}
|
|
|
|
export function buildDeployObservationSectionMap(
|
|
stdout: string,
|
|
): Map<string, string> {
|
|
const sections = new Map<string, string>();
|
|
let current: string | null = null;
|
|
let buffer: string[] = [];
|
|
|
|
for (const line of stdout.split(/\r?\n/)) {
|
|
const marker = line.match(/^__KT_SECTION:([A-Za-z0-9_-]+)__$/);
|
|
if (marker) {
|
|
if (current) sections.set(current, buffer.join('\n').trim());
|
|
current = marker[1];
|
|
buffer = [];
|
|
} else if (current) {
|
|
buffer.push(line);
|
|
}
|
|
}
|
|
|
|
if (current) sections.set(current, buffer.join('\n').trim());
|
|
|
|
return sections;
|
|
}
|
|
|
|
function normalizeJenkinsEvidence(
|
|
input: NormalizedInput,
|
|
sectionMap: Map<string, string>,
|
|
): DeployObservationJenkinsEvidence {
|
|
const meta = readKeyValueSection(sectionMap.get('jenkins_meta') || '');
|
|
const logTail = sectionMap.get('jenkins_log_tail') || '';
|
|
const buildXmlRevision = sectionMap.get('jenkins_build_xml') || '';
|
|
const commitHaystack = `${logTail}\n${buildXmlRevision}`.toLowerCase();
|
|
const logFinishedStatuses = [...logTail.matchAll(/Finished:\s*([A-Z_]+)/gi)];
|
|
const logFinishedStatus =
|
|
logFinishedStatuses.at(-1)?.[1]?.toUpperCase() || null;
|
|
const logTailAvailable =
|
|
logTail.trim().length > 0 && !/jenkins log not found:/i.test(logTail);
|
|
const finishedStatus =
|
|
logFinishedStatus ||
|
|
(logTailAvailable ? null : meta.result?.toUpperCase() || null);
|
|
const expectedCommit = input.expectedCommit || null;
|
|
|
|
return {
|
|
buildNumber: meta.buildNumber || input.buildNumber || null,
|
|
commitMatched: expectedCommit
|
|
? commitHaystack.includes(expectedCommit.toLowerCase())
|
|
: null,
|
|
expectedCommit,
|
|
finishedStatus,
|
|
jobName: input.jobName,
|
|
};
|
|
}
|
|
|
|
function normalizeDeploymentEvidence(
|
|
input: NormalizedInput,
|
|
deploymentJson: Record<string, unknown> | null,
|
|
): DeployObservationDeploymentEvidence {
|
|
const container = findNamedObject(
|
|
getArray(deploymentJson, ['spec', 'template', 'spec', 'containers']),
|
|
input.container,
|
|
);
|
|
|
|
return {
|
|
container: input.container,
|
|
containerFound: Boolean(container),
|
|
desiredReplicas:
|
|
numberOrNull(getPath(deploymentJson, ['spec', 'replicas'])) ??
|
|
numberOrNull(getPath(deploymentJson, ['status', 'replicas'])) ??
|
|
1,
|
|
generation: numberOrNull(getPath(deploymentJson, ['metadata', 'generation'])),
|
|
image: stringOrNull(container?.image),
|
|
namespace: input.namespace,
|
|
observedGeneration: numberOrNull(
|
|
getPath(deploymentJson, ['status', 'observedGeneration']),
|
|
),
|
|
readyReplicas: numberOrNull(
|
|
getPath(deploymentJson, ['status', 'readyReplicas']),
|
|
),
|
|
updatedReplicas: numberOrNull(
|
|
getPath(deploymentJson, ['status', 'updatedReplicas']),
|
|
),
|
|
};
|
|
}
|
|
|
|
function scorePod(
|
|
pod: Record<string, unknown>,
|
|
input: NormalizedInput,
|
|
): number {
|
|
const phase = stringOrNull(getPath(pod, ['status', 'phase']));
|
|
const statuses = getArray(pod, ['status', 'containerStatuses']);
|
|
const status = findNamedObject(statuses, input.container);
|
|
const image = stringOrNull(status?.image);
|
|
return (
|
|
(phase === 'Running' ? 4 : 0) +
|
|
(image && input.imageTag && image.includes(input.imageTag) ? 2 : 0) +
|
|
(status?.ready === true ? 1 : 0)
|
|
);
|
|
}
|
|
|
|
function normalizePodEvidence(
|
|
input: NormalizedInput,
|
|
podsJson: Record<string, unknown> | null,
|
|
): DeployObservationPodEvidence {
|
|
const pods = getArray(podsJson, ['items'])
|
|
.map(objectAt)
|
|
.filter((item): item is Record<string, unknown> => Boolean(item))
|
|
.sort((left, right) => scorePod(right, input) - scorePod(left, input));
|
|
const pod = pods[0] || null;
|
|
const status = findNamedObject(
|
|
getArray(pod, ['status', 'containerStatuses']),
|
|
input.container,
|
|
);
|
|
const specContainer = findNamedObject(
|
|
getArray(pod, ['spec', 'containers']),
|
|
input.container,
|
|
);
|
|
|
|
return {
|
|
containerFound: Boolean(status || specContainer),
|
|
image: stringOrNull(status?.image) || stringOrNull(specContainer?.image),
|
|
name: stringOrNull(getPath(pod, ['metadata', 'name'])),
|
|
phase: stringOrNull(getPath(pod, ['status', 'phase'])),
|
|
ready: typeof status?.ready === 'boolean' ? status.ready : null,
|
|
restartCount: numberOrNull(status?.restartCount),
|
|
};
|
|
}
|
|
|
|
function normalizeRuntimeHealthEvidence(
|
|
healthJson: Record<string, unknown> | null,
|
|
): DeployObservationRuntimeHealthEvidence {
|
|
const checks = getArray(healthJson, ['checks']);
|
|
|
|
return {
|
|
checkCount: checks.length || null,
|
|
service: stringOrNull(healthJson?.service),
|
|
status: stringOrNull(healthJson?.status),
|
|
};
|
|
}
|
|
|
|
function parseSmokeExitCode(sectionMap: Map<string, string>): number | null {
|
|
const status = readKeyValueSection(sectionMap.get('smoke_status') || '');
|
|
if (!status.exitCode?.trim()) return null;
|
|
const exitCode = Number(status.exitCode);
|
|
return Number.isInteger(exitCode) && exitCode >= 0 ? exitCode : null;
|
|
}
|
|
|
|
function buildAssertion(
|
|
name: string,
|
|
passed: boolean,
|
|
message: string,
|
|
critical = true,
|
|
): DeployObservationAssertion {
|
|
return {
|
|
critical,
|
|
message,
|
|
name,
|
|
passed,
|
|
};
|
|
}
|
|
|
|
function aggregateStatus(
|
|
assertions: DeployObservationAssertion[],
|
|
): DeployObservationStatus {
|
|
if (assertions.every((item) => item.passed)) return 'passed';
|
|
if (assertions.some((item) => item.critical && !item.passed)) return 'failed';
|
|
return 'blocked';
|
|
}
|
|
|
|
function toFilenameSlug(value: string): string {
|
|
const slug = value
|
|
.replace(/[^A-Za-z0-9_-]+/g, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
.slice(0, 80);
|
|
return slug || 'project';
|
|
}
|
|
|
|
export function normalizeDeployObservationEvidence({
|
|
checkedAt = new Date(),
|
|
input,
|
|
sectionMap,
|
|
}: NormalizeEvidenceInput): DeployObservationEvidence {
|
|
const normalized = normalizeInput(input);
|
|
const jenkins = normalizeJenkinsEvidence(normalized, sectionMap);
|
|
const deployment = normalizeDeploymentEvidence(
|
|
normalized,
|
|
parseJsonObject(sectionMap.get('deployment_json') || ''),
|
|
);
|
|
const pod = normalizePodEvidence(
|
|
normalized,
|
|
parseJsonObject(sectionMap.get('pods_json') || ''),
|
|
);
|
|
const runtimeHealth = normalizeRuntimeHealthEvidence(
|
|
parseJsonObject(sectionMap.get('health_json') || ''),
|
|
);
|
|
const smokeOutput = sectionMap.get('smoke_text') || '';
|
|
const smokeExitCode = parseSmokeExitCode(sectionMap);
|
|
const expectedImageTag = normalized.imageTag || normalized.expectedCommit;
|
|
const deploymentObserved =
|
|
deployment.generation !== null &&
|
|
deployment.observedGeneration !== null &&
|
|
deployment.observedGeneration >= deployment.generation;
|
|
const deploymentReplicasReady =
|
|
deployment.updatedReplicas !== null &&
|
|
deployment.readyReplicas !== null &&
|
|
deployment.desiredReplicas !== null &&
|
|
deployment.updatedReplicas >= deployment.desiredReplicas &&
|
|
deployment.readyReplicas >= deployment.desiredReplicas;
|
|
const runtimeHealthStatusAllowed =
|
|
runtimeHealth.status === 'ready' ||
|
|
runtimeHealth.status === 'live' ||
|
|
runtimeHealth.status === 'degraded';
|
|
const smokeSuccess = Boolean(normalized.smoke && smokeExitCode === 0);
|
|
const assertions = [
|
|
buildAssertion(
|
|
'jenkins-build-success',
|
|
jenkins.finishedStatus === 'SUCCESS',
|
|
`Jenkins ${jenkins.jobName} #${jenkins.buildNumber || 'unknown'} result=${jenkins.finishedStatus || 'missing'}`,
|
|
),
|
|
buildAssertion(
|
|
'jenkins-commit-match',
|
|
!jenkins.expectedCommit || jenkins.commitMatched === true,
|
|
jenkins.expectedCommit
|
|
? `Expected commit ${jenkins.expectedCommit} matched=${jenkins.commitMatched}`
|
|
: 'No expected commit supplied',
|
|
Boolean(jenkins.expectedCommit),
|
|
),
|
|
buildAssertion(
|
|
'deployment-container-found',
|
|
deployment.containerFound,
|
|
`Deployment container=${deployment.container} found=${deployment.containerFound}`,
|
|
),
|
|
buildAssertion(
|
|
'deployment-image-tag',
|
|
Boolean(
|
|
deployment.image &&
|
|
(!expectedImageTag || deployment.image.includes(expectedImageTag)),
|
|
),
|
|
`Deployment image=${deployment.image || 'missing'}`,
|
|
),
|
|
buildAssertion(
|
|
'deployment-generation-observed',
|
|
deploymentObserved,
|
|
`Deployment generation=${deployment.generation} observedGeneration=${deployment.observedGeneration}`,
|
|
),
|
|
buildAssertion(
|
|
'deployment-replicas-ready',
|
|
deploymentReplicasReady,
|
|
`Deployment desired=${deployment.desiredReplicas} ready=${deployment.readyReplicas} updated=${deployment.updatedReplicas}`,
|
|
),
|
|
buildAssertion(
|
|
'pod-container-found',
|
|
pod.containerFound,
|
|
`Pod container=${normalized.container} found=${pod.containerFound}`,
|
|
),
|
|
buildAssertion(
|
|
'pod-running',
|
|
pod.phase === 'Running' && pod.ready === true,
|
|
`Pod ${pod.name || 'missing'} phase=${pod.phase || 'missing'} ready=${pod.ready}`,
|
|
),
|
|
buildAssertion(
|
|
'pod-image-tag',
|
|
Boolean(pod.image && (!expectedImageTag || pod.image.includes(expectedImageTag))),
|
|
`Pod image=${pod.image || 'missing'}`,
|
|
),
|
|
buildAssertion(
|
|
'pod-restart-count-zero',
|
|
pod.restartCount === 0,
|
|
`Pod restartCount=${pod.restartCount}`,
|
|
),
|
|
buildAssertion(
|
|
'runtime-health-available',
|
|
runtimeHealth.service === 'kt-template-online-api' &&
|
|
runtimeHealthStatusAllowed,
|
|
`Runtime health status=${runtimeHealth.status || 'missing'}`,
|
|
),
|
|
buildAssertion(
|
|
'task-smoke-success',
|
|
smokeSuccess,
|
|
normalized.smoke
|
|
? `Task smoke exitCode=${smokeExitCode ?? 'missing'}`
|
|
: 'No task smoke command supplied; rollout evidence is not functional evidence',
|
|
),
|
|
];
|
|
|
|
return {
|
|
assertions,
|
|
details: {
|
|
deployment,
|
|
eventsTail: sectionMap.get('events_tail') || '',
|
|
jenkins,
|
|
pod,
|
|
runtimeHealth,
|
|
smoke: {
|
|
command: normalized.smoke || null,
|
|
exitCode: smokeExitCode,
|
|
output: smokeOutput,
|
|
success: smokeSuccess,
|
|
},
|
|
},
|
|
endedAt: checkedAt.toISOString(),
|
|
environment: 'production',
|
|
operation: 'kt_deploy_observation',
|
|
project: normalized.project,
|
|
schemaVersion: 1,
|
|
startedAt: checkedAt.toISOString(),
|
|
status: aggregateStatus(assertions),
|
|
target: `${normalized.namespace}/${normalized.deployment}`,
|
|
taskType: 'deploy',
|
|
title: 'KT API deployment observation',
|
|
};
|
|
}
|
|
|
|
function writeEvidenceArtifact(
|
|
artifactRoot: string,
|
|
evidence: DeployObservationEvidence,
|
|
): string {
|
|
const root = resolveInsideRoot(artifactRoot);
|
|
const directory = path.join(root, formatDateInShanghai());
|
|
const safeProject = toFilenameSlug(evidence.project);
|
|
const filePath = path.resolve(
|
|
directory,
|
|
`${safeProject}-${Date.now()}-deploy-observation.json`,
|
|
);
|
|
const relative = path.relative(root, filePath);
|
|
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
throw new Error(
|
|
`Deploy observation artifact path escaped artifact root: ${filePath}`,
|
|
);
|
|
}
|
|
|
|
mkdirSync(directory, { recursive: true });
|
|
writeFileSync(filePath, `${JSON.stringify(evidence, null, 2)}\n`, 'utf8');
|
|
return filePath;
|
|
}
|
|
|
|
export async function buildDeployObservation(
|
|
input: DeployObservationInput = {},
|
|
): Promise<DeployObservationResult> {
|
|
const normalized = normalizeInput(input);
|
|
const commands = buildDeployObservationCommands(normalized);
|
|
const requiredEvidence = [
|
|
'Jenkins job/build number and final Finished status',
|
|
'Expected commit hash match when a commit is supplied',
|
|
'K8s Deployment target container, desiredReplicas, updatedReplicas, readyReplicas, generation, observedGeneration',
|
|
'Running Pod selected by target container/current image tag with restartCount',
|
|
'GET /health/runtime response with service/status/check count',
|
|
'Task-specific smoke command output',
|
|
];
|
|
const notes = [
|
|
'Default execute=false only returns the bounded read-only command.',
|
|
'execute=true runs through ssh nas with CRLF-stripped here-string and writes local JSON evidence.',
|
|
'Jenkins/K8s rollout evidence is deployment evidence; task smoke is still required for functional completion.',
|
|
'The command reads Jenkins build files, kubectl status, events, and HTTP health only; it does not read Secrets or mutate remote state.',
|
|
];
|
|
|
|
if (!normalized.execute) {
|
|
return {
|
|
commands,
|
|
execute: false,
|
|
notes,
|
|
requiredEvidence,
|
|
};
|
|
}
|
|
|
|
const command = commands[0]?.command || '';
|
|
const result = await tryPowerShell(command, workspaceRoot, 180_000);
|
|
const sectionMap = buildDeployObservationSectionMap(result.stdout);
|
|
const evidence = normalizeDeployObservationEvidence({
|
|
input: normalized,
|
|
sectionMap,
|
|
});
|
|
const artifactPath = writeEvidenceArtifact(normalized.artifactRoot, evidence);
|
|
|
|
return {
|
|
artifactPath,
|
|
commands,
|
|
evidence,
|
|
execute: true,
|
|
notes,
|
|
requiredEvidence,
|
|
result,
|
|
};
|
|
}
|