From 285456a448999310b272622a723bcef9377e4b08 Mon Sep 17 00:00:00 2001 From: Soumyajit Das Date: Wed, 1 Apr 2026 23:39:56 -0700 Subject: [PATCH 1/3] fix: [CI-21866]: Add debug logs for certs AI-Session-Id: dec658ee-c3fe-4e29-b9a6-48c079ca369d AI-Tool: claude-code AI-Model: unknown --- app.go | 6 ++++ buildkit/version.json | 2 +- docker.go | 73 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/app.go b/app.go index 5425b83..7ab7ca6 100644 --- a/app.go +++ b/app.go @@ -411,6 +411,11 @@ func Run() { Usage: "Buildkit response header timeout override. Default value is 30s", EnvVar: "PLUGIN_BUILDKIT_RESPONSE_HEADER_TIMEOUT", }, + cli.StringFlag{ + Name: "buildkit-ca-cert-path", + Usage: "Path to CA certificate bundle to copy into the BuildKit container. Enables TLS verification with custom/corporate CAs for DLC S3 endpoints.", + EnvVar: "PLUGIN_BUILDKIT_CA_CERT_PATH", + }, cli.StringFlag{ Name: "harness-self-hosted-s3-access-key", Usage: "build target", @@ -568,6 +573,7 @@ func run(c *cli.Context) error { BuildkitVersion: c.String("buildkit-version"), BuildkitTLSHandshakeTimeout: c.String("buildkit-tls-handshake-timeout"), BuildkitResponseHeaderTimeout: c.String("buildkit-response-header-timeout"), + CACertPath: c.String("buildkit-ca-cert-path"), }, BaseImageRegistry: c.String("docker.baseimageregistry"), BaseImageUsername: c.String("docker.baseimageusername"), diff --git a/buildkit/version.json b/buildkit/version.json index 6118b3c..d714093 100644 --- a/buildkit/version.json +++ b/buildkit/version.json @@ -1,3 +1,3 @@ { - "buildkit_version": "harness/buildkit:1.0.16" + "buildkit_version": "harness/buildkit:1.0.16-debug" } \ No newline at end of file diff --git a/docker.go b/docker.go index a6fead5..f97645f 100644 --- a/docker.go +++ b/docker.go @@ -2,8 +2,10 @@ package docker import ( "bytes" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "os" "os/exec" @@ -33,7 +35,7 @@ type ( IPv6 bool // Docker daemon IPv6 networking RegistryType drone.RegistryType // Docker registry type ArtifactRegistry string // Docker registry where artifact can be viewed - RetryCount int // Number of retry attempts to reach Docker daemon + RetryCount int // Number of retry attempts to reach Docker daemon } Builder struct { @@ -48,6 +50,7 @@ type ( BuildkitVersion string // Buildkit version BuildkitTLSHandshakeTimeout string // Buildkit TLS handshake timeout BuildkitResponseHeaderTimeout string // Buildkit response header timeout + CACertPath string // Path to CA cert bundle to propagate into BuildKit container } // Login defines Docker login parameters. @@ -383,6 +386,10 @@ func (p Plugin) Exec() error { defer func() { removeCmd.Run() }() + + if p.Builder.Driver == dockerContainerDriver { + propagateCACertsToBuiltkit(p.Builder) + } } // Enforce mutual exclusivity: Bake mode and Push-only mode cannot be used together @@ -1261,3 +1268,67 @@ func (p Plugin) pushOnly() error { return nil } + +const buildkitCertPath = "/etc/ssl/certs/ca-certificates.crt" + +func propagateCACertsToBuiltkit(builder Builder) { + srcPath := builder.CACertPath + if srcPath == "" { + srcPath = buildkitCertPath + } + + info, err := os.Stat(srcPath) + if err != nil { + fmt.Printf("[CACert] CA cert bundle not found at %s: %v\n", srcPath, err) + if builder.CACertPath != "" { + fmt.Printf("[CACert] WARNING: PLUGIN_BUILDKIT_CA_CERT_PATH was explicitly set to %s but the file does not exist\n", builder.CACertPath) + } + return + } + fmt.Printf("[CACert] Found CA cert bundle at %s (size=%d bytes)\n", srcPath, info.Size()) + + data, err := os.ReadFile(srcPath) + if err != nil { + fmt.Printf("[CACert] Failed to read CA cert bundle: %v\n", err) + return + } + logCertBundleSummary(srcPath, data) + + containerName := fmt.Sprintf("buildx_buildkit_%s0", builder.Name) + fmt.Printf("[CACert] Copying CA cert bundle from %s to %s:%s\n", srcPath, containerName, buildkitCertPath) + + cmd := exec.Command(dockerExe, "cp", srcPath, containerName+":"+buildkitCertPath) + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("[CACert] WARNING: Failed to copy CA certs to BuildKit container: %v, output: %s\n", err, string(output)) + return + } + fmt.Printf("[CACert] Successfully copied CA cert bundle to BuildKit container %s\n", containerName) +} + +func logCertBundleSummary(path string, data []byte) { + var totalCerts int + rest := data + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type == "CERTIFICATE" { + totalCerts++ + cert, err := x509.ParseCertificate(block.Bytes) + if err == nil && totalCerts <= 3 { + fmt.Printf("[CACert] cert #%d: Subject=%s Issuer=%s\n", totalCerts, cert.Subject, cert.Issuer) + } + } + } + fmt.Printf("[CACert] Bundle %s contains %d certificates total\n", path, totalCerts) + + systemPool, err := x509.SystemCertPool() + if err != nil { + fmt.Printf("[CACert] Could not load system cert pool in plugin container: %v\n", err) + } else { + fmt.Printf("[CACert] Plugin container system cert pool has %d certificates\n", len(systemPool.Subjects())) + } +} From 3f2fd9879f2c5df86ec52d7f16c1c7b7f48c0121 Mon Sep 17 00:00:00 2001 From: Soumyajit Das Date: Thu, 2 Apr 2026 00:20:13 -0700 Subject: [PATCH 2/3] fix: [CI-21866]: Add debug logs for certs AI-Session-Id: dec658ee-c3fe-4e29-b9a6-48c079ca369d AI-Tool: claude-code AI-Model: unknown --- docker.go | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docker.go b/docker.go index f97645f..49775d7 100644 --- a/docker.go +++ b/docker.go @@ -383,12 +383,20 @@ func (p Plugin) Exec() error { } removeCmd := cmdRemoveBuildx(p.Builder.Name) + var stopLogStream func() defer func() { + if stopLogStream != nil { + stopLogStream() + } + if p.Builder.Driver == dockerContainerDriver { + dumpBuildKitLogs(p.Builder.Name) + } removeCmd.Run() }() if p.Builder.Driver == dockerContainerDriver { propagateCACertsToBuiltkit(p.Builder) + stopLogStream = streamBuildKitLogs(p.Builder.Name) } } @@ -1332,3 +1340,53 @@ func logCertBundleSummary(path string, data []byte) { fmt.Printf("[CACert] Plugin container system cert pool has %d certificates\n", len(systemPool.Subjects())) } } + +const buildkitLogFile = "/harness/buildkit-debug.log" + +func streamBuildKitLogs(builderName string) func() { + containerName := fmt.Sprintf("buildx_buildkit_%s0", builderName) + fmt.Printf("[BuildKitLogs] Starting live log stream from %s to %s\n", containerName, buildkitLogFile) + + logFile, err := os.Create(buildkitLogFile) + if err != nil { + fmt.Printf("[BuildKitLogs] WARNING: Failed to create %s: %v\n", buildkitLogFile, err) + return func() {} + } + + cmd := exec.Command(dockerExe, "logs", "-f", containerName) + cmd.Stdout = logFile + cmd.Stderr = logFile + + if err := cmd.Start(); err != nil { + fmt.Printf("[BuildKitLogs] WARNING: Failed to start log streaming: %v\n", err) + logFile.Close() + return func() {} + } + + fmt.Printf("[BuildKitLogs] Live streaming started (pid=%d) → %s\n", cmd.Process.Pid, buildkitLogFile) + + return func() { + cmd.Process.Kill() + cmd.Wait() + logFile.Close() + fmt.Printf("[BuildKitLogs] Live streaming stopped\n") + } +} + +func dumpBuildKitLogs(builderName string) { + containerName := fmt.Sprintf("buildx_buildkit_%s0", builderName) + fmt.Printf("[BuildKitLogs] Dumping final logs from container %s to %s\n", containerName, buildkitLogFile) + + cmd := exec.Command(dockerExe, "logs", containerName) + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("[BuildKitLogs] WARNING: Failed to get logs from %s: %v\n", containerName, err) + return + } + + if err := os.WriteFile(buildkitLogFile, output, 0644); err != nil { + fmt.Printf("[BuildKitLogs] WARNING: Failed to write logs to %s: %v\n", buildkitLogFile, err) + return + } + fmt.Printf("[BuildKitLogs] Successfully wrote %d bytes to %s\n", len(output), buildkitLogFile) +} From b32c73960dff7edb9392efcd419fe1c2a94e2974 Mon Sep 17 00:00:00 2001 From: Soumyajit Das Date: Thu, 2 Apr 2026 01:04:06 -0700 Subject: [PATCH 3/3] fix: [CI-21866]: Add debug logs for certs AI-Session-Id: dec658ee-c3fe-4e29-b9a6-48c079ca369d AI-Tool: claude-code AI-Model: unknown --- buildkit/version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkit/version.json b/buildkit/version.json index d714093..1847805 100644 --- a/buildkit/version.json +++ b/buildkit/version.json @@ -1,3 +1,3 @@ { - "buildkit_version": "harness/buildkit:1.0.16-debug" + "buildkit_version": "harness/buildkit:1.0.16-debug1" } \ No newline at end of file