Add nginx timeouts

.gitignore

@@ -58,3 +58,4 @@ env*
 
 # For integration tests.
 .tmptest/
+.golangci.yml

agent/cmd/cmd.go

@@ -390,8 +390,12 @@ func (a *App) Run(ctx context.Context) error {
 			"registry_server": nginx.GetServer(
 				a.config.Registry.Docker.HTTP.Net, a.config.Registry.Docker.HTTP.Addr),
 			"agent_server":    fmt.Sprintf("127.0.0.1:%d", a.flags.AgentServerPort),
-			"registry_backup": a.config.RegistryBackup},
-			nginx.WithTLS(a.config.TLS))
+			"registry_backup": a.config.RegistryBackup,
+			// Pass timeout parameters from agent server config
+			"download_timeout":           nginx.FormatDurationForNginx(a.config.AgentServer.DownloadTimeout),
+			"container_runtime_timeout":  nginx.FormatDurationForNginx(a.config.AgentServer.ContainerRuntimeTimeout),
+			"readiness_timeout":         nginx.FormatDurationForNginx(a.config.AgentServer.ReadinessTimeout),
+		}, nginx.WithTLS(a.config.TLS))
 		nginxDone <- err
 	}()
 

config/agent/base.yaml

@@ -58,6 +58,15 @@ registry:
 
 peer_id_factory: addr_hash
 
+agentserver:
+  # Timeout configurations (also used by nginx)
+  download_timeout: 5m             # nginx proxy_read_timeout for downloads
+  container_runtime_timeout: 10m   # nginx timeout for container operations (pull/preload)
+  readiness_timeout: 30s           # nginx timeout for health checks
+
+  # Request configuration
+  enable_request_logging: false    # Enable detailed request logging
+
 # Allow agent to only serve localhost and Docker default bridge requests.
 allowed_cidrs:
   - 127.0.0.1

config/origin/base.yaml

@@ -48,6 +48,17 @@ blobserver:
   listener:
     net: unix
     addr: /tmp/kraken-origin.sock
+
+  # Timeout configurations (also used by nginx)
+  download_timeout: 5m       # nginx proxy_read_timeout for downloads
+  upload_timeout: 10m        # nginx proxy_read_timeout/send_timeout for uploads  
+  replication_timeout: 3m    # nginx timeout for replication operations
+  backend_timeout: 2m        # nginx proxy_connect_timeout
+  readiness_timeout: 30s     # internal readiness check timeout
+
+  # Concurrency limits
+  max_concurrent_downloads: 10
+  max_concurrent_uploads: 5
 
 nginx:
   name: kraken-origin

config/tracker/base.yaml

@@ -40,6 +40,16 @@ trackerserver:
   listener:
     net: unix
     addr: /tmp/kraken-tracker.sock
+
+  # Timeout configurations (also used by nginx)
+  metainfo_timeout: 2m    # nginx proxy_read_timeout for metainfo requests to origins
+  announce_timeout: 30s   # nginx proxy_read_timeout for announce operations
+  readiness_timeout: 30s  # nginx timeout for health checks
+
+  # Rate limiting
+  get_metainfo_limit: 1s  # Limits unique metainfo requests per namespace/digest
+  announce_limit: 50      # Maximum peers returned on each announce
+  announce_interval: 3s   # How often peers should announce
 
 nginx:
   name: kraken-tracker

nginx/config/agent.go

@@ -40,13 +40,44 @@ server {
   gzip on;
   gzip_types text/plain test/csv application/json;
 
+  # Timeout configurations from agent server config
+  proxy_connect_timeout {{.readiness_timeout}};
+  proxy_send_timeout {{.download_timeout}};
+  proxy_read_timeout {{.download_timeout}};
+
   location ~ ^/(health|readiness)$ {
     proxy_pass http://agent-server;
+
+    # Use shorter timeout for health checks
+    proxy_read_timeout {{.readiness_timeout}};
+    proxy_send_timeout {{.readiness_timeout}};
+  }
+
+  # Container runtime operations (preload/pull) need longer timeouts
+  location ~ ^/preload/ {
+    proxy_pass http://agent-server;
+
+    # Use container runtime timeout for these operations
+    proxy_read_timeout {{.container_runtime_timeout}};
+    proxy_send_timeout {{.container_runtime_timeout}};
+  }
+
+  # Download operations
+  location ~ ^/namespace/.*/blobs/ {
+    proxy_pass http://agent-server;
+
+    # Use download timeout for blob operations
+    proxy_read_timeout {{.download_timeout}};
+    proxy_send_timeout {{.download_timeout}};
   }
 
   location / {
     proxy_pass http://registry-backend;
     proxy_next_upstream error timeout http_404 http_500;
+
+    # Standard timeouts for registry operations
+    proxy_read_timeout {{.download_timeout}};
+    proxy_send_timeout {{.download_timeout}};
   }
 }
 `

nginx/config/origin.go

@@ -28,8 +28,53 @@ server {
   gzip on;
   gzip_types text/plain test/csv application/json;
 
+  # Timeout configurations from origin server config
+  proxy_connect_timeout {{.backend_timeout}};
+  proxy_send_timeout {{.upload_timeout}};
+  proxy_read_timeout {{.download_timeout}};
+
+  # Keepalive settings
+  proxy_buffering off;
+  proxy_request_buffering off;
+
   location / {
     proxy_pass http://{{.server}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  # Special handling for upload operations with longer timeout
+  location ~ ^/namespace/.*/blobs/.*/uploads {
+    proxy_pass http://{{.server}};
+
+    # Use upload timeout for these operations
+    proxy_read_timeout {{.upload_timeout}};
+    proxy_send_timeout {{.upload_timeout}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  # Replication operations with their own timeout
+  location ~ ^/namespace/.*/blobs/.*/remote {
+    proxy_pass http://{{.server}};
+
+    # Use replication timeout for these operations
+    proxy_read_timeout {{.replication_timeout}};
+    proxy_send_timeout {{.replication_timeout}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
   }
 }
 `

nginx/config/tracker.go

@@ -29,10 +29,36 @@ server {
   access_log {{.access_log_path}};
   error_log {{.error_log_path}};
 
+  # Timeout configurations from tracker server config
+  proxy_connect_timeout {{.readiness_timeout}};
+  proxy_send_timeout {{.announce_timeout}};
+  proxy_read_timeout {{.announce_timeout}};
+
   location / {
     proxy_pass http://tracker;
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  # Health and readiness checks with shorter timeout
+  location ~ ^/(health|readiness)$ {
+    proxy_pass http://tracker;
+
+    proxy_read_timeout {{.readiness_timeout}};
+    proxy_send_timeout {{.readiness_timeout}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
   }
 
+  # Metainfo requests need longer timeout (cached)
   location ~* ^/namespace/.*/blobs/.*/metainfo$ {
     proxy_pass http://tracker;
 
@@ -41,6 +67,31 @@ server {
     proxy_cache_valid   200 5m;
     proxy_cache_valid   any 1s;
     proxy_cache_lock    on;
+
+    # Use metainfo timeout for these operations
+    proxy_read_timeout {{.metainfo_timeout}};
+    proxy_send_timeout {{.metainfo_timeout}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  # Announce operations
+  location ~ ^/announce {
+    proxy_pass http://tracker;
+
+    # Use announce timeout for these operations
+    proxy_read_timeout {{.announce_timeout}};
+    proxy_send_timeout {{.announce_timeout}};
+
+    # Pass original client info
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
   }
 }
 `

nginx/nginx.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"text/template"
+	"time"
 
 	"github.com/uber/kraken/nginx/config"
 	"github.com/uber/kraken/utils/httputil"
@@ -249,3 +250,21 @@ func GetServer(net, addr string) string {
 	}
 	return addr
 }
+
+func FormatDurationForNginx(d time.Duration) string {
+	// Add 30s buffer to ensure Go server times out first for observability
+	bufferedDuration := d + (30 * time.Second)
+
+	if bufferedDuration >= time.Minute {
+		minutes := int(bufferedDuration.Minutes())
+		if bufferedDuration == time.Duration(minutes)*time.Minute {
+			return fmt.Sprintf("%dm", minutes)
+		}
+	}
+	if bufferedDuration >= time.Second {
+		seconds := int(bufferedDuration.Seconds())
+		return fmt.Sprintf("%ds", seconds)
+	}
+	// Fallback to milliseconds for very short durations
+	return fmt.Sprintf("%dms", bufferedDuration.Milliseconds())
+}

origin/cmd/cmd.go

@@ -410,8 +410,12 @@ func startServices(config Config, flags *Flags, server *blobserver.Server, sched
 	log.Fatal(nginx.Run(
 		config.Nginx,
 		map[string]interface{}{
-			"port":   flags.BlobServerPort,
-			"server": nginx.GetServer(config.BlobServer.Listener.Net, config.BlobServer.Listener.Addr),
+			"port":               flags.BlobServerPort,
+			"server":             nginx.GetServer(config.BlobServer.Listener.Net, config.BlobServer.Listener.Addr),
+			"download_timeout":   nginx.FormatDurationForNginx(config.BlobServer.DownloadTimeout),
+			"upload_timeout":     nginx.FormatDurationForNginx(config.BlobServer.UploadTimeout),
+			"backend_timeout":    nginx.FormatDurationForNginx(config.BlobServer.BackendTimeout),
+			"replication_timeout": nginx.FormatDurationForNginx(config.BlobServer.ReplicationTimeout),
 		},
 		nginx.WithTLS(config.TLS)))
 }

tracker/cmd/cmd.go

@@ -158,6 +158,11 @@ func Run(flags *Flags, opts ...Option) {
 	log.Fatal(nginx.Run(config.Nginx, map[string]interface{}{
 		"port": flags.Port,
 		"server": nginx.GetServer(
-			config.TrackerServer.Listener.Net, config.TrackerServer.Listener.Addr)},
+			config.TrackerServer.Listener.Net, config.TrackerServer.Listener.Addr),
+		// Pass timeout parameters from tracker server config
+		"metainfo_timeout":  nginx.FormatDurationForNginx(config.TrackerServer.MetaInfoTimeout),
+		"announce_timeout":  nginx.FormatDurationForNginx(config.TrackerServer.AnnounceTimeout), 
+		"readiness_timeout": nginx.FormatDurationForNginx(config.TrackerServer.ReadinessTimeout),
+	},
 		nginx.WithTLS(config.TLS)))
 }

tracker/trackerserver/config.go

@@ -30,6 +30,11 @@ type Config struct {
 	AnnounceInterval time.Duration `yaml:"announce_interval"`
 
 	Listener listener.Config `yaml:"listener"`
+
+	// Timeout configurations
+	MetaInfoTimeout  time.Duration `yaml:"metainfo_timeout"`   // Timeout for metainfo requests to origins
+	AnnounceTimeout  time.Duration `yaml:"announce_timeout"`   // Timeout for announce operations  
+	ReadinessTimeout time.Duration `yaml:"readiness_timeout"`  // Timeout for readiness checks
 }
 
 func (c Config) applyDefaults() Config {
@@ -42,5 +47,14 @@ func (c Config) applyDefaults() Config {
 	if c.AnnounceInterval == 0 {
 		c.AnnounceInterval = 3 * time.Second
 	}
+	if c.MetaInfoTimeout == 0 {
+		c.MetaInfoTimeout = 2 * time.Minute
+	}
+	if c.AnnounceTimeout == 0 {
+		c.AnnounceTimeout = 30 * time.Second
+	}
+	if c.ReadinessTimeout == 0 {
+		c.ReadinessTimeout = 30 * time.Second
+	}
 	return c
 }