From 6711251854ff49b8fcdafa5aed9d510bddad15ee Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Wed, 11 Jan 2023 16:11:43 -0800
Subject: [PATCH 01/16] add min idle timer

---
 config/config.go    |  7 ++++---
 config/load_test.go |  7 ++++++-
 engine/engine.go    |  1 +
 engine/planner.go   | 10 ++++++++++
 4 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/config/config.go b/config/config.go
index 8764dc0..6a01890 100644
--- a/config/config.go
+++ b/config/config.go
@@ -32,9 +32,10 @@ type (
 		}
 
 		Pool struct {
-			Min    int           `default:"2"`
-			Max    int           `default:"4"`
-			MinAge time.Duration `default:"55m" split_words:"true"`
+			Min     int           `default:"2"`
+			Max     int           `default:"4"`
+			MinAge  time.Duration `default:"55m" split_words:"true"`
+			MinIdle time.Duration `default:"0" split_words:"true"`
 		}
 
 		Check struct {
diff --git a/config/load_test.go b/config/load_test.go
index 77a05f9..b0739c5 100644
--- a/config/load_test.go
+++ b/config/load_test.go
@@ -35,6 +35,9 @@ func TestDefaults(t *testing.T) {
 	if got, want := conf.Pool.MinAge, time.Minute*55; got != want {
 		t.Errorf("Want default DRONE_POOL_MIN_AGE of %d, got %d", want, got)
 	}
+	if got, want := conf.Pool.MinIdle, time.Minute*0; got != want {
+		t.Errorf("Want default DRONE_POOL_MIN_IDLE of %d, got %d", want, got)
+	}
 
 	if got, want := conf.Check.Interval, time.Minute; got != want {
 		t.Errorf("Want default DRONE_INSTALL_CHECK_INTERVAL of %s, got %s", want, got)
@@ -74,6 +77,7 @@ func TestLoad(t *testing.T) {
 		"DRONE_LOGS_PRETTY":                "true",
 		"DRONE_CAPACITY_BUFFER":            "3",
 		"DRONE_POOL_MIN_AGE":               "1h",
+		"DRONE_POOL_MIN_IDLE":              "15m",
 		"DRONE_POOL_MIN":                   "1",
 		"DRONE_POOL_MAX":                   "5",
 		"DRONE_SERVER_HOST":                "drone.company.com",
@@ -207,7 +211,8 @@ var jsonConfig = []byte(`{
   "Pool": {
     "Min": 1,
     "Max": 5,
-    "MinAge": 3600000000000
+    "MinAge": 3600000000000,
+    "MinIdle": 900000000000
   },
   "Server": {
     "Host": "drone.company.com",
diff --git a/engine/engine.go b/engine/engine.go
index f32e134..716e500 100644
--- a/engine/engine.go
+++ b/engine/engine.go
@@ -102,6 +102,7 @@ func New(
 			kernel:     config.Agent.Kernel,
 			buffer:     config.CapacityBuffer,
 			ttu:        config.Pool.MinAge,
+			tti:        config.Pool.MinIdle,
 			min:        config.Pool.Min,
 			max:        config.Pool.Max,
 			cap:        config.Agent.Concurrency,
diff --git a/engine/planner.go b/engine/planner.go
index ef9470b..f7ec59e 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -186,6 +186,16 @@ func (p *planner) mark(ctx context.Context, n int) error {
 			continue
 		}
 
+		// skipi servers that have not reached a min idle time
+		if time.Now().Before(time.Unix(server.Updated, 0).Add(p.tti)) {
+			logger.
+				WithField("server", server.Name).
+				WithField("idle", timeDiff(time.Now(), time.Unix(server.Updated, 0))).
+				WithField("min-idle", p.tti).
+				Debugln("server min-idle not reached")
+			continue
+		}
+
 		idle = append(idle, server)
 		logger.WithField("server", server.Name).
 			Debugln("server is idle")

From e7c74aba9eb88b00607167a5d6297bc2f79fa3bc Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Wed, 11 Jan 2023 16:50:25 -0800
Subject: [PATCH 02/16] fix typo

---
 engine/planner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/engine/planner.go b/engine/planner.go
index f7ec59e..152df93 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -186,7 +186,7 @@ func (p *planner) mark(ctx context.Context, n int) error {
 			continue
 		}
 
-		// skipi servers that have not reached a min idle time
+		// skip servers that have not reached a min idle time
 		if time.Now().Before(time.Unix(server.Updated, 0).Add(p.tti)) {
 			logger.
 				WithField("server", server.Name).

From 02d6dd030c9fec5e86f3eb5db8e0b99f65632aac Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Wed, 11 Jan 2023 18:51:15 -0800
Subject: [PATCH 03/16] add tti property to planner

---
 engine/planner.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/engine/planner.go b/engine/planner.go
index 152df93..b45006f 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -30,6 +30,7 @@ type planner struct {
 	cap        int           // capacity per-server
 	buffer     int           // buffer capacity to have warm and ready
 	ttu        time.Duration // minimum server age
+	tti        time.Duration // minimum server idle time
 	labels     map[string]string
 
 	client  drone.Client

From 1705bcdacdef2b8e8025d6f6fa3bff7e5ae09b27 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Wed, 11 Jan 2023 23:35:42 -0800
Subject: [PATCH 04/16] update server any time is busy to track idle time

---
 engine/planner.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/engine/planner.go b/engine/planner.go
index b45006f..afcde24 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -172,6 +172,14 @@ func (p *planner) mark(ctx context.Context, n int) error {
 	for _, server := range servers {
 		// skip busy servers
 		if _, ok := busy[server.Name]; ok {
+			// refresh Updated time to track idle time
+			err := p.servers.Update(ctx, server)
+			if err != nil {
+				logger.WithError(err).
+					WithField("server", server.Name).
+					WithField("state", server.State).
+					Errorln("cannot update server busy")
+			}
 			logger.WithField("server", server.Name).
 				Debugln("server is busy")
 			continue

From f35664ed640484d885e2fcc239c7c4ffd9fe3b19 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 00:20:36 -0800
Subject: [PATCH 05/16] force Updated field to update

---
 engine/planner.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index afcde24..bf64398 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -173,12 +173,13 @@ func (p *planner) mark(ctx context.Context, n int) error {
 		// skip busy servers
 		if _, ok := busy[server.Name]; ok {
 			// refresh Updated time to track idle time
+			server.Updated = time.Now().Unix()
 			err := p.servers.Update(ctx, server)
 			if err != nil {
 				logger.WithError(err).
 					WithField("server", server.Name).
-					WithField("state", server.State).
-					Errorln("cannot update server busy")
+					WithField("updated", server.Updated).
+					Errorln("cannot update busy server")
 			}
 			logger.WithField("server", server.Name).
 				Debugln("server is busy")

From 66787ede28611857ec5521a50ee72030bf80e41e Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 00:34:08 -0800
Subject: [PATCH 06/16] cannot abort early before checking busy servers

---
 engine/planner.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index bf64398..21ba446 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -153,12 +153,14 @@ func (p *planner) mark(ctx context.Context, n int) error {
 	// number of running servers, minus the total number
 	// of servers to terminate, falls below the minimum
 	// number of servers (including the buffer).
+	abort := false
 	if len(servers)-n < p.min {
 		logger.WithField("servers-to-terminate", n).
 			WithField("servers-running", len(servers)).
 			WithField("min-pool", p.min).
 			Debugf("abort terminating instances to ensure minimum capacity met")
-		return nil
+		// we abort later so we can still mark if servers are busy
+		abort = true
 	}
 
 	busy, err := p.listBusy(ctx)
@@ -181,8 +183,13 @@ func (p *planner) mark(ctx context.Context, n int) error {
 					WithField("updated", server.Updated).
 					Errorln("cannot update busy server")
 			}
-			logger.WithField("server", server.Name).
-				Debugln("server is busy")
+			if abort != true {
+				logger.WithField("server", server.Name).
+					Debugln("server is busy")
+			}
+			continue
+		}
+		if abort == true {
 			continue
 		}
 

From 7baa312ad0969e3820098f35e8a42faff97c35ab Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 00:44:10 -0800
Subject: [PATCH 07/16] go ahead and log that the server is busy

---
 engine/planner.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index 21ba446..3fa3721 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -183,10 +183,8 @@ func (p *planner) mark(ctx context.Context, n int) error {
 					WithField("updated", server.Updated).
 					Errorln("cannot update busy server")
 			}
-			if abort != true {
-				logger.WithField("server", server.Name).
-					Debugln("server is busy")
-			}
+			logger.WithField("server", server.Name).
+				Debugln("server is busy")
 			continue
 		}
 		if abort == true {

From cbbd407733a6bcb38476520b09d3ad045b89b1a7 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 01:30:22 -0800
Subject: [PATCH 08/16] check busy servers during Plan so always runs

---
 engine/planner.go | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index 3fa3721..9461487 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -77,6 +77,28 @@ func (p *planner) Plan(ctx context.Context) error {
 
 	ctx = logger.WithContext(ctx, log)
 
+	// check for busy servers to update idle timers
+	busy, err := p.listBusy(ctx)
+	if err != nil {
+		logger.WithError(err).
+			Errorln("cannot ascertain busy server list")
+		return err
+	}
+
+	for _, server := range servers {
+		if _, ok := busy[server.Name]; ok {
+			err := p.servers.Update(ctx, server)
+			if err != nil {
+				logger.WithError(err).
+					WithField("server", server.Name).
+					WithField("updated", server.Updated).
+					Errorln("cannot update busy server")
+			}
+			logger.WithField("server", server.Name).
+				Debugln("updated busy server")
+		}
+	}
+
 	free := max(capacity-running-p.buffer, 0)
 	diff := serverDiff(pending, free, p.cap)
 
@@ -153,14 +175,12 @@ func (p *planner) mark(ctx context.Context, n int) error {
 	// number of running servers, minus the total number
 	// of servers to terminate, falls below the minimum
 	// number of servers (including the buffer).
-	abort := false
 	if len(servers)-n < p.min {
 		logger.WithField("servers-to-terminate", n).
 			WithField("servers-running", len(servers)).
 			WithField("min-pool", p.min).
 			Debugf("abort terminating instances to ensure minimum capacity met")
-		// we abort later so we can still mark if servers are busy
-		abort = true
+		return nil
 	}
 
 	busy, err := p.listBusy(ctx)
@@ -174,22 +194,10 @@ func (p *planner) mark(ctx context.Context, n int) error {
 	for _, server := range servers {
 		// skip busy servers
 		if _, ok := busy[server.Name]; ok {
-			// refresh Updated time to track idle time
-			server.Updated = time.Now().Unix()
-			err := p.servers.Update(ctx, server)
-			if err != nil {
-				logger.WithError(err).
-					WithField("server", server.Name).
-					WithField("updated", server.Updated).
-					Errorln("cannot update busy server")
-			}
 			logger.WithField("server", server.Name).
 				Debugln("server is busy")
 			continue
 		}
-		if abort == true {
-			continue
-		}
 
 		// skip servers less than minage
 		if time.Now().Before(time.Unix(server.Created, 0).Add(p.ttu)) {

From b58e8311e3b5252fe01e289cb5e8c91d7cf07ccd Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 01:50:53 -0800
Subject: [PATCH 09/16] break it out into a separate function

---
 engine/planner.go | 60 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index 9461487..0b8ce93 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -77,28 +77,13 @@ func (p *planner) Plan(ctx context.Context) error {
 
 	ctx = logger.WithContext(ctx, log)
 
-	// check for busy servers to update idle timers
-	busy, err := p.listBusy(ctx)
+	_, err := p.updateBusy(ctx)
 	if err != nil {
-		logger.WithError(err).
-			Errorln("cannot ascertain busy server list")
+		log.WithError(err).
+			Errorln("cannot check for busy servers")
 		return err
 	}
 
-	for _, server := range servers {
-		if _, ok := busy[server.Name]; ok {
-			err := p.servers.Update(ctx, server)
-			if err != nil {
-				logger.WithError(err).
-					WithField("server", server.Name).
-					WithField("updated", server.Updated).
-					Errorln("cannot update busy server")
-			}
-			logger.WithField("server", server.Name).
-				Debugln("updated busy server")
-		}
-	}
-
 	free := max(capacity-running-p.buffer, 0)
 	diff := serverDiff(pending, free, p.cap)
 
@@ -127,6 +112,45 @@ func (p *planner) Plan(ctx context.Context) error {
 	return nil
 }
 
+
+// helper function checks for busy running instances and updates idle timer
+func (p *planner) updateBusy(ctx context.Context) (count int, error) {
+	logger := logger.FromContext(ctx)
+
+	servers, err := p.servers.ListState(ctx, autoscaler.StateRunning)
+	if err != nil {
+		logger.WithError(err).
+			Errorln("cannot fetch server list")
+		return count, err
+	}
+
+	// check for busy servers to update idle timers
+	busy, err := p.listBusy(ctx)
+	if err != nil {
+		logger.WithError(err).
+			Errorln("cannot ascertain busy server list")
+		return count, err
+	}
+
+	for _, server := range servers {
+		if _, ok := busy[server.Name]; ok {
+			err := p.servers.Update(ctx, server)
+			if err != nil {
+				logger.WithError(err).
+					WithField("server", server.Name).
+					WithField("updated", server.Updated).
+					Errorln("cannot update busy server")
+			}
+			logger.WithField("server", server.Name).
+				Debugln("updated busy server")
+			count++
+		}
+	}
+	logger.Debugf("found %d busy servers", count)
+	return count
+}
+
+
 // helper function allocates n new server instances.
 func (p *planner) alloc(ctx context.Context, n int) error {
 	logger := logger.FromContext(ctx)

From 4dc7230424551e964a1deec4bd35aa561891dcdb Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 01:53:16 -0800
Subject: [PATCH 10/16] fix lint

---
 engine/planner.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index 0b8ce93..915c848 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -77,7 +77,7 @@ func (p *planner) Plan(ctx context.Context) error {
 
 	ctx = logger.WithContext(ctx, log)
 
-	_, err := p.updateBusy(ctx)
+	_, err = p.updateBusy(ctx)
 	if err != nil {
 		log.WithError(err).
 			Errorln("cannot check for busy servers")
@@ -114,7 +114,7 @@ func (p *planner) Plan(ctx context.Context) error {
 
 
 // helper function checks for busy running instances and updates idle timer
-func (p *planner) updateBusy(ctx context.Context) (count int, error) {
+func (p *planner) updateBusy(ctx context.Context) (count int, err error) {
 	logger := logger.FromContext(ctx)
 
 	servers, err := p.servers.ListState(ctx, autoscaler.StateRunning)
@@ -147,7 +147,7 @@ func (p *planner) updateBusy(ctx context.Context) (count int, error) {
 		}
 	}
 	logger.Debugf("found %d busy servers", count)
-	return count
+	return count, nil
 }
 
 

From abb7aabcc3c12d89b1493417334412ef5d8474c8 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 02:44:12 -0800
Subject: [PATCH 11/16] update tests

---
 engine/planner_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/engine/planner_test.go b/engine/planner_test.go
index 1ee59a8..c34f06c 100644
--- a/engine/planner_test.go
+++ b/engine/planner_test.go
@@ -31,6 +31,7 @@ func TestPlan_Noop(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return([]*drone.Stage{
@@ -38,6 +39,11 @@ func TestPlan_Noop(t *testing.T) {
 		{Status: drone.StatusPending},
 		{Status: drone.StatusPending},
 	}, nil)
+	client.EXPECT().Queue().Return([]*drone.Stage{
+		{Status: drone.StatusRunning},
+		{Status: drone.StatusPending},
+		{Status: drone.StatusPending},
+	}, nil)
 
 	p := planner{
 		cap:     2,
@@ -72,9 +78,11 @@ func TestPlan_MinBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -117,9 +125,11 @@ func TestPlan_MaxBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     1,
@@ -159,10 +169,12 @@ func TestPlan_MoreBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -208,9 +220,11 @@ func TestPlan_MaxCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	config := config.Config{}
 	config.Pool.Min = 2
@@ -262,11 +276,13 @@ func TestPlan_MoreCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -301,9 +317,11 @@ func TestPlan_MinPool(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -341,10 +359,14 @@ func TestPlan_NoIdle(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().Update(gomock.Any(), servers[0]).Return(nil)
+	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -379,10 +401,12 @@ func TestScale_MinAge(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -399,6 +423,49 @@ func TestScale_MinAge(t *testing.T) {
 	}
 }
 
+// This test verifies that idle servers are not
+// garbage collected until the min-idle is reached.
+func TestScale_MinIdle(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	// x2 capacity
+	servers := []*autoscaler.Server{
+		{Name: "server1", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), Updated: time.Now().Unix()},
+		{Name: "server2", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), Updated: time.Now().Add(time.Hour * -1).Unix()},
+	}
+
+	// x0 running builds
+	// x0 pending builds
+	builds := []*drone.Stage{}
+
+	store := mocks.NewMockServerStore(controller)
+	store.EXPECT().List(gomock.Any()).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
+
+	client := mocks.NewMockClient(controller)
+	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
+
+	p := planner{
+		cap:     2,
+		min:     1,
+		max:     4,
+		ttu:     0,
+		tti:     time.Hour,
+		client:  client,
+		servers: store,
+	}
+
+	err := p.Plan(context.TODO())
+	if err != nil {
+		t.Error(err)
+	}
+}
+
 func TestPlan_ShutdownIdle(t *testing.T) {
 	controller := gomock.NewController(t)
 	defer controller.Finish()
@@ -417,12 +484,14 @@ func TestPlan_ShutdownIdle(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Update(gomock.Any(), servers[2]).Return(nil)
 	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -461,9 +530,11 @@ func TestPlan_ExcludePendingWhenTerminating(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers[:3], nil)
+	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers[:3], nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
+	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,

From 2e9eb7d8a827518d7bc65eaa14f594adfd66f350 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 03:21:04 -0800
Subject: [PATCH 12/16] add Busy function that is identical to Update but does
 not get wrapped by the slack notifier

---
 engine/planner.go      |  2 +-
 engine/planner_test.go |  4 ++--
 mocks/mock_server.go   | 14 ++++++++++++++
 server.go              |  3 +++
 store/servers.go       | 15 +++++++++++++++
 5 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index 915c848..e93832d 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -134,7 +134,7 @@ func (p *planner) updateBusy(ctx context.Context) (count int, err error) {
 
 	for _, server := range servers {
 		if _, ok := busy[server.Name]; ok {
-			err := p.servers.Update(ctx, server)
+			err := p.servers.Busy(ctx, server)
 			if err != nil {
 				logger.WithError(err).
 					WithField("server", server.Name).
diff --git a/engine/planner_test.go b/engine/planner_test.go
index c34f06c..0019adc 100644
--- a/engine/planner_test.go
+++ b/engine/planner_test.go
@@ -360,8 +360,8 @@ func TestPlan_NoIdle(t *testing.T) {
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
-	store.EXPECT().Update(gomock.Any(), servers[0]).Return(nil)
-	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
+	store.EXPECT().Busy(gomock.Any(), servers[0]).Return(nil)
+	store.EXPECT().Busy(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
diff --git a/mocks/mock_server.go b/mocks/mock_server.go
index 7a6064a..780c93d 100644
--- a/mocks/mock_server.go
+++ b/mocks/mock_server.go
@@ -134,3 +134,17 @@ func (mr *MockServerStoreMockRecorder) Update(arg0, arg1 interface{}) *gomock.Ca
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockServerStore)(nil).Update), arg0, arg1)
 }
+
+// Busy mocks base method
+func (m *MockServerStore) Busy(arg0 context.Context, arg1 *autoscaler.Server) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Busy", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Busy indicates an expected call of Busy
+func (mr *MockServerStoreMockRecorder) Busy(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Busy", reflect.TypeOf((*MockServerStore)(nil).Busy), arg0, arg1)
+}
diff --git a/server.go b/server.go
index 5f5e1c9..7d2b6ad 100644
--- a/server.go
+++ b/server.go
@@ -52,6 +52,9 @@ type ServerStore interface {
 	// Update the server record in the store.
 	Update(context.Context, *Server) error
 
+	// Update the server record that it is busy.
+	Busy(context.Context, *Server) error
+
 	// Delete the server record from the store.
 	Delete(context.Context, *Server) error
 
diff --git a/store/servers.go b/store/servers.go
index ed04433..329ed31 100644
--- a/store/servers.go
+++ b/store/servers.go
@@ -108,6 +108,21 @@ func (s *serverStore) Update(_ context.Context, server *autoscaler.Server) error
 	)
 }
 
+func (s *serverStore) Busy(_ context.Context, server *autoscaler.Server) error {
+	return retry.Do(
+		func() error {
+			if err := s.update(server); isConnReset(err) {
+				return err
+			} else {
+				return retry.Unrecoverable(err)
+			}
+		},
+		retry.Attempts(5),
+		retry.MaxDelay(time.Second*5),
+		retry.LastErrorOnly(true),
+	)
+}
+
 func (s *serverStore) update(server *autoscaler.Server) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

From 53f53d04ed5f50483e4d6d4098787af18bdb9e00 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 08:55:15 -0800
Subject: [PATCH 13/16] add LastBusy field in servers table

---
 engine/planner.go                             | 15 +++++++-----
 server.go                                     |  1 +
 store/migrate/mysql/ddl_gen.go                |  1 +
 .../mysql/files/001_create_table_servers.sql  |  1 +
 store/migrate/postgres/ddl_gen.go             |  1 +
 .../files/001_create_table_servers.sql        |  1 +
 store/migrate/sqlite/ddl_gen.go               |  1 +
 .../sqlite/files/001_create_table_servers.sql |  1 +
 store/servers.go                              | 24 ++++++++++++++++---
 9 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index e93832d..f4f6cb6 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -77,11 +77,14 @@ func (p *planner) Plan(ctx context.Context) error {
 
 	ctx = logger.WithContext(ctx, log)
 
-	_, err = p.updateBusy(ctx)
-	if err != nil {
-		log.WithError(err).
-			Errorln("cannot check for busy servers")
-		return err
+	// if MinIdle is being used, track busy servers
+	if p.tti > 0 {
+		_, err = p.updateBusy(ctx)
+		if err != nil {
+			log.WithError(err).
+				Errorln("cannot check for busy servers")
+			return err
+		}
 	}
 
 	free := max(capacity-running-p.buffer, 0)
@@ -139,7 +142,7 @@ func (p *planner) updateBusy(ctx context.Context) (count int, err error) {
 				logger.WithError(err).
 					WithField("server", server.Name).
 					WithField("updated", server.Updated).
-					Errorln("cannot update busy server")
+					Errorln("cannot update server as busy")
 			}
 			logger.WithField("server", server.Name).
 				Debugln("updated busy server")
diff --git a/server.go b/server.go
index 7d2b6ad..4a0063f 100644
--- a/server.go
+++ b/server.go
@@ -84,4 +84,5 @@ type Server struct {
 	Updated  int64        `db:"server_updated"  json:"updated"`
 	Started  int64        `db:"server_started"  json:"started"`
 	Stopped  int64        `db:"server_stopped"  json:"stopped"`
+	LastBusy int64        `db:"server_lastbusy" json:"lastbusy"`
 }
diff --git a/store/migrate/mysql/ddl_gen.go b/store/migrate/mysql/ddl_gen.go
index f9545de..886f161 100644
--- a/store/migrate/mysql/ddl_gen.go
+++ b/store/migrate/mysql/ddl_gen.go
@@ -121,6 +121,7 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 `
 
diff --git a/store/migrate/mysql/files/001_create_table_servers.sql b/store/migrate/mysql/files/001_create_table_servers.sql
index 794f01d..00ea268 100644
--- a/store/migrate/mysql/files/001_create_table_servers.sql
+++ b/store/migrate/mysql/files/001_create_table_servers.sql
@@ -21,6 +21,7 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/migrate/postgres/ddl_gen.go b/store/migrate/postgres/ddl_gen.go
index 63c5d2f..a3dbd40 100644
--- a/store/migrate/postgres/ddl_gen.go
+++ b/store/migrate/postgres/ddl_gen.go
@@ -121,6 +121,7 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 `
 
diff --git a/store/migrate/postgres/files/001_create_table_servers.sql b/store/migrate/postgres/files/001_create_table_servers.sql
index 45b6105..c204279 100644
--- a/store/migrate/postgres/files/001_create_table_servers.sql
+++ b/store/migrate/postgres/files/001_create_table_servers.sql
@@ -21,6 +21,7 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/migrate/sqlite/ddl_gen.go b/store/migrate/sqlite/ddl_gen.go
index 049aef1..86f93b5 100644
--- a/store/migrate/sqlite/ddl_gen.go
+++ b/store/migrate/sqlite/ddl_gen.go
@@ -121,6 +121,7 @@ CREATE TABLE IF NOT EXISTS servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 `
 
diff --git a/store/migrate/sqlite/files/001_create_table_servers.sql b/store/migrate/sqlite/files/001_create_table_servers.sql
index 246ca47..bb64ad5 100644
--- a/store/migrate/sqlite/files/001_create_table_servers.sql
+++ b/store/migrate/sqlite/files/001_create_table_servers.sql
@@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
+,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/servers.go b/store/servers.go
index 329ed31..1af10aa 100644
--- a/store/servers.go
+++ b/store/servers.go
@@ -108,10 +108,23 @@ func (s *serverStore) Update(_ context.Context, server *autoscaler.Server) error
 	)
 }
 
+func (s *serverStore) update(server *autoscaler.Server) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	server.Updated = time.Now().Unix()
+	stmt, args, err := s.db.BindNamed(serverUpdateStmt, server)
+	if err != nil {
+		return err
+	}
+	_, err = s.db.ExecContext(noContext, stmt, args...)
+	return err
+}
+
 func (s *serverStore) Busy(_ context.Context, server *autoscaler.Server) error {
 	return retry.Do(
 		func() error {
-			if err := s.update(server); isConnReset(err) {
+			if err := s.busy(server); isConnReset(err) {
 				return err
 			} else {
 				return retry.Unrecoverable(err)
@@ -123,11 +136,11 @@ func (s *serverStore) Busy(_ context.Context, server *autoscaler.Server) error {
 	)
 }
 
-func (s *serverStore) update(server *autoscaler.Server) error {
+func (s *serverStore) busy(server *autoscaler.Server) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	server.Updated = time.Now().Unix()
+	server.LastBusy = time.Now().Unix()
 	stmt, args, err := s.db.BindNamed(serverUpdateStmt, server)
 	if err != nil {
 		return err
@@ -182,6 +195,7 @@ SELECT
 ,server_updated
 ,server_started
 ,server_stopped
+,server_lastbusy
 FROM servers
 WHERE server_name=:server_name
 `
@@ -234,6 +248,7 @@ SELECT
 ,server_updated
 ,server_started
 ,server_stopped
+,server_lastbusy
 FROM servers
 WHERE server_state=:server_state
 ORDER BY server_created ASC
@@ -261,6 +276,7 @@ INSERT INTO servers (
 ,server_updated
 ,server_started
 ,server_stopped
+,server_lastbusy
 ) VALUES (
  :server_name
 ,:server_id
@@ -282,6 +298,7 @@ INSERT INTO servers (
 ,:server_updated
 ,:server_started
 ,:server_stopped
+,:server_lastbusy
 )
 `
 
@@ -305,6 +322,7 @@ UPDATE servers SET
 ,server_updated=:server_updated
 ,server_started=:server_started
 ,server_stopped=:server_stopped
+,server_lastbusy=:server_lastbusy
 WHERE server_name=:server_name
 `
 

From 3aa1a84b55c04e51f4c608c09f7fb09bd692203b Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 09:20:54 -0800
Subject: [PATCH 14/16] move to LastBusy field; fix tests again

---
 engine/planner.go      |  4 ++--
 engine/planner_test.go | 36 +++++-------------------------------
 2 files changed, 7 insertions(+), 33 deletions(-)

diff --git a/engine/planner.go b/engine/planner.go
index f4f6cb6..f1c4fe4 100644
--- a/engine/planner.go
+++ b/engine/planner.go
@@ -237,10 +237,10 @@ func (p *planner) mark(ctx context.Context, n int) error {
 		}
 
 		// skip servers that have not reached a min idle time
-		if time.Now().Before(time.Unix(server.Updated, 0).Add(p.tti)) {
+		if time.Now().Before(time.Unix(server.LastBusy, 0).Add(p.tti)) {
 			logger.
 				WithField("server", server.Name).
-				WithField("idle", timeDiff(time.Now(), time.Unix(server.Updated, 0))).
+				WithField("idle", timeDiff(time.Now(), time.Unix(server.LastBusy, 0))).
 				WithField("min-idle", p.tti).
 				Debugln("server min-idle not reached")
 			continue
diff --git a/engine/planner_test.go b/engine/planner_test.go
index 0019adc..3645626 100644
--- a/engine/planner_test.go
+++ b/engine/planner_test.go
@@ -31,7 +31,6 @@ func TestPlan_Noop(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return([]*drone.Stage{
@@ -39,11 +38,6 @@ func TestPlan_Noop(t *testing.T) {
 		{Status: drone.StatusPending},
 		{Status: drone.StatusPending},
 	}, nil)
-	client.EXPECT().Queue().Return([]*drone.Stage{
-		{Status: drone.StatusRunning},
-		{Status: drone.StatusPending},
-		{Status: drone.StatusPending},
-	}, nil)
 
 	p := planner{
 		cap:     2,
@@ -78,11 +72,9 @@ func TestPlan_MinBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -125,11 +117,9 @@ func TestPlan_MaxBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     1,
@@ -169,12 +159,10 @@ func TestPlan_MoreBufferCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -220,11 +208,9 @@ func TestPlan_MaxCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	config := config.Config{}
 	config.Pool.Min = 2
@@ -276,13 +262,11 @@ func TestPlan_MoreCapacity(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 	store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -317,11 +301,9 @@ func TestPlan_MinPool(t *testing.T) {
 
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -359,14 +341,10 @@ func TestPlan_NoIdle(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
-	store.EXPECT().Busy(gomock.Any(), servers[0]).Return(nil)
-	store.EXPECT().Busy(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -401,12 +379,10 @@ func TestScale_MinAge(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -431,8 +407,8 @@ func TestScale_MinIdle(t *testing.T) {
 
 	// x2 capacity
 	servers := []*autoscaler.Server{
-		{Name: "server1", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), Updated: time.Now().Unix()},
-		{Name: "server2", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), Updated: time.Now().Add(time.Hour * -1).Unix()},
+		{Name: "server1", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), LastBusy: time.Now().Add(time.Minute * 10 * -1).Unix()},
+		{Name: "server2", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix(), LastBusy: time.Now().Add(time.Minute * 60 * -1).Unix()},
 	}
 
 	// x0 running builds
@@ -443,6 +419,8 @@ func TestScale_MinIdle(t *testing.T) {
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
+	// we should expect a call to shut down only server2 since the last busy
+	// time of 1 hour ago satisfies the MinIdle (tti) of 30 minutes.
 	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
@@ -455,7 +433,7 @@ func TestScale_MinIdle(t *testing.T) {
 		min:     1,
 		max:     4,
 		ttu:     0,
-		tti:     time.Hour,
+		tti:     time.Minute * 30,
 		client:  client,
 		servers: store,
 	}
@@ -484,14 +462,12 @@ func TestPlan_ShutdownIdle(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil)
 	store.EXPECT().Update(gomock.Any(), servers[2]).Return(nil)
 	store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,
@@ -530,11 +506,9 @@ func TestPlan_ExcludePendingWhenTerminating(t *testing.T) {
 	store := mocks.NewMockServerStore(controller)
 	store.EXPECT().List(gomock.Any()).Return(servers, nil)
 	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers[:3], nil)
-	store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers[:3], nil)
 
 	client := mocks.NewMockClient(controller)
 	client.EXPECT().Queue().Return(builds, nil)
-	client.EXPECT().Queue().Return(builds, nil)
 
 	p := planner{
 		cap:     2,

From fdbb1d7ae655722a6bd8a44f70524d66516db492 Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 10:41:29 -0800
Subject: [PATCH 15/16] do a proper db migration to add lastbusy column

---
 store/migrate/mysql/ddl_gen.go                      | 13 ++++++++++++-
 .../mysql/files/001_create_table_servers.sql        |  1 -
 .../migrate/mysql/files/002_add_column_lastbusy.sql |  3 +++
 store/migrate/postgres/ddl_gen.go                   | 12 ++++++++++++
 .../postgres/files/001_create_table_servers.sql     |  1 -
 .../postgres/files/002_add_column_lastbusy.sql      |  3 +++
 store/migrate/sqlite/ddl_gen.go                     | 12 ++++++++++++
 .../sqlite/files/001_create_table_servers.sql       |  1 -
 .../sqlite/files/002_add_column_lastbusy.sql        |  3 +++
 9 files changed, 45 insertions(+), 4 deletions(-)
 create mode 100644 store/migrate/mysql/files/002_add_column_lastbusy.sql
 create mode 100644 store/migrate/postgres/files/002_add_column_lastbusy.sql
 create mode 100644 store/migrate/sqlite/files/002_add_column_lastbusy.sql

diff --git a/store/migrate/mysql/ddl_gen.go b/store/migrate/mysql/ddl_gen.go
index 886f161..b75fba6 100644
--- a/store/migrate/mysql/ddl_gen.go
+++ b/store/migrate/mysql/ddl_gen.go
@@ -20,6 +20,10 @@ var migrations = []struct {
 		name: "create-index-server-state",
 		stmt: createIndexServerState,
 	},
+	{
+		name: "alter-table-servers-add-column-server-lastbusy",
+		stmt: alterTableServersAddColumnServerLastbusy,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -121,7 +125,6 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 `
 
@@ -132,3 +135,11 @@ CREATE INDEX ix_servers_id ON servers (server_id);
 var createIndexServerState = `
 CREATE INDEX ix_servers_state ON servers (server_state);
 `
+
+//
+// 002_add_column_lastbusy.sql
+//
+
+var alterTableServersAddColumnServerLastbusy = `
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;
+`
diff --git a/store/migrate/mysql/files/001_create_table_servers.sql b/store/migrate/mysql/files/001_create_table_servers.sql
index 00ea268..794f01d 100644
--- a/store/migrate/mysql/files/001_create_table_servers.sql
+++ b/store/migrate/mysql/files/001_create_table_servers.sql
@@ -21,7 +21,6 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/migrate/mysql/files/002_add_column_lastbusy.sql b/store/migrate/mysql/files/002_add_column_lastbusy.sql
new file mode 100644
index 0000000..0a0a69d
--- /dev/null
+++ b/store/migrate/mysql/files/002_add_column_lastbusy.sql
@@ -0,0 +1,3 @@
+-- name: alter-table-servers-add-column-server-lastbusy
+
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;
diff --git a/store/migrate/postgres/ddl_gen.go b/store/migrate/postgres/ddl_gen.go
index a3dbd40..81d8668 100644
--- a/store/migrate/postgres/ddl_gen.go
+++ b/store/migrate/postgres/ddl_gen.go
@@ -20,6 +20,10 @@ var migrations = []struct {
 		name: "create-index-server-state",
 		stmt: createIndexServerState,
 	},
+	{
+		name: "alter-table-servers-add-column-server-lastbusy",
+		stmt: alterTableServersAddColumnServerLastbusy,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -132,3 +136,11 @@ CREATE INDEX ix_servers_id ON servers (server_id);
 var createIndexServerState = `
 CREATE INDEX ix_servers_state ON servers (server_state);
 `
+
+//
+// 002_add_column_lastbusy.sql
+//
+
+var alterTableServersAddColumnServerLastbusy = `
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;
+`
diff --git a/store/migrate/postgres/files/001_create_table_servers.sql b/store/migrate/postgres/files/001_create_table_servers.sql
index c204279..45b6105 100644
--- a/store/migrate/postgres/files/001_create_table_servers.sql
+++ b/store/migrate/postgres/files/001_create_table_servers.sql
@@ -21,7 +21,6 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/migrate/postgres/files/002_add_column_lastbusy.sql b/store/migrate/postgres/files/002_add_column_lastbusy.sql
new file mode 100644
index 0000000..0a0a69d
--- /dev/null
+++ b/store/migrate/postgres/files/002_add_column_lastbusy.sql
@@ -0,0 +1,3 @@
+-- name: alter-table-servers-add-column-server-lastbusy
+
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;
diff --git a/store/migrate/sqlite/ddl_gen.go b/store/migrate/sqlite/ddl_gen.go
index 86f93b5..d58a01a 100644
--- a/store/migrate/sqlite/ddl_gen.go
+++ b/store/migrate/sqlite/ddl_gen.go
@@ -20,6 +20,10 @@ var migrations = []struct {
 		name: "create-index-server-state",
 		stmt: createIndexServerState,
 	},
+	{
+		name: "alter-table-servers-add-column-server-lastbusy",
+		stmt: alterTableServersAddColumnServerLastbusy,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -132,3 +136,11 @@ CREATE INDEX IF NOT EXISTS ix_servers_id ON servers (server_id);
 var createIndexServerState = `
 CREATE INDEX IF NOT EXISTS ix_servers_state ON servers (server_state);
 `
+
+//
+// 002_add_column_lastbusy.sql
+//
+
+var alterTableServersAddColumnServerLastbusy = `
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;
+`
diff --git a/store/migrate/sqlite/files/001_create_table_servers.sql b/store/migrate/sqlite/files/001_create_table_servers.sql
index bb64ad5..246ca47 100644
--- a/store/migrate/sqlite/files/001_create_table_servers.sql
+++ b/store/migrate/sqlite/files/001_create_table_servers.sql
@@ -21,7 +21,6 @@ CREATE TABLE IF NOT EXISTS servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 
 -- name: create-index-server-id
diff --git a/store/migrate/sqlite/files/002_add_column_lastbusy.sql b/store/migrate/sqlite/files/002_add_column_lastbusy.sql
new file mode 100644
index 0000000..0a0a69d
--- /dev/null
+++ b/store/migrate/sqlite/files/002_add_column_lastbusy.sql
@@ -0,0 +1,3 @@
+-- name: alter-table-servers-add-column-server-lastbusy
+
+ALTER TABLE servers ADD COLUMN server_lastbusy INTEGER NOT NULL DEFAULT 0;

From 99ed3b3bd7476d8bd255ecd38dfa2a1f50d025bc Mon Sep 17 00:00:00 2001
From: Todd Stansell <todd@stansell.org>
Date: Thu, 12 Jan 2023 10:42:59 -0800
Subject: [PATCH 16/16] revert original create table statements

---
 store/migrate/postgres/ddl_gen.go | 1 -
 store/migrate/sqlite/ddl_gen.go   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/store/migrate/postgres/ddl_gen.go b/store/migrate/postgres/ddl_gen.go
index 81d8668..d749401 100644
--- a/store/migrate/postgres/ddl_gen.go
+++ b/store/migrate/postgres/ddl_gen.go
@@ -125,7 +125,6 @@ CREATE TABLE servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 `
 
diff --git a/store/migrate/sqlite/ddl_gen.go b/store/migrate/sqlite/ddl_gen.go
index d58a01a..c46b73f 100644
--- a/store/migrate/sqlite/ddl_gen.go
+++ b/store/migrate/sqlite/ddl_gen.go
@@ -125,7 +125,6 @@ CREATE TABLE IF NOT EXISTS servers (
 ,server_updated   INTEGER
 ,server_started   INTEGER
 ,server_stopped   INTEGER
-,server_lastbusy  INTEGER
 );
 `