Manager: speed up sequential job deletion by checking db when queue empty

Job deletions are placed in an in-memory queue in batches of 100 jobs.
Between batches the Manager's job deleter would idle for 1 minute. Now,
once the in-memory queue has been emptied, the job deleter will wait
only 100ms before checking the database again.

This 100ms might not be necessary either, but I think it's nice to give
the Manager a bit of a breather before diving into another batch of
deletions.

internal/manager/job_deleter/job_deleter.go

@@ -150,19 +150,28 @@ func (s *Service) Run(ctx context.Context) {
 	log.Debug().Msg("job deleter: running")
 	defer log.Debug().Msg("job deleter: shutting down")
 
+	waitTime := jobDeletionCheckInterval
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case jobUUID := <-s.queue:
 			s.deleteJob(ctx, jobUUID)
-		case <-time.After(jobDeletionCheckInterval):
+
+			if len(s.queue) == 0 {
+				waitTime = 100 * time.Millisecond
+			}
+		case <-time.After(waitTime):
 			// Inspect the database to see if there was anything marked for deletion
 			// without getting into our queue. This can happen when lots of jobs are
 			// queued in quick succession, as then the queue channel gets full.
 			if len(s.queue) == 0 {
 				s.queuePendingDeletions(ctx)
 			}
+
+			// The next iteration should just wait for the default duration.
+			waitTime = jobDeletionCheckInterval
 		}
 	}
 }