Automatically pause queue if index service is unavailable (#15066)

* Handle keyword search error when issue indexer service is not available

* Implement automatic disabling and resume of code indexer queue

modules/indexer/code/indexer.go

@@ -42,9 +42,11 @@ type SearchResultLanguages struct {
 
 // Indexer defines an interface to index and search code contents
 type Indexer interface {
+	Ping() bool
+	SetAvailabilityChangeCallback(callback func(bool))
 	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
 	Delete(repoID int64) error
-	Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
+	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
 	Close()
 }
 
@@ -140,6 +142,7 @@ func Init() {
 				return data
 			}
 
+			unhandled := make([]queue.Data, 0, len(data))
 			for _, datum := range data {
 				indexerData, ok := datum.(*IndexerData)
 				if !ok {
@@ -150,10 +153,14 @@ func Init() {
 
 				if err := index(ctx, indexer, indexerData.RepoID); err != nil {
 					log.Error("index: %v", err)
-					continue
+					if indexer.Ping() {
+						continue
+					}
+					// Add back to queue
+					unhandled = append(unhandled, datum)
 				}
 			}
-			return nil
+			return unhandled
 		}
 
 		indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{})
@@ -212,6 +219,18 @@ func Init() {
 
 		indexer.set(rIndexer)
 
+		if queue, ok := indexerQueue.(queue.Pausable); ok {
+			rIndexer.SetAvailabilityChangeCallback(func(available bool) {
+				if !available {
+					log.Info("Code index queue paused")
+					queue.Pause()
+				} else {
+					log.Info("Code index queue resumed")
+					queue.Resume()
+				}
+			})
+		}
+
 		// Start processing the queue
 		go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
 
@@ -262,6 +281,17 @@ func UpdateRepoIndexer(repo *repo_model.Repository) {
 	}
 }
 
+// IsAvailable checks if issue indexer is available
+func IsAvailable() bool {
+	idx, err := indexer.get()
+	if err != nil {
+		log.Error("IsAvailable(): unable to get indexer: %v", err)
+		return false
+	}
+
+	return idx.Ping()
+}
+
 // populateRepoIndexer populate the repo indexer with pre-existing data. This
 // should only be run when the indexer is created for the first time.
 func populateRepoIndexer(ctx context.Context) {