Github support rate limit (#85)

Reviewed-on: https://gitea.com/gitea/changelog/pulls/85

.gitea/workflows/release-nightly.yml

@@ -8,13 +8,13 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '>=1.20.1'
+          go-version-file: 'go.mod'
       - name: goreleaser
         uses: https://github.com/goreleaser/goreleaser-action@v4
         with:

.gitea/workflows/release-tag.yml

@@ -9,13 +9,13 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '>=1.20.1'
+          go-version-file: 'go.mod'
       - name: Import GPG key
         id: import_gpg
         uses: https://github.com/crazy-max/ghaction-import-gpg@v5

.gitea/workflows/test-pr.yml

@@ -6,10 +6,10 @@ jobs:
   check-and-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '>=1.20.1'
+          go-version-file: 'go.mod'
       - name: check-and-test
         run: |
           go test -race ./...

cmd/generate.go

@@ -21,7 +21,7 @@ var Generate = &cli.Command{
 	Action: runGenerate,
 }
 
-func runGenerate(_ *cli.Context) error {
+func runGenerate(ctx *cli.Context) error {
 	cfg, err := config.New(configPathFlag)
 	if err != nil {
 		return err
@@ -32,7 +32,7 @@ func runGenerate(_ *cli.Context) error {
 		return err
 	}
 
-	title, prs, err := s.Generate()
+	title, prs, err := s.Generate(ctx.Context)
 	if err != nil {
 		return err
 	}

cmd/init.go

@@ -6,7 +6,6 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 
 	"code.gitea.io/changelog/config"
@@ -37,7 +36,7 @@ func runInit(_ *cli.Context) error {
 		return fmt.Errorf("file '%s' already exists", nameFlag)
 	}
 
-	if err := ioutil.WriteFile(nameFlag, config.DefaultConfig, os.ModePerm); err != nil {
+	if err := os.WriteFile(nameFlag, config.DefaultConfig, os.ModePerm); err != nil {
 		return err
 	}
 

config/config.go

@@ -6,7 +6,7 @@ package config
 
 import (
 	_ "embed"
-	"io/ioutil"
+	"os"
 	"regexp"
 
 	"gopkg.in/yaml.v2"
@@ -64,7 +64,7 @@ func New(configPath string) (*Config, error) {
 	var err error
 	configContent := DefaultConfig
 	if len(configPath) != 0 {
-		configContent, err = ioutil.ReadFile(configPath)
+		configContent, err = os.ReadFile(configPath)
 		if err != nil {
 			return nil, err
 		}

go.mod

@@ -1,10 +1,10 @@
 module code.gitea.io/changelog
 
-go 1.18
+go 1.22
 
 require (
 	code.gitea.io/sdk/gitea v0.14.0
-	github.com/google/go-github/v50 v50.0.0
+	github.com/google/go-github/v61 v61.0.0
 	github.com/urfave/cli/v3 v3.0.0-alpha2
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	gopkg.in/yaml.v2 v2.3.0
@@ -17,7 +17,6 @@ require (
 	github.com/hashicorp/go-version v1.2.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 )

go.sum

@@ -8,9 +8,10 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-github/v50 v50.0.0 h1:gdO1AeuSZZK4iYWwVbjni7zg8PIQhp7QfmPunr016Jk=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v50 v50.0.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA=
+github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go=
+github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
@@ -27,8 +28,6 @@ github.com/urfave/cli/v3 v3.0.0-alpha2/go.mod h1:gHI/xEYplFhOa3Y90xJleh3kqqsSanB
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=

service/gitea.go

@@ -5,6 +5,7 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -23,7 +24,7 @@ type Gitea struct {
 }
 
 // Generate returns a Gitea changelog
-func (ge *Gitea) Generate() (string, []Entry, error) {
+func (ge *Gitea) Generate(_ context.Context) (string, []Entry, error) {
 	client, err := gitea.NewClient(ge.BaseURL, gitea.SetToken(ge.Token))
 	if err != nil {
 		return "", nil, err

service/github.go

@@ -8,13 +8,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	log "log/slog"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/google/go-github/v50/github"
+	"github.com/google/go-github/v61/github"
 	"golang.org/x/oauth2"
 )
 
@@ -25,7 +26,45 @@ type GitHub struct {
 	Token     string
 	Repo      string
 	Issues    bool
+	ctx       context.Context
 	client    *github.Client
+	rate      *github.Rate
+}
+
+func (gh *GitHub) setRate(rate *github.Rate) {
+	gh.rate = rate
+}
+
+func (gh *GitHub) RefreshRate() error {
+	rates, _, err := gh.client.RateLimit.Get(gh.ctx)
+	if err != nil {
+		// if rate limit is not enabled, ignore it
+		if strings.Contains(err.Error(), "404") {
+			gh.setRate(nil)
+			return nil
+		}
+		return err
+	}
+
+	gh.setRate(rates.GetCore())
+	return nil
+}
+
+func (gh *GitHub) waitAndPickClient() {
+	for gh.rate != nil && gh.rate.Remaining <= 0 {
+		timer := time.NewTimer(time.Until(gh.rate.Reset.Time))
+		select {
+		case <-gh.ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+
+		err := gh.RefreshRate()
+		if err != nil {
+			log.Error("g.getClient().RateLimit.Get: %s", err)
+		}
+	}
 }
 
 // OwnerRepo splits owner/repo
@@ -38,12 +77,11 @@ func (gh *GitHub) OwnerRepo() (string, string) {
 }
 
 // Generate returns a GitHub changelog
-func (gh *GitHub) Generate() (string, []Entry, error) {
+func (gh *GitHub) Generate(ctx context.Context) (string, []Entry, error) {
 	owner, repo := gh.OwnerRepo()
-	ctx := context.Background()
 	gh.initClient(ctx)
 
-	tagURL := fmt.Sprintf("## [%s](https://github.com/%s/releases/tag/%s) - %s", gh.Milestone, gh.Repo, gh.GitTag, time.Now().Format("2006-01-02"))
+	tagURL := fmt.Sprintf("## [%s](https://github.com/%s/releases/tag/v%s) - %s", gh.Milestone, gh.Repo, gh.GitTag, time.Now().Format("2006-01-02"))
 
 	prs := make([]Entry, 0)
 
@@ -55,7 +93,8 @@ func (gh *GitHub) Generate() (string, []Entry, error) {
 	p := 1
 	perPage := 100
 	for {
-		result, _, err := gh.client.Issues.ListByRepo(ctx, owner, repo, &github.IssueListByRepoOptions{
+		gh.waitAndPickClient()
+		result, resp, err := gh.client.Issues.ListByRepo(ctx, owner, repo, &github.IssueListByRepoOptions{
 			Milestone: strconv.Itoa(milestoneNum),
 			State:     "closed",
 			ListOptions: github.ListOptions{
@@ -66,6 +105,7 @@ func (gh *GitHub) Generate() (string, []Entry, error) {
 		if err != nil {
 			return "", nil, err
 		}
+		gh.setRate(&resp.Rate)
 		p++
 
 		isPull := !(gh.Issues)
@@ -111,7 +151,8 @@ func (gh *GitHub) Contributors() (ContributorList, error) {
 	p := 1
 	perPage := 100
 	for {
-		result, _, err := gh.client.Issues.ListByRepo(ctx, owner, repo, &github.IssueListByRepoOptions{
+		gh.waitAndPickClient()
+		result, resp, err := gh.client.Issues.ListByRepo(ctx, owner, repo, &github.IssueListByRepoOptions{
 			Milestone: strconv.Itoa(milestoneNum),
 			State:     "closed",
 			ListOptions: github.ListOptions{
@@ -122,6 +163,7 @@ func (gh *GitHub) Contributors() (ContributorList, error) {
 		if err != nil {
 			return nil, err
 		}
+		gh.setRate(&resp.Rate)
 		p++
 
 		for _, pr := range result {
@@ -158,6 +200,7 @@ func (gh *GitHub) initClient(ctx context.Context) {
 	}
 
 	gh.client = github.NewClient(cl)
+	gh.ctx = ctx
 }
 
 func (gh *GitHub) milestoneNum(ctx context.Context) (int, error) {
@@ -165,7 +208,8 @@ func (gh *GitHub) milestoneNum(ctx context.Context) (int, error) {
 	p := 1
 	perPage := 100
 	for {
-		milestones, _, err := gh.client.Issues.ListMilestones(ctx, owner, repo, &github.MilestoneListOptions{
+		gh.waitAndPickClient()
+		milestones, resp, err := gh.client.Issues.ListMilestones(ctx, owner, repo, &github.MilestoneListOptions{
 			State: "all",
 			ListOptions: github.ListOptions{
 				Page:    p,
@@ -177,6 +221,7 @@ func (gh *GitHub) milestoneNum(ctx context.Context) (int, error) {
 		if err != nil {
 			return 0, err
 		}
+		gh.setRate(&resp.Rate)
 		p++
 
 		for _, milestone := range milestones {

service/github_test.go

@@ -4,7 +4,10 @@
 
 package service
 
-import "testing"
+import (
+	"context"
+	"testing"
+)
 
 var gh = &GitHub{
 	Milestone: "1.1.0", // https://github.com/go-gitea/test_repo/milestone/2?closed=1
@@ -12,7 +15,7 @@ var gh = &GitHub{
 }
 
 func TestGitHubGenerate(t *testing.T) {
-	_, entries, err := gh.Generate()
+	_, entries, err := gh.Generate(context.Background())
 	if err != nil {
 		t.Log(err)
 		t.FailNow()

service/service.go

@@ -5,6 +5,7 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"unicode"
@@ -48,7 +49,7 @@ func New(serviceType, repo, baseURL, milestone, tag, token string, issues bool)
 
 // Service defines how a struct can be a Changelog Service
 type Service interface {
-	Generate() (string, []Entry, error)
+	Generate(ctx context.Context) (string, []Entry, error)
 	Contributors() (ContributorList, error)
 }
 
@@ -90,7 +91,6 @@ func (cl ContributorList) Swap(i, j int) {
 
 // CleanTitle returns the string with spaces trimmed and the first rune title-cased
 func CleanTitle(s string) string {
-
 	s = strings.TrimSpace(s)
 
 	r := []rune(s)