Skip to content
This repository has been archived by the owner on Jul 2, 2024. It is now read-only.

Commit

Permalink
adding script to find popular, but unused projects
Browse files Browse the repository at this point in the history
Issue: #14
  • Loading branch information
adamdecaf committed Sep 12, 2018
1 parent 139fce5 commit 08a6a8c
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 0 deletions.
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,6 @@ If you feel there's a project that would benefit from our help, please open an
issue above. The issue itself will have some questions that you can answer to
help us discover more about the project. From there we can begin to evaluate the
project, and our ability to properly support it.

We've also started writing a script that tries to find projects which are highly
used, but also lacking recent updates.
11 changes: 11 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
module github.com/gofrs/help-requests

require (
github.com/golang/protobuf v1.2.0 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
google.golang.org/appengine v1.1.0 // indirect
)
14 changes: 14 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
115 changes: 115 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package main

import (
"context"
"errors"
"fmt"
"math"
"net/http"
"os"
"strconv"
"strings"
"text/tabwriter"
"time"

"github.com/google/go-github/github"
"golang.org/x/net/html"
"golang.org/x/oauth2"
)

func main() {
ghClient, err := createGithubClient(context.Background())
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: problem creating github client: %v", err)
}

query := "stars:>100 pushed:<2018-01-01 language:Go"
repoRes, res, err := ghClient.Search.Repositories(context.Background(), query, &github.SearchOptions{
Sort: "stars",
Order: "desc",
ListOptions: github.ListOptions{
PerPage: 25,
Page: 1,
},
})
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: problem reading github repositories: %v", err)
}
res.Close = true

w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
fmt.Fprintf(w, "name\tstars\tlast commit (days)\timporters\n")
defer w.Flush()
for i := range repoRes.Repositories {
repo := repoRes.Repositories[i]

cleanName := strings.Replace(*repo.HTMLURL, `https://`, "", 1)

// TODO(adam): goroutines + sync.WaitGroup
importers, err := scrapeGodocImports(cleanName)
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: problem grabbing %s godoc importers: %v\n", cleanName, err)
}

days := int(math.Abs(float64(repo.PushedAt.Sub(time.Now()).Hours()) / 24.0))
fmt.Fprintf(w, "%s\t%d\t%d\t%d\n", cleanName, *repo.StargazersCount, days, importers)
}
}

func createGithubClient(ctx context.Context) (*github.Client, error) {
v := os.Getenv("GITHUB_TOKEN")
if v == "" {
return nil, errors.New("environment variable GITHUB_TOKEN is required")
}
ts := oauth2.StaticTokenSource(&oauth2.Token{
AccessToken: v,
})
tc := oauth2.NewClient(ctx, ts)
return github.NewClient(tc), nil
}

func scrapeGodocImports(importPath string) (int, error) {
req, err := http.NewRequest("GET", "https://godoc.org/"+importPath, nil)
if err != nil {
return -1, fmt.Errorf("problem loading godoc.org: %v", err)
}
req.Header.Set("User-Agent", "Gofrs popstalerepo bot")

resp, err := http.DefaultClient.Do(req)
if err != nil {
return -1, fmt.Errorf("problem loading %s: %v", req.URL, err)
}
defer resp.Body.Close()

// recursive search, from /x/net/html docs
var f func(n *html.Node) (int, error)
f = func(n *html.Node) (int, error) {
if n.Type == html.ElementNode && n.Data == "a" {
for _, a := range n.Attr {
// TODO(adam): we should try and refresh importers
// when running into errors.
if a.Key == "href" && strings.Contains(a.Val, "?importers") {
parts := strings.Fields(n.FirstChild.Data)
n, err := strconv.Atoi(parts[0])
if err != nil {
return -1, fmt.Errorf("couldn't parse %q: %v", parts[0], err)
}
return n, nil
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
n, err := f(c)
if err == nil && n > 0 {
return n, err
}
}
return -1, errors.New(`didn't find <a href="?importers">`)
}

doc, err := html.Parse(resp.Body)
if err != nil {
return -1, fmt.Errorf("couldn't parse html: %v", err)
}
return f(doc)
}

0 comments on commit 08a6a8c

Please sign in to comment.