Smart HTTP Git transport & partial clones (#291)

cmd/soft/root.go

@@ -54,6 +54,9 @@ func init() {
 func main() {
 	logger := NewDefaultLogger()
 
+	// Set global logger
+	log.SetDefault(logger)
+
 	// Set the max number of processes to the number of CPUs
 	// This is useful when running soft serve in a container
 	if _, err := maxprocs.Set(maxprocs.Logger(logger.Debugf)); err != nil {

git/repo.go

@@ -200,13 +200,6 @@ func (r *Repository) CommitsByPage(ref *Reference, page, size int) (Commits, err
 	return commits, nil
 }
 
-// UpdateServerInfo updates the repository server info.
-func (r *Repository) UpdateServerInfo() error {
-	cmd := git.NewCommand("update-server-info")
-	_, err := cmd.RunInDir(r.Path)
-	return err
-}
-
 // Config returns the config value for the given key.
 func (r *Repository) Config(key string, opts ...ConfigOptions) (string, error) {
 	dir, err := gitDir(r.Repository)

git/server.go

@@ -0,0 +1,18 @@
+package git
+
+import (
+	"context"
+
+	"github.com/gogs/git-module"
+)
+
+// UpdateServerInfo updates the server info file for the given repo path.
+func UpdateServerInfo(ctx context.Context, path string) error {
+	if !isGitDir(path) {
+		return ErrNotAGitRepository
+	}
+
+	cmd := git.NewCommand("update-server-info").WithContext(ctx).WithTimeout(-1)
+	_, err := cmd.RunInDir(path)
+	return err
+}

git/utils.go

@@ -1,6 +1,7 @@
 package git
 
 import (
+	"os"
 	"path/filepath"
 
 	"github.com/gobwas/glob"
@@ -49,3 +50,25 @@
 	}
 	return "", "", ErrFileNotFound
 }
+
+// Returns true if path is a directory containing an `objects` directory and a
+// `HEAD` file.
+func isGitDir(path string) bool {
+	stat, err := os.Stat(filepath.Join(path, "objects"))
+	if err != nil {
+		return false
+	}
+	if !stat.IsDir() {
+		return false
+	}
+
+	stat, err = os.Stat(filepath.Join(path, "HEAD"))
+	if err != nil {
+		return false
+	}
+	if stat.IsDir() {
+		return false
+	}
+
+	return true
+}

internal/log/log.go

@@ -32,6 +32,10 @@
 
 	if debug, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_DEBUG")); debug {
 		logger.SetLevel(log.DebugLevel)
+
+		if verbose, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_VERBOSE")); verbose {
+			logger.SetReportCaller(true)
+		}
 	}
 
 	logger.SetTimeFormat(cfg.Log.TimeFormat)

server/backend/sqlite/hooks.go

@@ -36,16 +36,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri
 
 	var wg sync.WaitGroup
 
-	// Update server info
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		if err := updateServerInfo(d, repo); err != nil {
-			d.logger.Error("error updating server-info", "repo", repo, "err", err)
-			return
-		}
-	}()
-
 	// Populate last-modified file.
 	wg.Add(1)
 	go func() {
@@ -59,20 +49,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri
 	wg.Wait()
 }
 
-func updateServerInfo(d *SqliteBackend, repo string) error {
-	rr, err := d.Repository(repo)
-	if err != nil {
-		return err
-	}
-
-	r, err := rr.Open()
-	if err != nil {
-		return err
-	}
-
-	return r.UpdateServerInfo()
-}
-
 func populateLastModified(d *SqliteBackend, repo string) error {
 	var rr *Repo
 	_rr, err := d.Repository(repo)

server/backend/sqlite/sqlite.go

@@ -151,17 +151,12 @@ func (d *SqliteBackend) CreateRepository(name string, opts backend.RepositoryOpt
 			return err
 		}
 
-		rr, err := git.Init(rp, true)
+		_, err := git.Init(rp, true)
 		if err != nil {
 			d.logger.Debug("failed to create repository", "err", err)
 			return err
 		}
 
-		if err := rr.UpdateServerInfo(); err != nil {
-			d.logger.Debug("failed to update server info", "err", err)
-			return err
-		}
-
 		return nil
 	}); err != nil {
 		d.logger.Debug("failed to create repository in database", "err", err)

server/config/config.go

@@ -114,6 +114,40 @@
 	Backend backend.Backend `yaml:"-"`
 }
 
+// Environ returns the config as a list of environment variables.
+func (c *Config) Environ() []string {
+	envs := []string{}
+	if c == nil {
+		return envs
+	}
+
+	// TODO: do this dynamically
+	envs = append(envs, []string{
+		fmt.Sprintf("SOFT_SERVE_NAME=%s", c.Name),
+		fmt.Sprintf("SOFT_SERVE_DATA_PATH=%s", c.DataPath),
+		fmt.Sprintf("SOFT_SERVE_INITIAL_ADMIN_KEYS=%s", strings.Join(c.InitialAdminKeys, "\n")),
+		fmt.Sprintf("SOFT_SERVE_SSH_LISTEN_ADDR=%s", c.SSH.ListenAddr),
+		fmt.Sprintf("SOFT_SERVE_SSH_PUBLIC_URL=%s", c.SSH.PublicURL),
+		fmt.Sprintf("SOFT_SERVE_SSH_KEY_PATH=%s", c.SSH.KeyPath),
+		fmt.Sprintf("SOFT_SERVE_SSH_CLIENT_KEY_PATH=%s", c.SSH.ClientKeyPath),
+		fmt.Sprintf("SOFT_SERVE_SSH_MAX_TIMEOUT=%d", c.SSH.MaxTimeout),
+		fmt.Sprintf("SOFT_SERVE_SSH_IDLE_TIMEOUT=%d", c.SSH.IdleTimeout),
+		fmt.Sprintf("SOFT_SERVE_GIT_LISTEN_ADDR=%s", c.Git.ListenAddr),
+		fmt.Sprintf("SOFT_SERVE_GIT_MAX_TIMEOUT=%d", c.Git.MaxTimeout),
+		fmt.Sprintf("SOFT_SERVE_GIT_IDLE_TIMEOUT=%d", c.Git.IdleTimeout),
+		fmt.Sprintf("SOFT_SERVE_GIT_MAX_CONNECTIONS=%d", c.Git.MaxConnections),
+		fmt.Sprintf("SOFT_SERVE_HTTP_LISTEN_ADDR=%s", c.HTTP.ListenAddr),
+		fmt.Sprintf("SOFT_SERVE_HTTP_TLS_KEY_PATH=%s", c.HTTP.TLSKeyPath),
+		fmt.Sprintf("SOFT_SERVE_HTTP_TLS_CERT_PATH=%s", c.HTTP.TLSCertPath),
+		fmt.Sprintf("SOFT_SERVE_HTTP_PUBLIC_URL=%s", c.HTTP.PublicURL),
+		fmt.Sprintf("SOFT_SERVE_STATS_LISTEN_ADDR=%s", c.Stats.ListenAddr),
+		fmt.Sprintf("SOFT_SERVE_LOG_FORMAT=%s", c.Log.Format),
+		fmt.Sprintf("SOFT_SERVE_LOG_TIME_FORMAT=%s", c.Log.TimeFormat),
+	}...)
+
+	return envs
+}
+
 func parseConfig(path string) (*Config, error) {
 	dataPath := filepath.Dir(path)
 	cfg := &Config{

server/daemon/conn.go

@@ -0,0 +1,105 @@
+package daemon
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"time"
+)
+
+// connections is a synchronizes access to to a net.Conn pool.
+type connections struct {
+	m  map[net.Conn]struct{}
+	mu sync.Mutex
+}
+
+func (m *connections) Add(c net.Conn) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.m[c] = struct{}{}
+}
+
+func (m *connections) Close(c net.Conn) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	err := c.Close()
+	delete(m.m, c)
+	return err
+}
+
+func (m *connections) Size() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return len(m.m)
+}
+
+func (m *connections) CloseAll() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var err error
+	for c := range m.m {
+		err = errors.Join(err, c.Close())
+		delete(m.m, c)
+	}
+
+	return err
+}
+
+// serverConn is a wrapper around a net.Conn that closes the connection when
+// the one of the timeouts is reached.
+type serverConn struct {
+	net.Conn
+
+	initTimeout   time.Duration
+	idleTimeout   time.Duration
+	maxDeadline   time.Time
+	closeCanceler context.CancelFunc
+}
+
+var _ net.Conn = (*serverConn)(nil)
+
+func (c *serverConn) Write(p []byte) (n int, err error) {
+	c.updateDeadline()
+	n, err = c.Conn.Write(p)
+	if _, isNetErr := err.(net.Error); isNetErr && c.closeCanceler != nil {
+		c.closeCanceler()
+	}
+	return
+}
+
+func (c *serverConn) Read(b []byte) (n int, err error) {
+	c.updateDeadline()
+	n, err = c.Conn.Read(b)
+	if _, isNetErr := err.(net.Error); isNetErr && c.closeCanceler != nil {
+		c.closeCanceler()
+	}
+	return
+}
+
+func (c *serverConn) Close() (err error) {
+	err = c.Conn.Close()
+	if c.closeCanceler != nil {
+		c.closeCanceler()
+	}
+	return
+}
+
+func (c *serverConn) updateDeadline() {
+	switch {
+	case c.initTimeout > 0:
+		initTimeout := time.Now().Add(c.initTimeout)
+		c.initTimeout = 0
+		if initTimeout.Unix() < c.maxDeadline.Unix() || c.maxDeadline.IsZero() {
+			c.Conn.SetDeadline(initTimeout)
+			return
+		}
+	case c.idleTimeout > 0:
+		idleDeadline := time.Now().Add(c.idleTimeout)
+		if idleDeadline.Unix() < c.maxDeadline.Unix() || c.maxDeadline.IsZero() {
+			c.Conn.SetDeadline(idleDeadline)
+			return
+		}
+	}
+	c.Conn.SetDeadline(c.maxDeadline)
+}