Update githttp (#40)

githttp now gained the ability to cache lockfiles and repositories.

cmd/omegaup-gitserver/health_test.go

@@ -30,6 +30,8 @@ func TestHealth(t *testing.T) {
 	if os.Getenv("PRESERVE") == "" {
 		defer os.RemoveAll(tmpDir)
 	}
+	m := githttp.NewLockfileManager()
+	defer m.Clear()
 
 	log, err := log15.New("info", false)
 	if err != nil {
@@ -47,6 +49,7 @@ func TestHealth(t *testing.T) {
 	ts := httptest.NewUnstartedServer(nil)
 	ts.Config.Handler = muxHandler(
 		nil,
+		m,
 		uint16(ts.Listener.Addr().(*net.TCPAddr).Port),
 		tmpDir,
 		gitserver.NewGitProtocol(gitserver.GitProtocolOpts{
@@ -57,6 +60,7 @@ func TestHealth(t *testing.T) {
 			AllowDirectPushToMaster:     config.Gitserver.AllowDirectPushToMaster,
 			HardOverallWallTimeLimit:    gitserver.OverallWallTimeHardLimit,
 			InteractiveSettingsCompiler: fakeInteractiveSettingsCompiler,
+			LockfileManager:             m,
 		}),
 		log,
 	)

cmd/omegaup-gitserver/main.go

@@ -78,6 +78,7 @@ type muxGitHandler struct {
 
 func muxHandler(
 	app *newrelic.Application,
+	lockfileManager *githttp.LockfileManager,
 	port uint16,
 	rootPath string,
 	protocol *githttp.GitProtocol,
@@ -86,18 +87,20 @@ func muxHandler(
 	metrics, metricsHandler := gitserver.SetupMetrics(ProgramVersion)
 	tracing := nrtracing.New(app)
 	_, wrappedGitHandler := tracing.WrapHandle("/", gitserver.NewGitHandler(gitserver.GitHandlerOpts{
-		RootPath: rootPath,
-		Protocol: protocol,
-		Metrics:  metrics,
-		Log:      log,
-		Tracing:  tracing,
+		RootPath:        rootPath,
+		Protocol:        protocol,
+		Metrics:         metrics,
+		Log:             log,
+		LockfileManager: lockfileManager,
+		Tracing:         tracing,
 	}))
 	_, wrappedZipHandler := tracing.WrapHandle("/", gitserver.NewZipHandler(gitserver.ZipHandlerOpts{
-		RootPath: rootPath,
-		Protocol: protocol,
-		Metrics:  metrics,
-		Log:      log,
-		Tracing:  tracing,
+		RootPath:        rootPath,
+		Protocol:        protocol,
+		Metrics:         metrics,
+		Log:             log,
+		LockfileManager: lockfileManager,
+		Tracing:         tracing,
 	}))
 	_, wrappedHealthHandler := tracing.WrapHandle("/health", gitserver.HealthHandler(
 		rootPath,
@@ -186,6 +189,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	lockfileManager := githttp.NewLockfileManager()
+	defer lockfileManager.Clear()
+
 	protocol := gitserver.NewGitProtocol(gitserver.GitProtocolOpts{
 		GitProtocolOpts: githttp.GitProtocolOpts{
 			AuthCallback:               authCallback,
@@ -198,6 +204,7 @@ func main() {
 			LibinteractiveJarPath: config.Gitserver.LibinteractivePath,
 			Log:                   log,
 		},
+		LockfileManager: lockfileManager,
 	})
 
 	var servers []*http.Server
@@ -208,6 +215,7 @@ func main() {
 		Handler: http.TimeoutHandler(
 			muxHandler(
 				app,
+				lockfileManager,
 				config.Gitserver.Port,
 				config.Gitserver.RootPath,
 				protocol,

cmd/omegaup-translate-problem/main.go

@@ -140,7 +140,10 @@ func createPackfileFromSplitCommit(
 		return errors.Wrapf(err, "failed to write packfile into the packbuilder")
 	}
 
-	lockfile := githttp.NewLockfile(destRepo.Path())
+	lockfileManager := githttp.NewLockfileManager()
+	defer lockfileManager.Clear()
+
+	lockfile := lockfileManager.NewLockfile(destRepo.Path())
 	defer lockfile.Unlock()
 
 	protocol := githttp.NewGitProtocol(githttp.GitProtocolOpts{
@@ -378,7 +381,10 @@ func createPackfileFromMergedCommit(
 		return nil, errors.Wrapf(err, "failed to write packfile into the packbuilder")
 	}
 
-	lockfile := githttp.NewLockfile(destRepo.Path())
+	lockfileManager := githttp.NewLockfileManager()
+	defer lockfileManager.Clear()
+
+	lockfile := lockfileManager.NewLockfile(destRepo.Path())
 	defer lockfile.Unlock()
 
 	overallWallTimeHardLimit := gitserver.OverallWallTimeHardLimit
@@ -390,6 +396,7 @@ func createPackfileFromMergedCommit(
 		GitProtocolOpts: githttp.GitProtocolOpts{
 			Log: log,
 		},
+		LockfileManager:          lockfileManager,
 		AllowDirectPushToMaster:  true,
 		HardOverallWallTimeLimit: overallWallTimeHardLimit,
 		InteractiveSettingsCompiler: &gitserver.LibinteractiveCompiler{

cmd/omegaup-update-problem/main.go

@@ -56,6 +56,7 @@ type BlobUpdate struct {
 }
 
 func commitZipFile(
+	lockfileManager *githttp.LockfileManager,
 	problemFiles common.ProblemFiles,
 	repo *git.Repository,
 	lockfile *githttp.Lockfile,
@@ -79,6 +80,7 @@ func commitZipFile(
 		GitProtocolOpts: githttp.GitProtocolOpts{
 			Log: log,
 		},
+		LockfileManager:          lockfileManager,
 		AllowDirectPushToMaster:  true,
 		HardOverallWallTimeLimit: gitserver.OverallWallTimeHardLimit,
 		InteractiveSettingsCompiler: &gitserver.LibinteractiveCompiler{
@@ -254,6 +256,7 @@ func convertBlobsToPackfile(
 }
 
 func commitBlobs(
+	lockfileManager *githttp.LockfileManager,
 	repo *git.Repository,
 	lockfile *githttp.Lockfile,
 	authorUsername string,
@@ -307,6 +310,7 @@ func commitBlobs(
 		GitProtocolOpts: githttp.GitProtocolOpts{
 			Log: log,
 		},
+		LockfileManager:          lockfileManager,
 		AllowDirectPushToMaster:  true,
 		HardOverallWallTimeLimit: gitserver.OverallWallTimeHardLimit,
 		InteractiveSettingsCompiler: &gitserver.LibinteractiveCompiler{
@@ -446,7 +450,10 @@ func main() {
 	}
 	defer repo.Free()
 
-	lockfile := githttp.NewLockfile(repo.Path())
+	lockfileManager := githttp.NewLockfileManager()
+	defer lockfileManager.Clear()
+
+	lockfile := lockfileManager.NewLockfile(repo.Path())
 	if ok, err := lockfile.TryLock(); !ok {
 		log.Info(
 			"Waiting for the lockfile",
@@ -506,6 +513,7 @@ func main() {
 		defer zipReader.Close()
 
 		updateResult, err = commitZipFile(
+			lockfileManager,
 			common.NewProblemFilesFromZip(&zipReader.Reader, *zipPath),
 			repo,
 			lockfile,
@@ -572,6 +580,7 @@ func main() {
 
 		var err error
 		updateResult, err = commitBlobs(
+			lockfileManager,
 			repo,
 			lockfile,
 			*author,
@@ -606,6 +615,7 @@ func main() {
 		}
 
 		updateResult, err = commitZipFile(
+			lockfileManager,
 			common.NewProblemFilesFromZip(zipReader, ":memory:"),
 			repo,
 			lockfile,

cmd/omegaup-update-problem/main_test.go

@@ -31,6 +31,8 @@ func getTreeOid(t *testing.T, extraFileContents map[string]io.Reader, log loggin
 	if os.Getenv("PRESERVE") == "" {
 		defer os.RemoveAll(tmpdir)
 	}
+	m := githttp.NewLockfileManager()
+	defer m.Clear()
 
 	fileContents := map[string]io.Reader{
 		"cases/0.in":             strings.NewReader("1 2"),
@@ -54,13 +56,14 @@ func getTreeOid(t *testing.T, extraFileContents map[string]io.Reader, log loggin
 		t.Fatalf("Failed to initialize bare repository: %v", err)
 	}
 
-	lockfile := githttp.NewLockfile(repo.Path())
+	lockfile := m.NewLockfile(repo.Path())
 	if err := lockfile.RLock(); err != nil {
 		t.Fatalf("Failed to acquire the lockfile: %v", err)
 	}
 	defer lockfile.Unlock()
 
 	if _, err := commitZipFile(
+		m,
 		common.NewProblemFilesFromZip(zipReader, ":memory:"),
 		repo,
 		lockfile,
@@ -217,14 +220,16 @@ func TestProblemUpdateZip(t *testing.T) {
 	if os.Getenv("PRESERVE") == "" {
 		defer os.RemoveAll(tmpdir)
 	}
+	m := githttp.NewLockfileManager()
+	defer m.Clear()
 	ctx := context.Background()
 
 	repo, err := gitserver.InitRepository(ctx, tmpdir)
 	if err != nil {
 		t.Fatalf("Failed to initialize bare repository: %v", err)
 	}
 
-	lockfile := githttp.NewLockfile(repo.Path())
+	lockfile := m.NewLockfile(repo.Path())
 	if err := lockfile.RLock(); err != nil {
 		t.Fatalf("Failed to acquire the lockfile: %v", err)
 	}
@@ -247,6 +252,7 @@ func TestProblemUpdateZip(t *testing.T) {
 		}
 
 		updateResult, err := commitZipFile(
+			m,
 			common.NewProblemFilesFromZip(zipReader, ":memory:"),
 			repo,
 			lockfile,
@@ -303,6 +309,7 @@ func TestProblemUpdateZip(t *testing.T) {
 		}
 
 		updateResult, err := commitZipFile(
+			m,
 			common.NewProblemFilesFromZip(zipReader, ":memory:"),
 			repo,
 			lockfile,
@@ -354,14 +361,16 @@ func TestProblemUpdateBlobs(t *testing.T) {
 	if os.Getenv("PRESERVE") == "" {
 		defer os.RemoveAll(tmpdir)
 	}
+	m := githttp.NewLockfileManager()
+	defer m.Clear()
 	ctx := context.Background()
 
 	repo, err := gitserver.InitRepository(ctx, tmpdir)
 	if err != nil {
 		t.Fatalf("Failed to initialize bare repository: %v", err)
 	}
 
-	lockfile := githttp.NewLockfile(repo.Path())
+	lockfile := m.NewLockfile(repo.Path())
 	if err := lockfile.RLock(); err != nil {
 		t.Fatalf("Failed to acquire the lockfile: %v", err)
 	}
@@ -384,6 +393,7 @@ func TestProblemUpdateBlobs(t *testing.T) {
 		}
 
 		updateResult, err := commitZipFile(
+			m,
 			common.NewProblemFilesFromZip(zipReader, ":memory:"),
 			repo,
 			lockfile,
@@ -427,6 +437,7 @@ func TestProblemUpdateBlobs(t *testing.T) {
 	{
 		// Typo has been corrected.
 		if _, err = commitBlobs(
+			m,
 			repo,
 			lockfile,
 			"test",

go.mod

@@ -9,11 +9,11 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/newrelic/go-agent/v3 v3.15.2
 	github.com/o1egl/paseto v1.0.0
-	github.com/omegaup/githttp/v2 v2.4.11
+	github.com/omegaup/githttp/v2 v2.4.13
 	github.com/omegaup/go-base/logging/log15/v3 v3.3.6
 	github.com/omegaup/go-base/tracing/newrelic/v3 v3.3.6
 	github.com/omegaup/go-base/v3 v3.3.6
-	github.com/omegaup/quark v1.9.32
+	github.com/omegaup/quark v1.9.33
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.8.0
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca

go.sum

@@ -232,8 +232,8 @@ github.com/o1egl/paseto v1.0.0/go.mod h1:5HxsZPmw/3RI2pAwGo1HhOOwSdvBpcuVzO7uDkm
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/omegaup/githttp/v2 v2.4.11 h1:vLONSlJbh4/d3gxGs5hRVqj9U/rP0QUqpU/n8ZWwk6o=
-github.com/omegaup/githttp/v2 v2.4.11/go.mod h1:CS+l7OI46d3JAnPEzqg6dJ5dYbIDuIxxb/mHDXNNFTQ=
+github.com/omegaup/githttp/v2 v2.4.13 h1:KGiSet8Xvy6yAo2UaoXKRD6BBAyRryheYva8E25aky8=
+github.com/omegaup/githttp/v2 v2.4.13/go.mod h1:CS+l7OI46d3JAnPEzqg6dJ5dYbIDuIxxb/mHDXNNFTQ=
 github.com/omegaup/go-base/logging/log15 v0.0.0-20211215145412-f1de9d5c6aee h1:pTkTShwDP49l9AAOXGCiyKb7kfwORzs1U4x4ocDCycU=
 github.com/omegaup/go-base/logging/log15 v0.0.0-20211215145412-f1de9d5c6aee/go.mod h1:CdmQRQaKuNhQU7HwaLgalgAUeXwnEoCEhhXN7Ipcyvw=
 github.com/omegaup/go-base/logging/log15/v3 v3.3.6 h1:smpRGjp923p+kf9tN+769ikOGY4DeriVJLX+u9o8hts=
@@ -242,8 +242,8 @@ github.com/omegaup/go-base/tracing/newrelic/v3 v3.3.6 h1:wQ4hdc+H2wH2whIvn2/DDhh
 github.com/omegaup/go-base/tracing/newrelic/v3 v3.3.6/go.mod h1:k8vryoCY2vbnSuC/lkuXQCts6o+0PxSvSS15f0dXoEw=
 github.com/omegaup/go-base/v3 v3.3.6 h1:3FybK1RM4rFPQQpMIFbO/cz3/EA9TWhOuzLhoxvG2w4=
 github.com/omegaup/go-base/v3 v3.3.6/go.mod h1:+N7tcCbx3AUEEwmUpsAzJktPCviwL57M8BTJ5m8GX9w=
-github.com/omegaup/quark v1.9.32 h1:vhLBdHGPMDnlwVJORbbmjJNHkB3170F4AqFKu4j1r9g=
-github.com/omegaup/quark v1.9.32/go.mod h1:9kPKAGE8r91HYNpxv2t3fUh4UEflLYGKCRxZqC8SbYI=
+github.com/omegaup/quark v1.9.33 h1:w7RUdUjc0vUhmN0+c6sXiCXSv3fRShVSpKBpr3vIToU=
+github.com/omegaup/quark v1.9.33/go.mod h1:kcOkBMBTHgz3/M3vxIAmpuD78Sbg6rLyB7Hui8CPGc8=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

handler.go

@@ -209,6 +209,7 @@ type gitProtocol struct {
 	hardOverallWallTimeLimit    base.Duration
 	interactiveSettingsCompiler InteractiveSettingsCompiler
 	log                         logging.Logger
+	lockfileManager             *githttp.LockfileManager
 }
 
 // GitProtocolOpts contains all the possible options to initialize the git protocol.
@@ -218,6 +219,7 @@ type GitProtocolOpts struct {
 	AllowDirectPushToMaster     bool
 	HardOverallWallTimeLimit    base.Duration
 	InteractiveSettingsCompiler InteractiveSettingsCompiler
+	LockfileManager             *githttp.LockfileManager
 }
 
 // NewGitProtocol creates a new GitProtocol with the provided authorization
@@ -228,6 +230,7 @@ func NewGitProtocol(opts GitProtocolOpts) *githttp.GitProtocol {
 		hardOverallWallTimeLimit:    opts.HardOverallWallTimeLimit,
 		interactiveSettingsCompiler: opts.InteractiveSettingsCompiler,
 		log:                         opts.Log,
+		lockfileManager:             opts.LockfileManager,
 	}
 	opts.AllowNonFastForward = true
 	opts.UpdateCallback = protocol.validateUpdate
@@ -1736,6 +1739,7 @@ func (p *gitProtocol) preprocessMaster(
 	spliceCommitSegment := txn.StartSegment("SpliceCommit")
 	newCommands, err := githttp.SpliceCommit(
 		originalRepo,
+		p.lockfileManager,
 		originalCommit,
 		masterCommit,
 		requestContext.UpdatedFiles,
@@ -1783,11 +1787,12 @@ func (p *gitProtocol) preprocess(
 
 // GitHandlerOpts contains all the possible options to initialize the git Server.
 type GitHandlerOpts struct {
-	RootPath string
-	Protocol *githttp.GitProtocol
-	Metrics  base.Metrics
-	Log      logging.Logger
-	Tracing  tracing.Provider
+	RootPath        string
+	Protocol        *githttp.GitProtocol
+	Metrics         base.Metrics
+	Log             logging.Logger
+	LockfileManager *githttp.LockfileManager
+	Tracing         tracing.Provider
 }
 
 // NewGitHandler is the HTTP handler for the omegaUp git server.
@@ -1803,8 +1808,9 @@ func NewGitHandler(opts GitHandlerOpts) http.Handler {
 		ContextCallback: func(ctx context.Context) context.Context {
 			return request.NewContext(ctx, opts.Metrics)
 		},
-		Log:     opts.Log,
-		Tracing: opts.Tracing,
+		Log:             opts.Log,
+		LockfileManager: opts.LockfileManager,
+		Tracing:         opts.Tracing,
 	})
 }