Experimental renderer

.gitignore

@@ -1,3 +1,8 @@
 .vscode
 data
 go_tile
+test.osm.pbf
+osm-tileserver
+mock_data/
+*.out
+*.test

go.mod

@@ -1,3 +1,18 @@
 module github.com/nielsole/go_tile
 
 go 1.20
+
+require github.com/paulmach/osm v0.7.1
+
+require (
+	git.sr.ht/~sbinet/gg v0.5.0 // indirect
+	github.com/campoy/embedmd v1.0.0 // indirect
+	github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 // indirect
+	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/paulmach/orb v0.1.3 // indirect
+	github.com/paulmach/protoscan v0.2.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/image v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+)

go.sum

@@ -0,0 +1,60 @@
+git.sr.ht/~sbinet/gg v0.5.0 h1:6V43j30HM623V329xA9Ntq+WJrMjDxRjuAB1LFWF5m8=
+git.sr.ht/~sbinet/gg v0.5.0/go.mod h1:G2C0eRESqlKhS7ErsNey6HHrqU1PwsnCQlekFi9Q2Oo=
+github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
+github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo=
+github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
+github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
+github.com/paulmach/osm v0.7.1 h1:dc84gLa4S/zCCqpBxb6jXTkN5dCI7VK7edt/tZTFG50=
+github.com/paulmach/osm v0.7.1/go.mod h1:v0vZa0rKnCsO8ovx0Z+hR9BWVD+vO4ogLOXcV18/0yk=
+github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
+golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

main.go

@@ -1,17 +1,22 @@
 package main
 
 import (
+	"context"
 	"encoding/binary"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"regexp"
-	"strconv"
+	"os/signal"
+	"syscall"
 	"time"
+
+	"github.com/nielsole/go_tile/renderer"
+	"github.com/nielsole/go_tile/utils"
 )
 
 /*
@@ -107,32 +112,8 @@ func writeTileResponse(writer http.ResponseWriter, req *http.Request, metatile_p
 	return nil
 }
 
-func parsePath(path string) (z, x, y uint32, err error) {
-	matcher := regexp.MustCompile(`^/tile/([0-9]+)/([0-9]+)/([0-9]+).png$`)
-	matches := matcher.FindStringSubmatch(path)
-	if len(matches) != 4 {
-		return 0, 0, 0, errors.New("could not match path")
-	}
-	zInt, err := strconv.Atoi(matches[1])
-	if err != nil {
-		return
-	}
-	xInt, err := strconv.Atoi(matches[2])
-	if err != nil {
-		return
-	}
-	yInt, err := strconv.Atoi(matches[3])
-	if err != nil {
-		return
-	}
-	z = uint32(zInt)
-	x = uint32(xInt)
-	y = uint32(yInt)
-	return
-}
-
 func handleRequest(resp http.ResponseWriter, req *http.Request, data_dir, map_name, renderd_socket string, renderd_timeout time.Duration, tile_expiration time.Duration) {
-	z, x, y, err := parsePath(req.URL.Path)
+	z, x, y, err := utils.ParsePath(req.URL.Path)
 	if err != nil {
 		resp.WriteHeader(http.StatusBadRequest)
 		resp.Write([]byte(err.Error()))
@@ -190,14 +171,14 @@ func main() {
 	http_listen_host := flag.String("host", "0.0.0.0", "HTTP Listening host")
 	http_listen_port := flag.Int("port", 8080, "HTTP Listening port")
 	https_listen_port := flag.Int("tls_port", 8443, "HTTPS Listening port. This listener is only enabled if both tls cert and key are set.")
-	data_dir := flag.String("data", "./data", "Path to directory containing tiles")
+	//data_dir := flag.String("data", "./data", "Path to directory containing tiles")
 	static_dir := flag.String("static", "./static/", "Path to static file directory")
 	renderd_socket := flag.String("socket", "", "Unix domain socket path or hostname:port for contacting renderd. Rendering disabled by default.")
 	renderd_timeout_duration := flag.Duration("renderd-timeout", (time.Duration(60) * time.Second), "Timeout duration after which renderd returns an error to the client (I.E. '30s' for thirty seconds). Set negative to disable")
 	map_name := flag.String("map", "ajt", "Name of map. This value is also used to determine the metatile subdirectory")
 	tls_cert_path := flag.String("tls_cert_path", "", "Path to TLS certificate")
 	tls_key_path := flag.String("tls_key_path", "", "Path to TLS key")
-	tile_expiration_duration := flag.Duration("tile_expiration", 0, "Duration after which tiles are considered stale (I.E. '168h' for one week). Tile expiration disabled by default")
+	//tile_expiration_duration := flag.Duration("tile_expiration", 0, "Duration after which tiles are considered stale (I.E. '168h' for one week). Tile expiration disabled by default")
 	verbose := flag.Bool("verbose", false, "Output debug log messages")
 
 	flag.Parse()
@@ -226,6 +207,29 @@ func main() {
 		logInfof("Rendering is disabled")
 	}
 
+	// Create a temp file.
+	var err error
+	tempFile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		fmt.Println("Cannot create temp file:", err)
+		os.Exit(1)
+	}
+
+	data, err := renderer.LoadData("/home/nokadmin/projects/go_tile/mock_data/test.osm.pbf", 15, tempFile)
+	if err != nil {
+		logFatalf("There was an error loading data: %v", err)
+	}
+	tempFileName := tempFile.Name()
+	tempFile.Close()
+
+	// Memory-map the file
+	mmapData, mmapFile, err := renderer.Mmap(tempFileName)
+	if err != nil {
+		logFatalf("There was an error memory-mapping temp file: %v", err)
+	}
+	defer syscall.Munmap(*mmapData)
+	defer mmapFile.Close()
+
 	// HTTP request multiplexer
 	httpServeMux := http.NewServeMux()
 
@@ -238,7 +242,8 @@ func main() {
 			http.Error(w, "Only GET requests allowed", http.StatusMethodNotAllowed)
 			return
 		}
-		handleRequest(w, r, *data_dir, *map_name, *renderd_socket, *renderd_timeout_duration, *tile_expiration_duration)
+		renderer.HandleRenderRequest(w, r, *renderd_timeout_duration, data, 15, mmapData)
+		//handleRequest(w, r, *data_dir, *map_name, *renderd_socket, *renderd_timeout_duration, *tile_expiration_duration)
 	})
 
 	// Static HTTP request handler
@@ -249,41 +254,73 @@ func main() {
 		Handler: httpServeMux,
 	}
 
-	// HTTPS listener
-	if len(*tls_cert_path) > 0 && len(*tls_key_path) > 0 {
-		go func() {
-			httpsAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", *http_listen_host, *https_listen_port))
-			if err != nil {
-				logFatalf("Failed to resolve TCP address: %v", err)
-			}
-			httpsListener, err := net.ListenTCP("tcp", httpsAddr)
-			if err != nil {
-				logFatalf("Failed to start TCP listener: %v", err)
-			} else {
-				logInfof("Started HTTPS listener on %s\n", httpsAddr)
-			}
-			err = httpServer.ServeTLS(httpsListener, *tls_cert_path, *tls_key_path)
-			if err != nil {
-				logFatalf("Failed to start HTTPS server: %v", err)
-			}
-		}()
-	} else {
-		logInfof("TLS is disabled")
-	}
+	go func() {
 
-	// HTTP listener
-	httpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", *http_listen_host, *http_listen_port))
-	if err != nil {
-		logFatalf("Failed to resolve TCP address: %v", err)
-	}
-	httpListener, err := net.ListenTCP("tcp", httpAddr)
-	if err != nil {
-		logFatalf("Failed to start TCP listener: %v", err)
-	} else {
-		logInfof("Started HTTP listener on %s\n", httpAddr)
-	}
-	err = httpServer.Serve(httpListener)
-	if err != nil {
-		logFatalf("Failed to start HTTP server: %v", err)
+		// HTTPS listener
+		if len(*tls_cert_path) > 0 && len(*tls_key_path) > 0 {
+			go func() {
+				httpsAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", *http_listen_host, *https_listen_port))
+				if err != nil {
+					logFatalf("Failed to resolve TCP address: %v", err)
+				}
+				httpsListener, err := net.ListenTCP("tcp", httpsAddr)
+				if err != nil {
+					logFatalf("Failed to start TCP listener: %v", err)
+				} else {
+					logInfof("Started HTTPS listener on %s\n", httpsAddr)
+				}
+				err = httpServer.ServeTLS(httpsListener, *tls_cert_path, *tls_key_path)
+				if err != nil && err != http.ErrServerClosed {
+					logFatalf("Failed to start HTTPS server: %v", err)
+				}
+			}()
+		} else {
+			logInfof("TLS is disabled")
+		}
+
+		// HTTP listener
+		httpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", *http_listen_host, *http_listen_port))
+		if err != nil {
+			logFatalf("Failed to resolve TCP address: %v", err)
+		}
+		httpListener, err := net.ListenTCP("tcp", httpAddr)
+		if err != nil {
+			logFatalf("Failed to start TCP listener: %v", err)
+		} else {
+			logInfof("Started HTTP listener on %s\n", httpAddr)
+		}
+		err = httpServer.Serve(httpListener)
+		if err != nil && err != http.ErrServerClosed {
+			logFatalf("Failed to start HTTP server: %v", err)
+		}
+	}()
+	// Setup signal capturing.
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
+
+	// Waiting for SIGINT (Ctrl+C)
+	select {
+	case <-stop:
+		fmt.Println("\nShutting down the server...")
+
+		// Create a deadline for the shutdown process.
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+
+		// Start shutdown.
+		if err := httpServer.Shutdown(ctx); err != nil {
+			fmt.Println("Error during server shutdown:", err)
+		}
+
+		// Cleanup the temp file.
+		if err := os.Remove(tempFile.Name()); err != nil {
+			fmt.Println("Failed to remove temp file:", err)
+		} else {
+			fmt.Println("Temp file removed.")
+		}
+
+		// Additional cleanup code here...
+
+		fmt.Println("Server gracefully stopped.")
 	}
 }

main_test.go

@@ -6,17 +6,9 @@ import (
 	"net/http/httptest"
 	"testing"
 	"time"
-)
 
-func TestParse(t *testing.T) {
-	z, x, y, err := parsePath("/tile/4/3/2.png")
-	if err != nil {
-		t.Error(err)
-	}
-	if z != 4 || x != 3 || y != 2 {
-		t.Fail()
-	}
-}
+	"github.com/nielsole/go_tile/utils"
+)
 
 func TestParseError(t *testing.T) {
 	invalid_paths := []string{
@@ -32,7 +24,7 @@ func TestParseError(t *testing.T) {
 		"/tile/abc/3/2.png",
 	}
 	for _, path := range invalid_paths {
-		_, _, _, err := parsePath(path)
+		_, _, _, err := utils.ParsePath(path)
 		if err == nil {
 			t.Errorf("expected error for path %s", path)
 		}
@@ -52,7 +44,7 @@ func BenchmarkPngRead(b *testing.B) {
 
 func BenchmarkParsePath(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		_, _, _, err := parsePath("/tile/4/3/2.png")
+		_, _, _, err := utils.ParsePath("/tile/4/3/2.png")
 		if err != nil {
 			b.Error(err)
 		}

renderer/.gitignore

@@ -0,0 +1,3 @@
+cpu.out
+*.svg
+*.test