Add progress bar for mcap convert (foxglove#973)

### Public-Facing Changes

This adds a progress bar to show percentage completion when running `mcap convert ...`

.github/workflows/ci.yml

@@ -347,7 +347,7 @@ jobs:
         with:
           go-version-file: go/go.work
       - name: install golangci-lint
-        run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.49.0
+        run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2
       - run: make lint
       - run: make test
       - name: Check library version

go/cli/mcap/cmd/convert.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/foxglove/go-rosbag"
+	"github.com/foxglove/mcap/go/cli/mcap/utils"
 	"github.com/foxglove/mcap/go/mcap"
 	"github.com/foxglove/mcap/go/ros"
 	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
@@ -107,7 +109,27 @@ var convertCmd = &cobra.Command{
 
 		switch filetype {
 		case FileTypeRos1:
-			err = ros.Bag2MCAP(bw, f, opts)
+			rosReader, err := rosbag.NewReader(f)
+			if err != nil {
+				die("failed to make new ROS reader: %s", err)
+			}
+
+			rosInfo, err := rosReader.Info()
+			if err != nil {
+				die("failed to get info from ROS file: %s", err)
+			}
+
+			messageCount := rosInfo.MessageCount
+			progressBar := utils.NewProgressBar(int64(messageCount))
+			_, err = f.Seek(0, io.SeekStart) // Info() moved the seeker to EOF. This brings it back to the start.
+			if err != nil {
+				die("failed to seek to start of file: %s", err)
+			}
+
+			err = ros.Bag2MCAP(bw, f, opts, func(data []byte) error {
+				progressBar.Add64(1)
+				return nil
+			})
 			if err != nil && !errors.Is(err, io.EOF) {
 				die("failed to convert file: %s", err)
 			}
@@ -118,13 +140,23 @@ var convertCmd = &cobra.Command{
 				die("failed to open sqlite3: %s", err)
 			}
 
+			var messageCount int
+			err = db.QueryRow("select count(*) from messages").Scan(&messageCount)
+			if err != nil {
+				die("failed to query db for message counts")
+			}
+			progressBar := utils.NewProgressBar(int64(messageCount))
+
 			amentPath := convertAmentPrefixPath
 			prefixPath := os.Getenv("AMENT_PREFIX_PATH")
 			if prefixPath != "" {
 				amentPath += string(os.PathListSeparator) + prefixPath
 			}
 			dirs := strings.FieldsFunc(amentPath, func(c rune) bool { return (c == os.PathListSeparator) })
-			err = ros.DB3ToMCAP(bw, db, opts, dirs)
+			err = ros.DB3ToMCAP(bw, db, opts, dirs, func(b []byte) error {
+				progressBar.Add(1)
+				return nil
+			})
 			if err != nil {
 				die("failed to convert file: %s", err)
 			}

go/cli/mcap/go.mod

@@ -1,19 +1,21 @@
 module github.com/foxglove/mcap/go/cli/mcap
 
-go 1.18
+go 1.20
 
 require (
 	cloud.google.com/go/storage v1.23.0
 	github.com/fatih/color v1.13.0
+	github.com/foxglove/go-rosbag v0.0.5
 	github.com/foxglove/mcap/go/mcap v0.4.0
 	github.com/foxglove/mcap/go/ros v0.0.0-20230114025807-456e6a6ca1be
 	github.com/klauspost/compress v1.16.7
 	github.com/mattn/go-sqlite3 v1.14.14
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pierrec/lz4/v4 v4.1.17
+	github.com/pierrec/lz4/v4 v4.1.18
+	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.12.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.4
 	google.golang.org/protobuf v1.28.0
 )
 
@@ -34,8 +36,9 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
@@ -49,7 +52,8 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
 	golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect
-	golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/api v0.86.0 // indirect

go/cli/mcap/utils/utils.go

@@ -10,10 +10,12 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/foxglove/mcap/go/mcap"
 	"github.com/olekukonko/tablewriter"
+	"github.com/schollz/progressbar/v3"
 )
 
 var (
@@ -169,3 +171,20 @@ func DefaultString(strings ...string) string {
 	}
 	return ""
 }
+
+// NewProgressBar returns an instance of progressbar.ProgresBar.
+// `max` is the denominator of the progress.
+func NewProgressBar(max int64) *progressbar.ProgressBar {
+	return progressbar.NewOptions64(
+		max,
+		progressbar.OptionThrottle(65*time.Millisecond),
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSetWidth(10),
+		progressbar.OptionOnCompletion(func() {
+			fmt.Fprint(os.Stderr, "\n")
+		}),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionSetRenderBlankState(true),
+	)
+
+}

go/go.work

@@ -1,4 +1,4 @@
-go 1.18
+go 1.20
 
 use (
 	./cli/mcap

go/go.work.sum

@@ -0,0 +1,30 @@
+cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
+github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/foxglove/mcap/go/mcap v0.0.0-20220328132551-ffb9c0b0ebdc/go.mod h1:gQrB8PzccHW69xedSZ0uVDQVgDd3h1qX+otbS6fjSkE=
+github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=

go/ros/bag2mcap.go

@@ -225,7 +225,7 @@ func channelIDForConnection(connID uint32) (uint16, error) {
 	return uint16(connID), nil
 }
 
-func Bag2MCAP(w io.Writer, r io.Reader, opts *mcap.WriterOptions) error {
+func Bag2MCAP(w io.Writer, r io.Reader, opts *mcap.WriterOptions, messageCallbacks ...func([]byte) error) error {
 	writer, err := mcap.NewWriter(w, opts)
 	if err != nil {
 		return err
@@ -315,6 +315,12 @@ func Bag2MCAP(w io.Writer, r io.Reader, opts *mcap.WriterOptions) error {
 				return err
 			}
 			seq++
+			for _, callback := range messageCallbacks {
+				err = callback(data)
+				if err != nil {
+					return err
+				}
+			}
 			return nil
 		},
 		true,

go/ros/ros2db3_to_mcap.go

@@ -258,11 +258,17 @@ func transformMessages(db *sql.DB, f func(*sql.Rows) error) error {
 	return nil
 }
 
-func DB3ToMCAP(w io.Writer, db *sql.DB, opts *mcap.WriterOptions, searchdirs []string) error {
+func DB3ToMCAP(w io.Writer,
+	db *sql.DB,
+	opts *mcap.WriterOptions,
+	searchdirs []string,
+	callbacks ...func([]byte) error,
+) error {
 	topics, err := getTopics(db)
 	if err != nil {
 		return err
 	}
+
 	types := make([]string, len(topics))
 	for i := range topics {
 		types[i] = topics[i].typ
@@ -338,6 +344,13 @@ func DB3ToMCAP(w io.Writer, db *sql.DB, opts *mcap.WriterOptions, searchdirs []s
 			return err
 		}
 		seq[topicID]++
+
+		for _, callback := range callbacks {
+			err = callback(messageData)
+			if err != nil {
+				return err
+			}
+		}
 		return nil
 	})
 	if err != nil {