connect: support Tarantool tuple format

Support format for Tarantool tuples. This support only works for
Tarantool versions >= 3.2.

Closes #832

@TarantoolBot document
Title: Support Tarantool tuple format

This patch adds support for formatting Tarantool tuples. Installed
Tarantool version must be >= 3.2. Otherwise, there will be no change.

Examples:
```
> f = box.tuple.format.new({{'id', 'number'}, {'name', 'string'}})
---
...

> t = box.tuple.new({1, 'Flint', true}, {format = f})
---
...

> \xt
> t
+----+-------+------+
| id | name  | col1 |
+----+-------+------+
| 1  | Flint | true |
+----+-------+------+
> box.space.customers:select()
+----+-----------+-----+
| id | name      | age |
+----+-----------+-----+
| 1  | Elizabeth | 12  |
+----+-----------+-----+
| 2  | Mary      | 46  |
+----+-----------+-----+
> \xT
> t
+------+-------+
| id   | 1     |
+------+-------+
| name | Flint |
+------+-------+
| col1 | true  |
+------+-------+
> box.space.customers:select()
+------+-----------+------+
| id   | 1         | 2    |
+------+-----------+------+
| name | Elizabeth | Mary |
+------+-----------+------+
| age  | 12        | 46   |
+------+-----------+------+
```

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - `tt log`: a module for viewing instances logs. Supported options:
   * `--lines` number of lines to print.
   * `--follow` print appended data as log files grow.
+- `tt connect`: support format for Tarantool tuples for Tarantool
+  versions >= 3.2.
 
 ### Fixed
 

cli/codegen/generate_code.go

@@ -27,7 +27,6 @@ var luaCodeFiles = []generateLuaCodeOpts{
 		PackageName: "connect",
 		FileName:    "cli/connect/lua_code_gen.go",
 		VariablesMap: map[string]string{
-			"consoleEvalFuncBody":    "cli/connect/lua/console_eval_func_body.lua",
 			"evalFuncBody":           "cli/connect/lua/eval_func_body.lua",
 			"getSuggestionsFuncBody": "cli/connect/lua/get_suggestions_func_body.lua",
 		},

cli/connect/connect.go

@@ -97,23 +97,24 @@ func Eval(connectCtx ConnectCtx, connOpts connector.ConnectOpts, args []string)
 	}
 	defer conn.Close()
 
-	var eval string
-	evalArgs := []interface{}{command}
+	evalArgs := []interface{}{command, connectCtx.Language == SQLLanguage}
 	if connectCtx.Language != DefaultLanguage {
 		// Change a language.
 		if err := ChangeLanguage(conn, connectCtx.Language); err != nil {
 			return nil, fmt.Errorf("unable to change a language: %s", err)
 		}
-		eval = consoleEvalFuncBody
+		evalArgs = append(evalArgs, false)
 	} else {
-		eval = evalFuncBody
+		needMetaInfo := connectCtx.Format == formatter.TableFormat ||
+			connectCtx.Format == formatter.TTableFormat
+		evalArgs = append(evalArgs, needMetaInfo)
 		for i := range args {
 			evalArgs = append(evalArgs, args[i])
 		}
 	}
 
 	// Execution of the command.
-	response, err := conn.Eval(eval, evalArgs, connector.RequestOpts{})
+	response, err := conn.Eval(evalFuncBody, evalArgs, connector.RequestOpts{})
 	if err != nil {
 		return nil, err
 	}

cli/connect/console.go

@@ -218,7 +218,10 @@ func getExecutor(console *Console) func(string) {
 		}
 
 		var results []string
-		args := []interface{}{console.input}
+		needMetaInfo := console.format == formatter.TableFormat ||
+			console.format == formatter.TTableFormat
+		args := []interface{}{console.input, console.language == SQLLanguage,
+			needMetaInfo}
 		opts := connector.RequestOpts{
 			PushCallback: func(pushedData interface{}) {
 				encodedData, err := yaml.Marshal(pushedData)
@@ -233,7 +236,7 @@ func getExecutor(console *Console) func(string) {
 		}
 
 		var data string
-		if _, err := console.conn.Eval(consoleEvalFuncBody, args, opts); err != nil {
+		if _, err := console.conn.Eval(evalFuncBody, args, opts); err != nil {
 			if err == io.EOF {
 				// We need to call 'console.Close()' here because in some cases (e.g 'os.exit()')
 				// it won't be called from 'defer console.Close' in 'connect.runConsole()'.

cli/connect/language.go

@@ -60,7 +60,7 @@ func ChangeLanguage(evaler connector.Evaler, lang Language) error {
 	}
 
 	languageCmd := setLanguagePrefix + " " + lang.String()
-	response, err := evaler.Eval(consoleEvalFuncBody,
+	response, err := evaler.Eval(evalFuncBody,
 		[]interface{}{languageCmd},
 		connector.RequestOpts{},
 	)

cli/connect/language_test.go

@@ -2,6 +2,7 @@ package connect_test
 
 import (
 	"errors"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -82,7 +83,11 @@ func (evaler *inputEvaler) Eval(fun string,
 }
 
 func TestChangeLanguage_requestInputs(t *testing.T) {
-	expectedFun := "return require('console').eval(...)\n"
+	rawFun, err := os.ReadFile("./lua/eval_func_body.lua")
+	if err != nil {
+		t.Fatal("Failed to read lua file:", err)
+	}
+	expectedFun := string(rawFun)
 	expectedOpts := connector.RequestOpts{}
 	cases := []struct {
 		lang Language

cli/connect/lua/eval_func_body.lua

@@ -1,6 +1,18 @@
 local yaml = require('yaml')
+yaml.cfg{ encode_use_tostring = true }
 local args = {...}
 local cmd = table.remove(args, 1)
+local is_sql_language = table.remove(args, 1)
+local need_metainfo = table.remove(args, 1)
+
+local function is_command(line)
+    return line:sub(1, 1) == '\\'
+end
+
+if is_command(cmd) or is_sql_language == true then
+    return require('console').eval(cmd)
+end
+
 local fun, errmsg = loadstring("return "..cmd)
 if not fun then
     fun, errmsg = loadstring(cmd)
@@ -13,9 +25,26 @@ local function table_pack(...)
     return {n = select('#', ...), ...}
 end
 
+local function format_equal(f1, f2)
+    if #f1 ~= #f2 then
+        return false
+    end
+
+    for i, field in ipairs(f1) do
+        if field.name ~= f2[i].name or field.type ~= f2[i].type then
+            return false
+        end
+    end
+    return true
+end
+
 local ret = table_pack(pcall(fun, unpack(args)))
 if not ret[1] then
-    return yaml.encode({box.NULL})
+    local err = unpack(ret, 2, ret.n)
+    if err == nil then
+        err = box.NULL
+    end
+    return yaml.encode({{error = err}})
 end
 if ret.n == 1 then
     return "---\n...\n"
@@ -25,4 +54,49 @@ for i=2,ret.n do
         ret[i] = box.NULL
     end
 end
+if not need_metainfo then
+    return yaml.encode({unpack(ret, 2, ret.n)})
+end
+
+for i=2,ret.n do
+    if box.tuple.is ~= nil and box.tuple.is(ret[i]) and ret[i].format then
+        local ret_with_format = {
+            metadata = ret[i]:format(),
+            rows = { ret[i] },
+        }
+        if #ret_with_format.metadata > 0 then
+            ret[i] = ret_with_format
+        end
+    end
+
+    if type(ret[i]) == 'table' and #ret[i] > 0 then
+        local ret_with_format = {
+            metadata = {},
+            rows = ret[i],
+        }
+
+        local same_format = true
+        for tuple_ind, tuple in ipairs(ret[i]) do
+            local is_tuple = box.tuple.is ~= nil and box.tuple.is(tuple) and tuple.format
+
+            if tuple_ind == 1 then
+                if not is_tuple then
+                    same_format = false
+                    break
+                end
+
+                ret_with_format.metadata = tuple:format()
+            elseif not is_tuple or
+                    not format_equal(tuple:format(), ret_with_format.metadata) then
+                same_format = false
+                break
+            end
+        end
+
+        if #ret_with_format.metadata > 0 and same_format then
+            ret[i] = ret_with_format
+        end
+    end
+end
+
 return yaml.encode({unpack(ret, 2, ret.n)})

cli/formatter/table.go

@@ -10,12 +10,14 @@ import (
 
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
+	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 )
 
-// decodeYamlArr decodes yaml string as []any content.
-func decodeYamlArr(input string) ([]any, error) {
-	var decoded []any
+// lazyDecodeYaml decodes yaml string as []lazyMessage content.
+// Each array member needs to be decoded later.
+func lazyDecodeYaml(input string) ([]lazyMessage, error) {
+	var decoded []lazyMessage
 	err := yaml.Unmarshal([]byte(input), &decoded)
 	if err != nil {
 		return nil, err
@@ -40,8 +42,7 @@ func castMapToUMap(src map[any]any) unorderedMap[any] {
 		}
 	}
 	sort.Slice(sortedKeys, func(i, j int) bool {
-		return fmt.Sprintf("%v", sortedKeys[i]) <
-			fmt.Sprintf("%v", sortedKeys[j])
+		return fmt.Sprint(sortedKeys[i]) < fmt.Sprint(sortedKeys[j])
 	})
 
 	dst := createUnorderedMap[any](len(convertedSrc))
@@ -55,19 +56,6 @@ func castMapToUMap(src map[any]any) unorderedMap[any] {
 // deepCastAnyMapToStringMap casts all map[any]any to map[string]any deeply.
 func deepCastAnyMapToStringMap(v any) interface{} {
 	switch x := v.(type) {
-	case unorderedMap[any]:
-		m := createUnorderedMap[string](x.len())
-		for _, k := range x.keys {
-			v2 := x.innerMap[k]
-			switch k2 := k.(type) {
-			case string:
-				m.insert(k2, deepCastAnyMapToStringMap(v2))
-			default:
-				m.insert(fmt.Sprint(k), deepCastAnyMapToStringMap(v2))
-			}
-		}
-		v = m
-
 	case []any:
 		for i, v2 := range x {
 			x[i] = deepCastAnyMapToStringMap(v2)
@@ -353,10 +341,10 @@ func isMapKeysEqual(x unorderedMap[any], y unorderedMap[any]) bool {
 	}
 
 	sort.Slice(keysX, func(i, j int) bool {
-		return fmt.Sprintf("%v", keysX[i]) < fmt.Sprintf("%v", keysX[j])
+		return fmt.Sprint(keysX[i]) < fmt.Sprint(keysX[j])
 	})
 	sort.Slice(keysY, func(i, j int) bool {
-		return fmt.Sprintf("%v", keysY[i]) < fmt.Sprintf("%v", keysY[j])
+		return fmt.Sprint(keysY[i]) < fmt.Sprint(keysY[j])
 	})
 
 	for k, v := range keysX {
@@ -451,10 +439,23 @@ type metadataRows struct {
 // remapMetadataRows creates maps from rows with a metainformation.
 func remapMetadataRows(meta metadataRows) []any {
 	var nodes []any
+	maxLen := 0
+	for _, row := range meta.Rows {
+		if len(row) > maxLen {
+			maxLen = len(row)
+		}
+	}
 	for _, row := range meta.Rows {
 		index := 1
 		mapped := createUnorderedMap[any](len(row))
-		for i, column := range row {
+		for i := 0; i < maxLen; i++ {
+			if i >= len(row) {
+				mapped.insert(index, "")
+				index++
+				continue
+			}
+
+			column := row[i]
 			if len(meta.Metadata) > i && meta.Metadata[i].Name != "" {
 				mapped.insert(meta.Metadata[i].Name, column)
 			} else {
@@ -467,27 +468,62 @@ func remapMetadataRows(meta metadataRows) []any {
 	return nodes
 }
 
+func insertCollectedFields(fields metadataRows, nodes []any) []any {
+	if len(fields.Metadata) > 0 && len(fields.Rows) > 0 {
+		return append(nodes, remapMetadataRows(fields)...)
+	}
+	return nodes
+}
+
 // makeTableOutput returns tables as string for table/ttable output formats.
 func makeTableOutput(input string, transpose bool, opts Opts) (string, error) {
 	// Handle empty input from remote console.
 	if input == "---\n- \n...\n" || input == "---\n-\n...\n" {
 		input = "--- ['']\n...\n"
 	}
 
-	var meta []metadataRows
 	var nodes []any
 
-	// First of all try to read it as tuples with metadata (SQL output format).
-	err := yaml.Unmarshal([]byte(input), &meta)
-	if err == nil && len(meta) > 0 && len(meta[0].Rows) > 0 && len(meta[0].Metadata) > 0 {
-		nodes = remapMetadataRows(meta[0])
-	} else {
-		// Failed. Try to read it as an array.
-		nodes, err = decodeYamlArr(input)
-		if err != nil {
-			return "", fmt.Errorf("not yaml array, cannot render tables: %s", err)
+	// We need to decode input lazy here. This is the case because we can get
+	// the input as an array, where some elements have metadata.
+	// So we need to decode input as an array first and then each element
+	// separately: as a metadataRows type or some other any value.
+	// If we decode everything as []any first, it would be problematic to
+	// convert any value to metadataRows type.
+	lazyNodes, err := lazyDecodeYaml(input)
+	if err != nil {
+		return "", fmt.Errorf("not yaml array, cannot render tables: %s", err)
+	}
+
+	var metaFields metadataRows
+
+	for _, lazyNode := range lazyNodes {
+		var meta metadataRows
+
+		// First of all, try to read it as tuples with metadata
+		// (SQL output format).
+		err = lazyNode.Unmarshal(&meta)
+		if err == nil && len(meta.Rows) > 0 && len(meta.Metadata) > 0 {
+			if slices.Equal(meta.Metadata, metaFields.Metadata) {
+				metaFields.Rows = append(metaFields.Rows, meta.Rows...)
+			} else {
+				nodes = insertCollectedFields(metaFields, nodes)
+				metaFields = meta
+			}
+		} else {
+			nodes = insertCollectedFields(metaFields, nodes)
+			metaFields = metadataRows{}
+
+			// Failed. Try to read it as an any.
+			var node any
+			err = lazyNode.Unmarshal(&node)
+			if err != nil {
+				return "", fmt.Errorf("not yaml any: %s", err)
+			}
+			nodes = append(nodes, node)
 		}
 	}
+	nodes = insertCollectedFields(metaFields, nodes)
 
 	if len(nodes) == 0 {
 		nodes = append(nodes, []any{""})

cli/formatter/yaml.go

@@ -0,0 +1,20 @@
+package formatter
+
+// lazyMessage is used for lazy Unmarshal.
+type lazyMessage struct {
+	unmarshal func(any) error
+}
+
+// UnmarshalYAML makes lazyMessage an instance of yaml.Unmarshaler.
+func (msg *lazyMessage) UnmarshalYAML(unmarshal func(any) error) error {
+	msg.unmarshal = unmarshal
+	return nil
+}
+
+// Unmarshal the lazyMessage.
+func (msg *lazyMessage) Unmarshal(v any) error {
+	if msg.unmarshal == nil {
+		return nil
+	}
+	return msg.unmarshal(v)
+}