From 5eda7f578d232c0a8151e18e679cfb64c249c2de Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 25 Jul 2024 08:41:00 +0200 Subject: [PATCH] refactor: break down json grammar parser in different files (#3004) * refactor: break down json grammar parser in different files Signed-off-by: Ettore Di Giacinto * fix: patch to `refactor_grammars` - propagate errors (#3006) propagate errors around Signed-off-by: Dave Lee --------- Signed-off-by: Ettore Di Giacinto Signed-off-by: Dave Lee Co-authored-by: Dave --- core/http/endpoints/openai/chat.go | 10 +- pkg/functions/bnf_rules.go | 47 ++++++ pkg/functions/function_structure.go | 25 +++ pkg/functions/functions.go | 17 ++ pkg/functions/functions_suite_test.go | 14 +- pkg/functions/grammar_json_schema.go | 179 +++++++--------------- pkg/functions/grammar_json_schema_test.go | 51 +++--- pkg/functions/json_mode.go | 28 ++++ 8 files changed, 218 insertions(+), 153 deletions(-) create mode 100644 pkg/functions/bnf_rules.go create mode 100644 pkg/functions/function_structure.go create mode 100644 pkg/functions/json_mode.go diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f63a991319d..c7afb7bf95e 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -226,9 +226,15 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup // Update input grammar jsStruct := funcs.ToJSONStructure(config.FunctionsConfig.FunctionNameKey, config.FunctionsConfig.FunctionNameKey) - config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) + g, err := jsStruct.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) + if err == nil { + config.Grammar = g + } case input.JSONFunctionGrammarObject != nil: - config.Grammar = input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) + g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) + if err == nil { + config.Grammar = g + } default: // Force picking one of the functions by the request if config.FunctionToCall() != "" { diff --git a/pkg/functions/bnf_rules.go b/pkg/functions/bnf_rules.go new file mode 100644 index 00000000000..13aa365420e --- /dev/null +++ b/pkg/functions/bnf_rules.go @@ -0,0 +1,47 @@ +package functions + +import "regexp" + +var ( + PRIMITIVE_RULES = map[string]string{ + "boolean": `("true" | "false") space`, + "number": `("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space`, + "integer": `("-"? ([0-9] | [1-9] [0-9]*)) space`, + "string": `"\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space`, + // TODO: we shouldn't forbid \" and \\ or all unicode and have this branch here, + // however, if we don't have it, the grammar will be ambiguous and + // empirically results are way worse. + "freestring": `( + [^\x00] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space`, + "null": `"null" space`, + } + + INVALID_RULE_CHARS_RE = regexp.MustCompile(`[^a-zA-Z0-9-]+`) + GRAMMAR_LITERAL_ESCAPE_RE = regexp.MustCompile(`[\r\n"]`) + GRAMMAR_LITERAL_ESCAPES = map[string]string{ + "\r": `\r`, + "\n": `\n`, + `"`: `\"`, + } +) + +const ( + SPACE_RULE = `" "?` + + arrayNewLines = `arr ::= + "[\n" ( + realvalue + (",\n" realvalue)* + )? "]"` + + array = `arr ::= + "[" ( + realvalue + ("," realvalue)* + )? "]"` +) diff --git a/pkg/functions/function_structure.go b/pkg/functions/function_structure.go new file mode 100644 index 00000000000..62cc68fa0c1 --- /dev/null +++ b/pkg/functions/function_structure.go @@ -0,0 +1,25 @@ +package functions + +import "encoding/json" + +type Item struct { + Type string `json:"type"` + Properties map[string]interface{} `json:"properties"` +} + +type JSONFunctionStructure struct { + OneOf []Item `json:"oneOf,omitempty"` + AnyOf []Item `json:"anyOf,omitempty"` + Defs map[string]interface{} `json:"$defs,omitempty"` +} + +func (j JSONFunctionStructure) Grammar(options ...func(*GrammarOption)) (string, error) { + grammarOpts := &GrammarOption{} + grammarOpts.Apply(options...) + + dat, err := json.Marshal(j) + if err != nil { + return "", err + } + return NewJSONSchemaConverter(grammarOpts.PropOrder).GrammarFromBytes(dat, options...) +} diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index 49e9fc93e85..2690b8ec4cd 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -18,6 +18,15 @@ type Function struct { } type Functions []Function +type FunctionName struct { + Const string `json:"const"` +} + +type Argument struct { + Type string `json:"type"` + Properties map[string]interface{} `json:"properties"` +} + type Tool struct { Type string `json:"type"` Function Function `json:"function,omitempty"` @@ -86,3 +95,11 @@ func (f Functions) Select(name string) Functions { return funcs } + +func jsonString(v interface{}) (string, error) { + b, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/pkg/functions/functions_suite_test.go b/pkg/functions/functions_suite_test.go index 8964b1c806a..59a90ab0e12 100644 --- a/pkg/functions/functions_suite_test.go +++ b/pkg/functions/functions_suite_test.go @@ -1,8 +1,10 @@ -package functions +package functions_test import ( "testing" + . "github.com/mudler/LocalAI/pkg/functions" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +13,13 @@ func TestGrammar(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Grammar test suite") } + +func createFunction(field1 string, field2 string, name string, properties map[string]interface{}) map[string]interface{} { + property := map[string]interface{}{} + property[field1] = FunctionName{Const: name} + property[field2] = Argument{ + Type: "object", + Properties: properties, + } + return property +} diff --git a/pkg/functions/grammar_json_schema.go b/pkg/functions/grammar_json_schema.go index 7356d01d54b..5ffc0ba5e85 100644 --- a/pkg/functions/grammar_json_schema.go +++ b/pkg/functions/grammar_json_schema.go @@ -5,70 +5,12 @@ package functions import ( "encoding/json" "fmt" - "regexp" "sort" "strings" "github.com/mudler/LocalAI/pkg/utils" ) -const ( - JSONBNF = `root ::= object -value ::= object | array | string | number | ("true" | "false" | "null") ws - -object ::= - "{" ws ( - string ":" ws value - ("," ws string ":" ws value)* - )? "}" ws - -array ::= - "[" ws ( - value - ("," ws value)* - )? "]" ws - -string ::= - "\"" ( - [^"\\] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes - )* "\"" ws - -number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws - -ws ::= ([ \t\n] ws)?` -) - -var ( - SPACE_RULE = `" "?` - - PRIMITIVE_RULES = map[string]string{ - "boolean": `("true" | "false") space`, - "number": `("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space`, - "integer": `("-"? ([0-9] | [1-9] [0-9]*)) space`, - "string": `"\"" ( - [^"\\] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\"" space`, - // TODO: we shouldn't forbid \" and \\ or all unicode and have this branch here, - // however, if we don't have it, the grammar will be ambiguous and - // empirically results are way worse. - "freestring": `( - [^\x00] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* space`, - "null": `"null" space`, - } - - INVALID_RULE_CHARS_RE = regexp.MustCompile(`[^a-zA-Z0-9-]+`) - GRAMMAR_LITERAL_ESCAPE_RE = regexp.MustCompile(`[\r\n"]`) - GRAMMAR_LITERAL_ESCAPES = map[string]string{ - "\r": `\r`, - "\n": `\n`, - `"`: `\"`, - } -) - type JSONSchemaConverter struct { propOrder map[string]int rules map[string]string @@ -90,11 +32,15 @@ func NewJSONSchemaConverter(propOrder string) *JSONSchemaConverter { } } -func (sc *JSONSchemaConverter) formatLiteral(literal interface{}) string { - escaped := GRAMMAR_LITERAL_ESCAPE_RE.ReplaceAllStringFunc(jsonString(literal), func(match string) string { +func (sc *JSONSchemaConverter) formatLiteral(literal interface{}) (string, error) { + jLiteral, err := jsonString(literal) + if err != nil { + return "", err + } + escaped := GRAMMAR_LITERAL_ESCAPE_RE.ReplaceAllStringFunc(jLiteral, func(match string) string { return GRAMMAR_LITERAL_ESCAPES[match] }) - return fmt.Sprintf(`"%s"`, escaped) + return fmt.Sprintf(`"%s"`, escaped), nil } func (sc *JSONSchemaConverter) addRule(name, rule string) string { @@ -114,18 +60,6 @@ func (sc *JSONSchemaConverter) addRule(name, rule string) string { return key } -const arrayNewLines = `arr ::= - "[\n" ( - realvalue - (",\n" realvalue)* - )? "]"` - -const array = `arr ::= - "[" ( - realvalue - ("," realvalue)* - )? "]"` - func (sc *JSONSchemaConverter) finalizeGrammar(options ...func(*GrammarOption)) string { grammarOpts := &GrammarOption{} @@ -210,7 +144,7 @@ func (sc *JSONSchemaConverter) finalizeGrammar(options ...func(*GrammarOption)) return strings.Join(lines, "\n") } -func (sc *JSONSchemaConverter) visit(schema map[string]interface{}, name string, rootSchema map[string]interface{}) string { +func (sc *JSONSchemaConverter) visit(schema map[string]interface{}, name string, rootSchema map[string]interface{}) (string, error) { st, existType := schema["type"] var schemaType string if existType { @@ -229,31 +163,44 @@ func (sc *JSONSchemaConverter) visit(schema map[string]interface{}, name string, if oneOfExists { for i, altSchema := range oneOfSchemas { - alternative := sc.visit(altSchema.(map[string]interface{}), fmt.Sprintf("%s-%d", ruleName, i), rootSchema) + alternative, err := sc.visit(altSchema.(map[string]interface{}), fmt.Sprintf("%s-%d", ruleName, i), rootSchema) + if err != nil { + return "", err + } alternatives = append(alternatives, alternative) } } else if anyOfExists { for i, altSchema := range anyOfSchemas { - alternative := sc.visit(altSchema.(map[string]interface{}), fmt.Sprintf("%s-%d", ruleName, i), rootSchema) + alternative, err := sc.visit(altSchema.(map[string]interface{}), fmt.Sprintf("%s-%d", ruleName, i), rootSchema) + if err != nil { + return "", err + } alternatives = append(alternatives, alternative) } } rule := strings.Join(alternatives, " | ") - return sc.addRule(ruleName, rule) + return sc.addRule(ruleName, rule), nil } else if ref, exists := schema["$ref"].(string); exists { referencedSchema := sc.resolveReference(ref, rootSchema) return sc.visit(referencedSchema, name, rootSchema) } else if constVal, exists := schema["const"]; exists { - return sc.addRule(ruleName, sc.formatLiteral(constVal)) + literal, err := sc.formatLiteral((constVal)) + if err != nil { + return "", err + } + return sc.addRule(ruleName, literal), nil } else if enumVals, exists := schema["enum"].([]interface{}); exists { var enumRules []string for _, enumVal := range enumVals { - enumRule := sc.formatLiteral(enumVal) + enumRule, err := sc.formatLiteral(enumVal) + if err != nil { + return "", err + } enumRules = append(enumRules, enumRule) } rule := strings.Join(enumRules, " | ") - return sc.addRule(ruleName, rule) + return sc.addRule(ruleName, rule), nil } else if properties, exists := schema["properties"].(map[string]interface{}); schemaType == "object" && exists { propOrder := sc.propOrder var propPairs []struct { @@ -283,21 +230,30 @@ func (sc *JSONSchemaConverter) visit(schema map[string]interface{}, name string, for i, propPair := range propPairs { propName := propPair.propName propSchema := propPair.propSchema - propRuleName := sc.visit(propSchema, fmt.Sprintf("%s-%s", ruleName, propName), rootSchema) - + propRuleName, err := sc.visit(propSchema, fmt.Sprintf("%s-%s", ruleName, propName), rootSchema) + if err != nil { + return "", err + } + lPropName, err := sc.formatLiteral(propName) + if err != nil { + return "", err + } if i > 0 { rule.WriteString(` "," space`) } - rule.WriteString(fmt.Sprintf(` %s space ":" space %s`, sc.formatLiteral(propName), propRuleName)) + rule.WriteString(fmt.Sprintf(` %s space ":" space %s`, lPropName, propRuleName)) } rule.WriteString(` "}" space`) - return sc.addRule(ruleName, rule.String()) + return sc.addRule(ruleName, rule.String()), nil } else if items, exists := schema["items"].(map[string]interface{}); schemaType == "array" && exists { - itemRuleName := sc.visit(items, fmt.Sprintf("%s-item", ruleName), rootSchema) + itemRuleName, err := sc.visit(items, fmt.Sprintf("%s-item", ruleName), rootSchema) + if err != nil { + return "", err + } rule := fmt.Sprintf(`"[" space (%s ("," space %s)*)? "]" space`, itemRuleName, itemRuleName) - return sc.addRule(ruleName, rule) + return sc.addRule(ruleName, rule), nil } else { primitiveRule, exists := PRIMITIVE_RULES[schemaType] if !exists { @@ -306,7 +262,7 @@ func (sc *JSONSchemaConverter) visit(schema map[string]interface{}, name string, if ruleName == "root" { schemaType = "root" } - return sc.addRule(schemaType, primitiveRule) + return sc.addRule(schemaType, primitiveRule), nil } } func (sc *JSONSchemaConverter) resolveReference(ref string, rootSchema map[string]interface{}) map[string]interface{} { @@ -332,47 +288,20 @@ func (sc *JSONSchemaConverter) resolveReference(ref string, rootSchema map[strin return def } -func (sc *JSONSchemaConverter) Grammar(schema map[string]interface{}, options ...func(*GrammarOption)) string { +func (sc *JSONSchemaConverter) Grammar(schema map[string]interface{}, options ...func(*GrammarOption)) (string, error) { sc.addRule("freestring", PRIMITIVE_RULES["freestring"]) - sc.visit(schema, "", schema) - return sc.finalizeGrammar(options...) + _, err := sc.visit(schema, "", schema) + if err != nil { + return "", err + } + return sc.finalizeGrammar(options...), nil } -func (sc *JSONSchemaConverter) GrammarFromBytes(b []byte, options ...func(*GrammarOption)) string { +func (sc *JSONSchemaConverter) GrammarFromBytes(b []byte, options ...func(*GrammarOption)) (string, error) { var schema map[string]interface{} - _ = json.Unmarshal(b, &schema) + err := json.Unmarshal(b, &schema) + if err != nil { + return "", err + } return sc.Grammar(schema, options...) } - -func jsonString(v interface{}) string { - b, _ := json.Marshal(v) - return string(b) -} - -type FunctionName struct { - Const string `json:"const"` -} - -type Argument struct { - Type string `json:"type"` - Properties map[string]interface{} `json:"properties"` -} - -type Item struct { - Type string `json:"type"` - Properties map[string]interface{} `json:"properties"` -} - -type JSONFunctionStructure struct { - OneOf []Item `json:"oneOf,omitempty"` - AnyOf []Item `json:"anyOf,omitempty"` - Defs map[string]interface{} `json:"$defs,omitempty"` -} - -func (j JSONFunctionStructure) Grammar(options ...func(*GrammarOption)) string { - grammarOpts := &GrammarOption{} - grammarOpts.Apply(options...) - - dat, _ := json.Marshal(j) - return NewJSONSchemaConverter(grammarOpts.PropOrder).GrammarFromBytes(dat, options...) -} diff --git a/pkg/functions/grammar_json_schema_test.go b/pkg/functions/grammar_json_schema_test.go index bf52bd8d49f..56c5fe1e611 100644 --- a/pkg/functions/grammar_json_schema_test.go +++ b/pkg/functions/grammar_json_schema_test.go @@ -3,22 +3,11 @@ package functions_test import ( "strings" - "github.com/mudler/LocalAI/pkg/functions" . "github.com/mudler/LocalAI/pkg/functions" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -func createFunction(field1 string, field2 string, name string, properties map[string]interface{}) map[string]interface{} { - property := map[string]interface{}{} - property[field1] = FunctionName{Const: name} - property[field2] = Argument{ - Type: "object", - Properties: properties, - } - return property -} - var testFunctions = []Item{ { Type: "object", @@ -245,7 +234,8 @@ root-1-name ::= "\"search\""` var _ = Describe("JSON schema grammar tests", func() { Context("JSON", func() { It("generates a valid grammar from JSON schema", func() { - grammar := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput1)) + grammar, err := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput1)) + Expect(err).To(BeNil()) results := strings.Split(inputResult1, "\n") for _, r := range results { if r != "" { @@ -255,7 +245,8 @@ var _ = Describe("JSON schema grammar tests", func() { Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n")))) }) It("generates a valid grammar from JSON schema", func() { - grammar := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput2)) + grammar, err := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput2)) + Expect(err).To(BeNil()) results := strings.Split(inputResult3, "\n") for _, r := range results { if r != "" { @@ -269,7 +260,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctions} - grammar := structuredGrammar.Grammar() + grammar, err := structuredGrammar.Grammar() + Expect(err).To(BeNil()) results := strings.Split(inputResult1, "\n") for _, r := range results { if r != "" { @@ -283,7 +275,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctions} - grammar := structuredGrammar.Grammar(functions.EnableMaybeArray) + grammar, err := structuredGrammar.Grammar(EnableMaybeArray) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ inputResult2, @@ -301,7 +294,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.EnableMaybeArray) + grammar, err := structuredGrammar.Grammar(EnableMaybeArray) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ inputResult4, @@ -319,10 +313,11 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar( - functions.SetPrefix("suffix"), - functions.EnableMaybeArray, + grammar, err := structuredGrammar.Grammar( + SetPrefix("suffix"), + EnableMaybeArray, ) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`"suffix" arr | realvalue`), @@ -339,7 +334,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix")) + grammar, err := structuredGrammar.Grammar(SetPrefix("suffix")) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`"suffix" realvalue`), @@ -356,7 +352,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix"), functions.EnableMaybeString) + grammar, err := structuredGrammar.Grammar(SetPrefix("suffix"), EnableMaybeString) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`( "suffix" realvalue | mixedstring )`), @@ -373,7 +370,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix"), functions.EnableMaybeString, functions.EnableMaybeArray) + grammar, err := structuredGrammar.Grammar(SetPrefix("suffix"), EnableMaybeString, EnableMaybeArray) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`( "suffix" (arr | realvalue) | mixedstring )`), @@ -392,7 +390,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray) + grammar, err := structuredGrammar.Grammar(EnableMaybeString, EnableMaybeArray) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`mixedstring | arr | realvalue`), @@ -410,7 +409,8 @@ var _ = Describe("JSON schema grammar tests", func() { structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray, functions.NoMixedFreeString) + grammar, err := structuredGrammar.Grammar(EnableMaybeString, EnableMaybeArray, NoMixedFreeString) + Expect(err).To(BeNil()) results := strings.Split( strings.Join([]string{ rootResult(`freestring | arr | realvalue`), @@ -432,7 +432,8 @@ var _ = Describe("JSON schema grammar tests", func() { realvalue ("," realvalue)* )? "]"` - grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray, functions.DisableParallelNewLines) + grammar, err := structuredGrammar.Grammar(EnableMaybeString, EnableMaybeArray, DisableParallelNewLines) + Expect(err).To(BeNil()) results := strings.Split(content, "\n") for _, r := range results { if r != "" { diff --git a/pkg/functions/json_mode.go b/pkg/functions/json_mode.go new file mode 100644 index 00000000000..46361b74a8b --- /dev/null +++ b/pkg/functions/json_mode.go @@ -0,0 +1,28 @@ +package functions + +const ( + JSONBNF = `root ::= object +value ::= object | array | string | number | ("true" | "false" | "null") ws + +object ::= + "{" ws ( + string ":" ws value + ("," ws string ":" ws value)* + )? "}" ws + +array ::= + "[" ws ( + value + ("," ws value)* + )? "]" ws + +string ::= + "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes + )* "\"" ws + +number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws + +ws ::= ([ \t\n] ws)?` +)