From 1ffdfa75de8589334781e53860e52be49dfd1e2c Mon Sep 17 00:00:00 2001 From: Matthew Nitschke <39171685+matthewnitschke-wk@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:49:05 -0600 Subject: [PATCH] feat(cli): Adds a 'test' command for targeted testing (#236) Unlike the 'snapshot' command, this allows for checking the data in specific occurrences instead of checking ALL the SCIP data in a given Document. This command can potentially be used by people working on SCIP indexers. Co-authored-by: Varun Gandhi --- cmd/scip/main.go | 12 +- cmd/scip/main_test.go | 115 +++++ cmd/scip/snapshot.go | 14 +- cmd/scip/test.go | 454 ++++++++++++++++++ cmd/scip/test_test.go | 62 +++ .../fails-incorrect-diagnostic.repro | 3 + .../diagnostics/fails-no-diagnostic.repro | 3 + .../tests/test_cmd/diagnostics/passes.repro | 8 + cmd/scip/tests/test_cmd/ranges/fails.repro | 2 + cmd/scip/tests/test_cmd/ranges/passes.repro | 4 + .../test_cmd/roles/fails-wrong-role.repro | 2 + .../test_cmd/roles/fails-wrong-symbol.repro | 2 + cmd/scip/tests/test_cmd/roles/passes.repro | 9 + docs/CLI.md | 35 ++ docs/test_file_format.md | 54 +++ go.mod | 2 +- 16 files changed, 772 insertions(+), 9 deletions(-) create mode 100644 cmd/scip/test.go create mode 100644 cmd/scip/test_test.go create mode 100644 cmd/scip/tests/test_cmd/diagnostics/fails-incorrect-diagnostic.repro create mode 100644 cmd/scip/tests/test_cmd/diagnostics/fails-no-diagnostic.repro create mode 100644 cmd/scip/tests/test_cmd/diagnostics/passes.repro create mode 100644 cmd/scip/tests/test_cmd/ranges/fails.repro create mode 100644 cmd/scip/tests/test_cmd/ranges/passes.repro create mode 100644 cmd/scip/tests/test_cmd/roles/fails-wrong-role.repro create mode 100644 cmd/scip/tests/test_cmd/roles/fails-wrong-symbol.repro create mode 100644 cmd/scip/tests/test_cmd/roles/passes.repro create mode 100644 docs/test_file_format.md diff --git a/cmd/scip/main.go b/cmd/scip/main.go index c4a05e8..9da6ad9 100644 --- a/cmd/scip/main.go +++ b/cmd/scip/main.go @@ -23,7 +23,8 @@ func commands() []*cli.Command { print := printCommand() snapshot := snapshotCommand() stats := statsCommand() - return []*cli.Command{&lint, &print, &snapshot, &stats} + test := testCommand() + return []*cli.Command{&lint, &print, &snapshot, &stats, &test} } //go:embed version.txt @@ -74,6 +75,15 @@ func fromFlag(storage *string) *cli.StringFlag { } } +func commentSyntaxFlag(storage *string) *cli.StringFlag { + return &cli.StringFlag{ + Name: "comment-syntax", + Usage: "Comment syntax to use for snapshot files", + Destination: storage, + Value: "//", + } +} + func projectRootFlag(storage *string) *cli.StringFlag { return &cli.StringFlag{ Name: "project-root", diff --git a/cmd/scip/main_test.go b/cmd/scip/main_test.go index 981af4c..307b88a 100644 --- a/cmd/scip/main_test.go +++ b/cmd/scip/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "flag" "fmt" "os" @@ -8,7 +9,9 @@ import ( "strings" "testing" + "github.com/hexops/autogold/v2" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "github.com/sourcegraph/scip/bindings/go/scip" "github.com/sourcegraph/scip/bindings/go/scip/testutil" @@ -101,3 +104,115 @@ func TestSCIPSnapshots(t *testing.T) { return snapshots }) } + +func unwrap[T any](v T, err error) func(*testing.T) T { + return func(t *testing.T) T { + require.NoError(t, err) + return v + } +} + +func TestSCIPTests(t *testing.T) { + cwd := unwrap(os.Getwd())(t) + testDir := filepath.Join(cwd, "tests", "test_cmd") + testPaths := unwrap(os.ReadDir(testDir))(t) + require.Truef(t, len(testPaths) >= 1, "Expected at least one test case in directory: %v", testDir) + + os.Setenv("NO_COLOR", "1") + t.Cleanup(func() { + os.Unsetenv("NO_COLOR") + }) + + type TestCase struct { + dir string + passOutput autogold.Value + failOutput autogold.Value + } + + // To update the snapshot values, run 'go test ./cmd/scip -update'. + testCases := []TestCase{ + {"roles", + autogold.Expect("✓ passes.repro (3 assertions)\n"), + autogold.Expect(`✗ fails-wrong-role.repro + Failure - row: 0, column: 13 + Expected: 'reference reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().' + Actual: + - 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello().'✗ fails-wrong-symbol.repro + Failure - row: 0, column: 13 + Expected: 'definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello2().' + Actual: + - 'definition reprolang repro_manager roles 1.0.0 fails-wrong-symbol.repro/hello().'`), + }, + {"ranges", + autogold.Expect("✓ passes.repro (3 assertions)\n"), + autogold.Expect(`✗ fails.repro + Failure - row: 0, column: 10 + Expected: 'definition passes.repro/hello().' + Actual: + - No attributes found`), + }, + {"diagnostics", + autogold.Expect("✓ passes.repro (2 assertions)\n"), + autogold.Expect(`✗ fails-incorrect-diagnostic.repro + Failure - row: 0, column: 11 + Expected: 'diagnostic Warning:' + 'THIS IS NOT CORRECT' + Actual: + - 'definition reprolang repro_manager diagnostics 1.0.0 fails-incorrect-diagnostic.repro/deprecatedMethod.' + - 'diagnostic Warning' + 'deprecated identifier'✗ fails-no-diagnostic.repro + Failure - row: 0, column: 11 + Expected: 'diagnostic Warning:' + 'deprecated identifier' + Actual: + - 'definition reprolang repro_manager diagnostics 1.0.0 fails-no-diagnostic.repro/hello().'`), + }, + } + + for _, testPath := range testPaths { + require.Truef(t, slices.ContainsFunc(testCases, func(testCase TestCase) bool { + return testCase.dir == testPath.Name() + }), "Missing entry in testOutputs for %q", testPath.Name()) + } + + for _, testCase := range testCases { + var dirEntry os.DirEntry + require.Truef(t, slices.ContainsFunc(testPaths, func(entry os.DirEntry) bool { + if entry.Name() == testCase.dir { + dirEntry = entry + return true + } + return false + }), "Stale entry in testOutputs for %q; did you rename or remove the directory", testCase.dir) + + subtestDir := filepath.Join(testDir, dirEntry.Name()) + require.Truef(t, dirEntry.IsDir(), "not a directory: %q", subtestDir) + + t.Run(testCase.dir, func(t *testing.T) { + sources := unwrap(scip.NewSourcesFromDirectory(subtestDir))(t) + index := unwrap(repro.Index("file:/"+subtestDir, dirEntry.Name(), sources, []*repro.Dependency{}))(t) + + var passFiles, failFiles []string + testFiles := unwrap(os.ReadDir(subtestDir))(t) + for _, testFile := range testFiles { + if strings.HasPrefix(testFile.Name(), "passes") { + passFiles = append(passFiles, testFile.Name()) + } else if strings.HasPrefix(testFile.Name(), "fails") { + failFiles = append(failFiles, testFile.Name()) + } else { + t.Fatalf("Test files must start with 'passes' or 'fails'. Received %v", testFile.Name()) + } + } + + var passOutput bytes.Buffer + err := testMain(subtestDir, passFiles, index, "#", &passOutput) + require.NoError(t, err) + testCase.passOutput.Equal(t, passOutput.String()) + + var failOutput bytes.Buffer + err = testMain(subtestDir, failFiles, index, "#", &failOutput) + require.Error(t, err) + testCase.failOutput.Equal(t, failOutput.String()) + }) + } +} diff --git a/cmd/scip/snapshot.go b/cmd/scip/snapshot.go index 73e69c4..1415f9a 100644 --- a/cmd/scip/snapshot.go +++ b/cmd/scip/snapshot.go @@ -28,7 +28,12 @@ func snapshotCommand() cli.Command { Description: `The snapshot subcommand generates snapshot files which can be use for inspecting the output of an index in a visual way. Occurrences are marked with caret signs (^) -and symbol information.`, +and symbol information. + +For testing a SCIP indexer, you can either use this subcommand +along with 'git diff' or equivalent, or you can use the dedicated +'test' subcommand for more targeted checks. +`, Flags: []cli.Flag{ fromFlag(&snapshotFlags.from), &cli.StringFlag{ @@ -44,12 +49,7 @@ and symbol information.`, Destination: &snapshotFlags.strict, Value: true, }, - &cli.StringFlag{ - Name: "comment-syntax", - Usage: "Comment syntax to use for snapshot files", - Destination: &snapshotFlags.commentSyntax, - Value: "//", - }, + commentSyntaxFlag(&snapshotFlags.commentSyntax), }, Action: func(c *cli.Context) error { return snapshotMain(snapshotFlags) diff --git a/cmd/scip/test.go b/cmd/scip/test.go new file mode 100644 index 0000000..42e4d23 --- /dev/null +++ b/cmd/scip/test.go @@ -0,0 +1,454 @@ +package main + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/sourcegraph/scip/bindings/go/scip" + "github.com/urfave/cli/v2" + "golang.org/x/exp/slices" +) + +type testFlags struct { + from string // default: 'index.scip' + commentSyntax string // default: '//' + pathFilters cli.StringSlice +} + +func testCommand() cli.Command { + var testFlags testFlags + test := cli.Command{ + Name: "test", + Usage: "Validate a SCIP index against test files", + Description: fmt.Sprintf(`Validates whether the SCIP data present in an index +matches that specified in human-readable test files, using syntax +similar to the 'snapshot' subcommand. Test file syntax reference: + + https://github.com/sourcegraph/scip/blob/v%s/docs/test_file_format.md + +The test files are located based on the relative_path field +in the SCIP document, interpreted relative to the the directory +the CLI is invoked in. + +If you want to instead check all the data in a SCIP index, +use the 'snapshot' subcommand.`, version), + Flags: []cli.Flag{ + fromFlag(&testFlags.from), + commentSyntaxFlag(&testFlags.commentSyntax), + &cli.StringSliceFlag{ + Name: "filter", + Aliases: []string{"f"}, + Usage: "Explicit list of test files to check. Can be specified multiple times. If not specified, all files are tested.", + Destination: &testFlags.pathFilters, + }, + }, + Action: func(c *cli.Context) error { + dir := c.Args().Get(0) + + index, err := readFromOption(testFlags.from) + if err != nil { + return err + } + + return testMain(dir, testFlags.pathFilters.Value(), index, testFlags.commentSyntax, os.Stdout) + }, + } + return test +} + +func testMain( + directory string, + fileFilters []string, + index *scip.Index, + commentSyntax string, + output io.Writer, +) error { + hasFailure := false + + fileFilterSet := map[string]struct{}{} + for _, file := range fileFilters { + fileFilterSet[file] = struct{}{} + } + + allTestFilesSet := map[string]struct{}{} + if err := filepath.WalkDir(directory, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Ext(path) == ".scip" { + return nil + } + if len(fileFilterSet) > 0 { + if _, ok := fileFilterSet[path]; ok { + allTestFilesSet[path] = struct{}{} + } + } else if !d.IsDir() { + allTestFilesSet[path] = struct{}{} + } + return nil + }); err != nil { + return err + } + + for _, document := range index.Documents { + sourceFilePath := filepath.Join(directory, document.RelativePath) + + if len(fileFilters) > 0 { + if !slices.Contains(fileFilters, document.RelativePath) { + continue + } + } + + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + delete(allTestFilesSet, sourceFilePath) + + failures := []string{} + successCount := 0 + + lines := strings.Split(string(data), "\n") + for lineNumber := 0; lineNumber < len(lines); lineNumber++ { + testCasesAtLine, usedLines := testCasesForLine(lineNumber, lines, commentSyntax) + + // if the test file contains no test lines, skip it. Only test the lines + // that the test file dictates should be tested + if len(testCasesAtLine) == 0 { + continue + } + + attributes := attributesForOccurrencesAtLine(lineNumber, document.Occurrences) + for _, testCase := range testCasesAtLine { + filteredAttrs := filterAttributesForTestCase(testCase, attributes) + if !testCase.checkAll(filteredAttrs) { + failures = append(failures, formatFailure(lineNumber, testCase, filteredAttrs)) + } else { + successCount++ + } + } + + lineNumber += usedLines + } + + if len(failures) > 0 { + hasFailure = true + red := color.New(color.FgRed) + red.Fprintf(output, "✗ %s\n", document.RelativePath) + + for _, failure := range failures { + fmt.Fprintf(output, indent(failure, 4)) + } + } else { + green := color.New(color.FgGreen) + green.Fprintf(output, "✓ %s (%d assertions)\n", document.RelativePath, successCount) + } + } + + if len(allTestFilesSet) > 0 { + sortedFiles := []string{} + for f, _ := range allTestFilesSet { + sortedFiles = append(sortedFiles, f) + } + slices.Sort(sortedFiles) + red := color.New(color.FgRed) + red.Fprintf(output, "✗ Missing documents in SCIP index\n") + for _, path := range sortedFiles { + fmt.Fprintf(output, " %s\n", path) + } + hasFailure = true + } + + if hasFailure { + return cli.Exit("", 1) + } + + return nil +} + +// symbolAttribute refers to a single attribute of a symbol. +// This can be a definition, reference, documentation, or diagnostic +type symbolAttribute struct { + // the column number where this symbol starts + start int + + // the length of the symbol's name + length int + + // the type of attribute that this is + kind symbolAttributeKind + + // contextual information about the attribute, as determined + // by the [kind] + data string + + // any additional information necessary to represent this attribute + // each line should be considered a "newline", and is used for multiline + // comments (in documentation), and diagnostic information + additionalData []string +} + +type symbolAttributeKind string + +const ( + definitionAttrKind symbolAttributeKind = "definition" + referenceAttrKind symbolAttributeKind = "reference" + forwardDefinitionAttrKind symbolAttributeKind = "forward_definition" + diagnosticAttrKind symbolAttributeKind = "diagnostic" +) + +func symbolAttributeKindFromStr(str string) symbolAttributeKind { + switch str { + case "definition": + return definitionAttrKind + case "reference": + return referenceAttrKind + case "forward_definition": + return forwardDefinitionAttrKind + case "diagnostic": + return diagnosticAttrKind + default: + panic(fmt.Sprintf("Unknown symbolAttributeKind: %s", str)) + } +} + +// symbolAttributeTestCase refers to metadata used to validate +// [symbolAttributes] +type symbolAttributeTestCase struct { + attribute symbolAttribute + enforceLength bool +} + +// commentsForLine returns the list of lines, after a provided [lineNumber], which are +// classified as comment. The comment type can be configured using [commentSyntax]. +// +// Returns the list of symbolAttributeTestCase(s) for the provided line, and the number of +// of lines that were "consumed" by the cases on this line +func testCasesForLine(lineNumber int, lines []string, commentSyntax string) ([]symbolAttributeTestCase, int) { + testCases := []symbolAttributeTestCase{} + + // if the specified lineNumber is outside the bounds of lines + // return an empty array + if lineNumber >= len(lines)-1 { + return testCases, 0 + } + + testLines := []symbolAttributeTestCase{} + usedLines := 0 + for i := lineNumber + 1; i < len(lines); i++ { + line := lines[i] + + if !strings.HasPrefix(strings.TrimSpace(line), commentSyntax) { + break + } + testCase := parseTestCase(line, lines[i+1:], commentSyntax) + + testLines = append(testLines, testCase) + i += len(testCase.attribute.additionalData) + usedLines += 1 + len(testCase.attribute.additionalData) + } + + return testLines, usedLines +} + +func filterAttributesForTestCase(testCase symbolAttributeTestCase, attributes []symbolAttribute) []symbolAttribute { + filteredAttrs := []symbolAttribute{} + for _, attr := range attributes { + if testCase.attribute.start >= attr.start && testCase.attribute.start <= (attr.start+attr.length)-1 { + filteredAttrs = append(filteredAttrs, attr) + } + } + return filteredAttrs +} + +func attributesForOccurrencesAtLine(lineNumber int, occurrences []*scip.Occurrence) []symbolAttribute { + result := []symbolAttribute{} + for _, occ := range occurrences { + if occ.Range[0] == int32(lineNumber) { + pos, _ := scip.NewRange(occ.Range) + + start := int(pos.Start.Character) + length := int(pos.End.Character - pos.Start.Character) + + kind := referenceAttrKind + if scip.SymbolRole_Definition.Matches(occ) { + kind = definitionAttrKind + } else if scip.SymbolRole_ForwardDefinition.Matches(occ) { + kind = forwardDefinitionAttrKind + } + result = append(result, symbolAttribute{ + start: start, + length: length, + kind: kind, + data: occ.Symbol, + additionalData: []string{}, + }) + + for _, diagnostic := range occ.Diagnostics { + result = append(result, symbolAttribute{ + start: start, + length: length, + kind: diagnosticAttrKind, + data: diagnostic.Severity.String(), + additionalData: []string{ + diagnostic.Message, + }, + }) + } + } + } + return result +} + +func parseTestCase(line string, leadingLines []string, commentSyntax string) symbolAttributeTestCase { + start := 0 + length := 0 + enforceLength := false + + if strings.Contains(line, "<-") { + // if the test line selects via `<-`, treat the symbol selection + // as the location of the commentSyntax + start = strings.Index(line, commentSyntax) + line = strings.Replace(line, "<-", "", 1) + } else { + // otherwise treat the start as the first `^` + start = strings.Index(line, "^") + + // a single `^` dictates no length enforcement + // anything more signifies length should be verified + if strings.Contains(line, "^^") { + enforceLength = true + length = strings.Count(line, "^") + } + line = strings.ReplaceAll(line, "^", "") + } + + // remove the comment prefix & whitespace + line = strings.TrimSpace(strings.Replace(line, commentSyntax, "", 1)) + + // the type of the symbol should be the first word + // this is "definition", "reference", "documentation", "diagnostic", etc.. + kindStr := strings.Split(line, " ")[0] + + // the data is everything except the type + data := strings.TrimSpace(strings.Replace(line, kindStr, "", 1)) + + additionalData := []string{} + for i := range leadingLines { + leadingLine := leadingLines[i] + // if the leadingLine is not a comment line, we're outside of the test case block. + if !strings.HasPrefix(strings.TrimSpace(leadingLine), commentSyntax) { + break + } + + // remove the comment character(s) + leadingLine = strings.Replace(leadingLine, commentSyntax, "", 1) + + // if the leading line doesn't start with '>', its not a multiline case + if !strings.HasPrefix(strings.TrimSpace(leadingLine), ">") { + break + } + + // remove the '>' character + leadingLine = strings.Replace(leadingLine, ">", "", 1) + additionalData = append(additionalData, strings.TrimSpace(leadingLine)) + } + + return symbolAttributeTestCase{ + attribute: symbolAttribute{ + kind: symbolAttributeKindFromStr(kindStr), + start: start, + length: length, + data: data, + additionalData: additionalData, + }, + enforceLength: enforceLength, + } +} + +func (s symbolAttributeTestCase) checkAll(attributes []symbolAttribute) bool { + for _, attr := range attributes { + if s.check(attr) { + return true + } + } + return false +} + +func (s symbolAttributeTestCase) check(attr symbolAttribute) bool { + if s.enforceLength { + if s.attribute.length != attr.length || s.attribute.start != attr.start { + return false + } + } else { + if s.attribute.start < attr.start || s.attribute.start > (attr.start+attr.length)-1 { + return false + } + } + + if s.attribute.kind != attr.kind { + return false + } + + // check if symbols are equal, a `.` character in the testCaseSymbol is considered + // a wildcard, and matches the correlating group + testCaseSymbolParts := strings.Split(s.attribute.data, " ") + attrSymbolParts := strings.Split(attr.data, " ") + for i, testCaseSymbolPart := range testCaseSymbolParts { + if testCaseSymbolPart == "." { + continue + } + if testCaseSymbolPart != attrSymbolParts[i] { + return false + } + } + + // only validate additionalData if the testCases provides one + // otherwise, ignore what the attribute specifies + if len(s.attribute.additionalData) > 0 { + if !slices.Equal(s.attribute.additionalData, attr.additionalData) { + return false + } + } + + return true +} + +func formatFailure(lineNumber int, testCase symbolAttributeTestCase, attributesAtLine []symbolAttribute) string { + failureDesc := []string{ + fmt.Sprintf("Failure - row: %d, column: %d", lineNumber, testCase.attribute.start), + fmt.Sprintf(" Expected: '%s %s'", testCase.attribute.kind, testCase.attribute.data), + } + for _, add := range testCase.attribute.additionalData { + failureDesc = append(failureDesc, indent(fmt.Sprintf("'%s'", add), 12)) + } + + failureDesc = append(failureDesc, " Actual:") + if (len(attributesAtLine)) == 0 { + failureDesc = append(failureDesc, " - No attributes found") + } else { + for _, attr := range attributesAtLine { + failureDesc = append(failureDesc, fmt.Sprintf(" - '%s %s'", attr.kind, attr.data)) + for _, add := range attr.additionalData { + failureDesc = append(failureDesc, indent(fmt.Sprintf("'%s'", add), 6)) + } + } + } + return strings.Join(failureDesc, "\n") +} + +// --------------------------------- Utils --------------------------------- + +func indent(str string, count int) string { + lines := strings.Split(str, "\n") + newLines := []string{} + for _, line := range lines { + newLines = append(newLines, strings.Repeat(" ", count)+line) + } + return strings.Join(newLines, "\n") +} diff --git a/cmd/scip/test_test.go b/cmd/scip/test_test.go new file mode 100644 index 0000000..1ce6baa --- /dev/null +++ b/cmd/scip/test_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + + "golang.org/x/exp/slices" + + "github.com/stretchr/testify/require" +) + +func TestTestCasesForLine(t *testing.T) { + actual, linesUsed := testCasesForLine( + 2, + []string{ + "void main() {", + " // ^ definition ", + " print(foo)", + " // ^^^ reference ", + " // ^ reference ", + " // <- reference ", + " // ^ diagnostic Warning", + " // > with multiline", + " // > additional data", + " final test = 4", + " ^ definition ", + "}", + }, + "//", + ) + + // total of 4 test cases on the specified line + require.Equal(t, 4, len(actual)) + require.Equal(t, 6, linesUsed) + + // only the first test case has enforceLength = true + require.True(t, actual[0].enforceLength) + require.Equal(t, 3, actual[0].attribute.length) + require.False(t, actual[1].enforceLength) + require.False(t, actual[2].enforceLength) + require.False(t, actual[3].enforceLength) + + // all test cases should have the same start + require.Equal(t, 8, actual[0].attribute.start) + require.Equal(t, 9, actual[1].attribute.start) // different start, same symbol + require.Equal(t, 8, actual[2].attribute.start) + require.Equal(t, 8, actual[3].attribute.start) + + // validate kind on each test case + require.Equal(t, referenceAttrKind, actual[0].attribute.kind) + require.Equal(t, referenceAttrKind, actual[1].attribute.kind) + require.Equal(t, referenceAttrKind, actual[2].attribute.kind) + require.Equal(t, diagnosticAttrKind, actual[3].attribute.kind) + + // validate that additionalData is correctly parsed + require.True( + t, + slices.Equal( + actual[3].attribute.additionalData, + []string{"with multiline", "additional data"}, + ), + ) +} diff --git a/cmd/scip/tests/test_cmd/diagnostics/fails-incorrect-diagnostic.repro b/cmd/scip/tests/test_cmd/diagnostics/fails-incorrect-diagnostic.repro new file mode 100644 index 0000000..dfaf9f6 --- /dev/null +++ b/cmd/scip/tests/test_cmd/diagnostics/fails-incorrect-diagnostic.repro @@ -0,0 +1,3 @@ +definition deprecatedMethod. +# ^ diagnostic Warning: +# > THIS IS NOT CORRECT \ No newline at end of file diff --git a/cmd/scip/tests/test_cmd/diagnostics/fails-no-diagnostic.repro b/cmd/scip/tests/test_cmd/diagnostics/fails-no-diagnostic.repro new file mode 100644 index 0000000..3e77905 --- /dev/null +++ b/cmd/scip/tests/test_cmd/diagnostics/fails-no-diagnostic.repro @@ -0,0 +1,3 @@ +definition hello(). +# ^ diagnostic Warning: +# > deprecated identifier diff --git a/cmd/scip/tests/test_cmd/diagnostics/passes.repro b/cmd/scip/tests/test_cmd/diagnostics/passes.repro new file mode 100644 index 0000000..97c032a --- /dev/null +++ b/cmd/scip/tests/test_cmd/diagnostics/passes.repro @@ -0,0 +1,8 @@ +definition deprecatedMethod. +# ^ diagnostic Warning +# > deprecated identifier + +reference deprecatedMethod. +# ^ diagnostic Warning +# > deprecated identifier + diff --git a/cmd/scip/tests/test_cmd/ranges/fails.repro b/cmd/scip/tests/test_cmd/ranges/fails.repro new file mode 100644 index 0000000..0a2b492 --- /dev/null +++ b/cmd/scip/tests/test_cmd/ranges/fails.repro @@ -0,0 +1,2 @@ +definition hello(). +# ^ definition passes.repro/hello(). \ No newline at end of file diff --git a/cmd/scip/tests/test_cmd/ranges/passes.repro b/cmd/scip/tests/test_cmd/ranges/passes.repro new file mode 100644 index 0000000..c1faaa6 --- /dev/null +++ b/cmd/scip/tests/test_cmd/ranges/passes.repro @@ -0,0 +1,4 @@ +definition hello(). +# ^^^^^^^^ definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). +# ^ definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). + # <- definition reprolang repro_manager ranges 1.0.0 passes.repro/hello(). \ No newline at end of file diff --git a/cmd/scip/tests/test_cmd/roles/fails-wrong-role.repro b/cmd/scip/tests/test_cmd/roles/fails-wrong-role.repro new file mode 100644 index 0000000..b097686 --- /dev/null +++ b/cmd/scip/tests/test_cmd/roles/fails-wrong-role.repro @@ -0,0 +1,2 @@ +definition hello(). +# ^ reference reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello(). \ No newline at end of file diff --git a/cmd/scip/tests/test_cmd/roles/fails-wrong-symbol.repro b/cmd/scip/tests/test_cmd/roles/fails-wrong-symbol.repro new file mode 100644 index 0000000..8d6f177 --- /dev/null +++ b/cmd/scip/tests/test_cmd/roles/fails-wrong-symbol.repro @@ -0,0 +1,2 @@ +definition hello(). +# ^ definition reprolang repro_manager roles 1.0.0 fails-wrong-role.repro/hello2(). \ No newline at end of file diff --git a/cmd/scip/tests/test_cmd/roles/passes.repro b/cmd/scip/tests/test_cmd/roles/passes.repro new file mode 100644 index 0000000..5aca097 --- /dev/null +++ b/cmd/scip/tests/test_cmd/roles/passes.repro @@ -0,0 +1,9 @@ +definition hello(). +# ^ definition reprolang repro_manager roles 1.0.0 passes.repro/hello(). + +reference hello(). +# ^ reference reprolang repro_manager roles 1.0.0 passes.repro/hello(). + +definition abc# +reference forward_definition abc# +# ^ forward_definition reprolang repro_manager roles 1.0.0 passes.repro/abc# \ No newline at end of file diff --git a/docs/CLI.md b/docs/CLI.md index 5cd50e1..286ab59 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -29,6 +29,7 @@ COMMANDS: print Print a SCIP index for debugging snapshot Generate snapshot files for golden testing stats Output useful statistics about a SCIP index + test Validate a SCIP index against test files help, h Shows a list of commands or help for one command GLOBAL OPTIONS: @@ -88,6 +89,10 @@ DESCRIPTION: visual way. Occurrences are marked with caret signs (^) and symbol information. + For testing a SCIP indexer, you can either use this subcommand + along with 'git diff' or equivalent, or you can use the dedicated + 'test' subcommand for more targeted checks. + OPTIONS: --from value Path to SCIP index file (default: "index.scip") --to value Path to output directory for snapshot files (default: "scip-snapshot") @@ -96,6 +101,36 @@ OPTIONS: --comment-syntax value Comment syntax to use for snapshot files (default: "//") ``` +## `scip test` + +``` +NAME: + scip test - Validate a SCIP index against test files + +USAGE: + scip test [command options] [arguments...] + +DESCRIPTION: + Validates whether the SCIP data as + in a given SCIP index matches that specified in human-readable test files, + using syntax similar to the 'snapshot subcommand'. Test file syntax reference: + + https://github.com/sourcegraph/scip/blob/v0.4.0/docs/test_file_format.md + + The test files are located based on the relative_path field + in the SCIP document, interpreted relative to the the directory + the CLI is invoked in. + + If you want to instead check all the data in a SCIP index, + use the 'snapshot' subcommand. + +OPTIONS: + --from value Path to SCIP index file (default: "index.scip") + --comment-syntax value Comment syntax to use for snapshot files (default: "//") + --filter value, -f value [ --filter value, -f value ] Explicit list of test files to check. Can be specified multiple times. If not specified, all files are tested. + --help, -h show help +``` + ## `scip stats` ``` diff --git a/docs/test_file_format.md b/docs/test_file_format.md new file mode 100644 index 0000000..c7a3550 --- /dev/null +++ b/docs/test_file_format.md @@ -0,0 +1,54 @@ +# `scip test` file format + +The `scip test` command validates whether a provided SCIP index contains the data specified in a human-readable test file. +The test file syntax is inspired by [Sublime Text's syntax highlighting tests](https://www.sublimetext.com/docs/syntax.html#testing). + +## File Format + +Test cases are made up of a range, type, and data attribute. + +### Ranges + +Three range selection comment formats are supported: + +- `// ^^^` (2 or more `^`): enforces the length of the occurrence. Will fail if the range at this location does not equal 3 characters +- `// ^`: ignore length, `^` can occur at any point to any character in the occurrence +- `// <-`: ignore length, and treat the character above the first comment character as the start of the occurrence, similar to Sublime Text + +```js +function someFunction() { + // ^ ... + // ^^^^^^^^^^^^ ... + // <- ... +} +``` + +### Type and Data + +There are four possible types test cases. The chosen test case is determined by the first word after the range selection + +- `definition [symbol]` - validates that the specified range has a symbol with the role of "definition" with the specified `[symbol]` +- `reference [symbol]` - validates that the specified range has a symbol with the role of "reference" with the specified `[symbol]` +- `forward_definition [symbol]` - validates that the specified range has a symbol with the role of "forward_definition" with the specified `[symbol]` +- `diagnostic [severity] [message]` - validates that the specified range has a diagnostic with the given `[severity]` and `[message]` + +```js +function someFunction() { + // ^ definition scip-typescript npm test_package 1.0.0 lib/`test.js`/someFunction(). + + someOtherFunction() + // <- reference scip-typescript npm test_package 1.0.0 lib/`test.js`/someOtherFunction(). +} +``` + +The message for diagnostics can be specified on the following line using `>`, +and may span over multiple lines. + +```js +function someFn() { + let someVar = '' + // ^ diagnostic Warning + // > someVar is unused. + // > remove it or use it. +} +``` diff --git a/go.mod b/go.mod index e43cb35..78152d4 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/bufbuild/buf v1.25.0 github.com/cockroachdb/errors v1.8.9 + github.com/fatih/color v1.15.0 github.com/google/go-cmp v0.5.9 github.com/google/gofuzz v1.1.0 github.com/hexops/autogold/v2 v2.2.1 @@ -43,7 +44,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect - github.com/fatih/color v1.15.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect