From 7c70b2c415ae0c4ddaf6ac1e00439d92b7e4a881 Mon Sep 17 00:00:00 2001 From: Stefan Profanter Date: Mon, 23 Oct 2023 22:17:50 +0200 Subject: [PATCH] feat: add conaninfo.txt parser to detect conan packages in docker images (#2234) * feat: add conaninfo.txt parser to detect conan packages in docker images Signed-off-by: Stefan Profanter * fix: add NewConanInfoCataloger as a separate cataloger Signed-off-by: Stefan Profanter --------- Signed-off-by: Stefan Profanter --- syft/pkg/cataloger/cataloger.go | 1 + syft/pkg/cataloger/cpp/cataloger.go | 8 + syft/pkg/cataloger/cpp/cataloger_test.go | 25 ++++ syft/pkg/cataloger/cpp/package.go | 8 + syft/pkg/cataloger/cpp/parse_conaninfo.go | 141 ++++++++++++++++++ .../pkg/cataloger/cpp/parse_conaninfo_test.go | 134 +++++++++++++++++ .../conaninfo.txt | 115 ++++++++++++++ .../glob-paths/somewhere/src/conaninfo.txt | 0 8 files changed, 432 insertions(+) create mode 100644 syft/pkg/cataloger/cpp/parse_conaninfo.go create mode 100644 syft/pkg/cataloger/cpp/parse_conaninfo_test.go create mode 100644 syft/pkg/cataloger/cpp/test-fixtures/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt create mode 100644 syft/pkg/cataloger/cpp/test-fixtures/glob-paths/somewhere/src/conaninfo.txt diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index 3b836ee885f..d16026e380f 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -45,6 +45,7 @@ func ImageCatalogers(cfg Config) []pkg.Cataloger { alpm.NewAlpmdbCataloger(), apkdb.NewApkdbCataloger(), binary.NewCataloger(), + cpp.NewConanInfoCataloger(), deb.NewDpkgdbCataloger(), dotnet.NewDotnetPortableExecutableCataloger(), golang.NewGoModuleBinaryCataloger(cfg.Golang), diff --git a/syft/pkg/cataloger/cpp/cataloger.go b/syft/pkg/cataloger/cpp/cataloger.go index 80c6b5b11b2..5756e78ca4f 100644 --- a/syft/pkg/cataloger/cpp/cataloger.go +++ b/syft/pkg/cataloger/cpp/cataloger.go @@ -12,3 +12,11 @@ func NewConanCataloger() *generic.Cataloger { WithParserByGlobs(parseConanfile, "**/conanfile.txt"). WithParserByGlobs(parseConanlock, "**/conan.lock") } + +const catalogerNameInfo = "conan-info-cataloger" + +// NewConanInfoCataloger returns a new C++ conaninfo.txt cataloger object. +func NewConanInfoCataloger() *generic.Cataloger { + return generic.NewCataloger(catalogerNameInfo). + WithParserByGlobs(parseConaninfo, "**/conaninfo.txt") +} diff --git a/syft/pkg/cataloger/cpp/cataloger_test.go b/syft/pkg/cataloger/cpp/cataloger_test.go index 144d4cc915c..56887ac02f4 100644 --- a/syft/pkg/cataloger/cpp/cataloger_test.go +++ b/syft/pkg/cataloger/cpp/cataloger_test.go @@ -31,3 +31,28 @@ func TestCataloger_Globs(t *testing.T) { }) } } + +func TestCatalogerInfo_Globs(t *testing.T) { + tests := []struct { + name string + fixture string + expected []string + }{ + { + name: "obtain conan files", + fixture: "test-fixtures/glob-paths", + expected: []string{ + "somewhere/src/conaninfo.txt", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pkgtest.NewCatalogTester(). + FromDirectory(t, test.fixture). + ExpectsResolverContentQueries(test.expected). + TestCataloger(t, NewConanInfoCataloger()) + }) + } +} diff --git a/syft/pkg/cataloger/cpp/package.go b/syft/pkg/cataloger/cpp/package.go index b093c928d09..8b8bd91684a 100644 --- a/syft/pkg/cataloger/cpp/package.go +++ b/syft/pkg/cataloger/cpp/package.go @@ -14,6 +14,7 @@ type conanRef struct { User string Channel string Revision string + PackageID string Timestamp string } @@ -32,6 +33,13 @@ func splitConanRef(ref string) *conanRef { cref.Timestamp = tokens[1] } + // package_id + tokens = strings.Split(text, ":") + text = tokens[0] + if len(tokens) == 2 { + cref.PackageID = tokens[1] + } + // revision tokens = strings.Split(text, "#") ref = tokens[0] diff --git a/syft/pkg/cataloger/cpp/parse_conaninfo.go b/syft/pkg/cataloger/cpp/parse_conaninfo.go new file mode 100644 index 00000000000..b8f532b6fcf --- /dev/null +++ b/syft/pkg/cataloger/cpp/parse_conaninfo.go @@ -0,0 +1,141 @@ +package cpp + +import ( + "bufio" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +var _ generic.Parser = parseConaninfo + +func parseConanMetadataFromFilePath(path string) (pkg.ConanLockMetadata, error) { + // fullFilePath = str(reader.Location.VirtualPath) + // Split the full patch into the folders we expect. I.e.: + // $HOME/.conan/data/////package//conaninfo.txt + re := regexp.MustCompile(`.*[/\\](?P[^/\\]+)[/\\](?P[^/\\]+)[/\\](?P[^/\\]+)[/\\](?P[^/\\]+)[/\\]package[/\\](?P[^/\\]+)[/\\]conaninfo\.txt`) + matches := re.FindStringSubmatch(path) + if len(matches) != 6 { + return pkg.ConanLockMetadata{}, fmt.Errorf("failed to get parent package info from conaninfo file path") + } + mainPackageRef := fmt.Sprintf("%s/%s@%s/%s", matches[1], matches[2], matches[3], matches[4]) + return pkg.ConanLockMetadata{ + Ref: mainPackageRef, + PackageID: matches[5], + }, nil +} + +func getRelationships(pkgs []pkg.Package, mainPackageRef pkg.Package) []artifact.Relationship { + var relationships []artifact.Relationship + for _, p := range pkgs { + // this is a pkg that package "main_package" depends on... make a relationship + relationships = append(relationships, artifact.Relationship{ + From: p, + To: mainPackageRef, + Type: artifact.DependencyOfRelationship, + }) + } + return relationships +} + +func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[]pkg.Package) { + if len(line) == 0 { + return + } + + cref := splitConanRef(line) + + meta := pkg.ConanLockMetadata{ + Ref: line, + PackageID: cref.PackageID, + } + + p := newConanlockPackage( + meta, + reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), + ) + if p != nil { + *pkgs = append(*pkgs, *p) + } +} + +// parseConaninfo is a parser function for conaninfo.txt contents, returning all packages discovered. +// The conaninfo.txt file is typically present for an installed conan package under: +// $HOME/.conan/data/////package//conaninfo.txt +// Based on the relative path we can get: +// - package name +// - package version +// - package id +// - user +// - channel +// The conaninfo.txt gives: +// - package requires (full_requires) +// - recipe revision (recipe_hash) +func parseConaninfo(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + // First set the base package info by checking the relative path + fullFilePath := string(reader.Location.LocationData.Reference().RealPath) + if len(fullFilePath) == 0 { + fullFilePath = reader.Location.LocationData.RealPath + } + + mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath) + if err != nil { + return nil, nil, err + } + + r := bufio.NewReader(reader) + inRequirements := false + inRecipeHash := false + var pkgs []pkg.Package + + for { + line, err := r.ReadString('\n') + switch { + case errors.Is(io.EOF, err): + mainPackage := newConanlockPackage( + mainMetadata, + reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), + ) + + mainPackageRef := *mainPackage + relationships := getRelationships(pkgs, mainPackageRef) + + pkgs = append(pkgs, mainPackageRef) + + return pkgs, relationships, nil + case err != nil: + return nil, nil, fmt.Errorf("failed to parse conaninfo.txt file: %w", err) + } + + switch { + case strings.Contains(line, "[full_requires]"): + inRequirements = true + inRecipeHash = false + continue + case strings.Contains(line, "[recipe_hash]"): + inRequirements = false + inRecipeHash = true + continue + case strings.ContainsAny(line, "[]") || strings.HasPrefix(strings.TrimSpace(line), "#"): + inRequirements = false + inRecipeHash = false + continue + } + + if inRequirements { + parseFullRequiresLine(strings.Trim(line, "\n "), reader, &pkgs) + } + if inRecipeHash { + // add recipe hash to the metadata ref + mainMetadata.Ref = mainMetadata.Ref + "#" + strings.Trim(line, "\n ") + inRecipeHash = false + } + } +} diff --git a/syft/pkg/cataloger/cpp/parse_conaninfo_test.go b/syft/pkg/cataloger/cpp/parse_conaninfo_test.go new file mode 100644 index 00000000000..e7fe4f18103 --- /dev/null +++ b/syft/pkg/cataloger/cpp/parse_conaninfo_test.go @@ -0,0 +1,134 @@ +package cpp + +import ( + "testing" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" +) + +func TestParseConaninfo(t *testing.T) { + fixture := "test-fixtures/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt" + expected := []pkg.Package{ + { + Name: "mfast", + Version: "1.2.2", + PURL: "pkg:conan/my_user/mfast@1.2.2?channel=my_channel", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "mfast/1.2.2@my_user/my_channel#c6f6387c9b99780f0ee05e25f99d0f39", + PackageID: "9d1f076b471417647c2022a78d5e2c1f834289ac", + }, + }, + { + Name: "boost", + Version: "1.75.0", + PURL: "pkg:conan/boost@1.75.0", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "boost/1.75.0:dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978", + PackageID: "dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978", + }, + }, + { + Name: "zlib", + Version: "1.2.13", + PURL: "pkg:conan/zlib@1.2.13", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "zlib/1.2.13:dfbe50feef7f3c6223a476cd5aeadb687084a646", + PackageID: "dfbe50feef7f3c6223a476cd5aeadb687084a646", + }, + }, + { + Name: "bzip2", + Version: "1.0.8", + PURL: "pkg:conan/bzip2@1.0.8", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "bzip2/1.0.8:c32092bf4d4bb47cf962af898e02823f499b017e", + PackageID: "c32092bf4d4bb47cf962af898e02823f499b017e", + }, + }, + { + Name: "libbacktrace", + Version: "cci.20210118", + PURL: "pkg:conan/libbacktrace@cci.20210118", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "libbacktrace/cci.20210118:dfbe50feef7f3c6223a476cd5aeadb687084a646", + PackageID: "dfbe50feef7f3c6223a476cd5aeadb687084a646", + }, + }, + { + Name: "tinyxml2", + Version: "9.0.0", + PURL: "pkg:conan/tinyxml2@9.0.0", + Locations: file.NewLocationSet(file.NewLocation(fixture)), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: pkg.ConanLockMetadata{ + Ref: "tinyxml2/9.0.0:6557f18ca99c0b6a233f43db00e30efaa525e27e", + PackageID: "6557f18ca99c0b6a233f43db00e30efaa525e27e", + }, + }, + } + + // relationships require IDs to be set to be sorted similarly + for i := range expected { + expected[i].SetID() + } + + var expectedRelationships = []artifact.Relationship{ + { + From: expected[1], // boost + To: expected[0], // mfast + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: expected[5], // tinyxml2 + To: expected[0], // mfast + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: expected[2], // zlib + To: expected[0], // mfast + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: expected[3], // bzip2 + To: expected[0], // mfast + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: expected[4], // libbacktrace + To: expected[0], // mfast + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + } + + pkgtest.TestFileParser(t, fixture, parseConaninfo, expected, expectedRelationships) +} diff --git a/syft/pkg/cataloger/cpp/test-fixtures/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt b/syft/pkg/cataloger/cpp/test-fixtures/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt new file mode 100644 index 00000000000..fad6e3f3aa7 --- /dev/null +++ b/syft/pkg/cataloger/cpp/test-fixtures/conaninfo/mfast/1.2.2/my_user/my_channel/package/9d1f076b471417647c2022a78d5e2c1f834289ac/conaninfo.txt @@ -0,0 +1,115 @@ +[settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + +[requires] + boost/1.Y.Z + tinyxml2/9.Y.Z + +[options] + fPIC=True + shared=False + with_sqlite3=False + +[full_settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + +[full_requires] + boost/1.75.0:dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978 + bzip2/1.0.8:c32092bf4d4bb47cf962af898e02823f499b017e + libbacktrace/cci.20210118:dfbe50feef7f3c6223a476cd5aeadb687084a646 + tinyxml2/9.0.0:6557f18ca99c0b6a233f43db00e30efaa525e27e + zlib/1.2.13:dfbe50feef7f3c6223a476cd5aeadb687084a646 + +[full_options] + fPIC=True + shared=False + with_sqlite3=False + boost:addr2line_location=/usr/bin/addr2line + boost:asio_no_deprecated=False + boost:buildid=None + boost:bzip2=True + boost:debug_level=0 + boost:diagnostic_definitions=False + boost:error_code_header_only=False + boost:extra_b2_flags=None + boost:fPIC=True + boost:filesystem_no_deprecated=False + boost:filesystem_use_std_fs=False + boost:filesystem_version=None + boost:header_only=False + boost:i18n_backend=deprecated + boost:i18n_backend_iconv=libc + boost:i18n_backend_icu=False + boost:layout=system + boost:lzma=False + boost:magic_autolink=False + boost:multithreading=True + boost:namespace=boost + boost:namespace_alias=False + boost:numa=True + boost:pch=True + boost:python_executable=None + boost:python_version=None + boost:segmented_stacks=False + boost:shared=False + boost:system_no_deprecated=False + boost:system_use_utf8=False + boost:visibility=hidden + boost:with_stacktrace_backtrace=True + boost:without_atomic=False + boost:without_chrono=False + boost:without_container=False + boost:without_context=False + boost:without_contract=False + boost:without_coroutine=False + boost:without_date_time=False + boost:without_exception=False + boost:without_fiber=False + boost:without_filesystem=False + boost:without_graph=False + boost:without_graph_parallel=True + boost:without_iostreams=False + boost:without_json=False + boost:without_locale=False + boost:without_log=False + boost:without_math=False + boost:without_mpi=True + boost:without_nowide=False + boost:without_program_options=False + boost:without_python=True + boost:without_random=False + boost:without_regex=False + boost:without_serialization=False + boost:without_stacktrace=False + boost:without_system=False + boost:without_test=False + boost:without_thread=False + boost:without_timer=False + boost:without_type_erasure=False + boost:without_wave=False + boost:zlib=True + boost:zstd=False + bzip2:build_executable=True + bzip2:fPIC=True + bzip2:shared=False + libbacktrace:fPIC=True + libbacktrace:shared=False + tinyxml2:fPIC=True + tinyxml2:shared=False + zlib:fPIC=True + zlib:shared=False + +[recipe_hash] + c6f6387c9b99780f0ee05e25f99d0f39 + +[env] diff --git a/syft/pkg/cataloger/cpp/test-fixtures/glob-paths/somewhere/src/conaninfo.txt b/syft/pkg/cataloger/cpp/test-fixtures/glob-paths/somewhere/src/conaninfo.txt new file mode 100644 index 00000000000..e69de29bb2d