Guess unpinned versions in python requirements.txt (#1966)

* feat: python requirements.txt parsing inclusive

Signed-off-by: manifestori <[email protected]>

* refactor: parseVersion

Signed-off-by: manifestori <[email protected]>

* add python config for optional requirements version constraint resolution

Signed-off-by: Alex Goodman <[email protected]>

* fix tests

Signed-off-by: Alex Goodman <[email protected]>

* allow for python requirements metadata to be optional

Signed-off-by: Alex Goodman <[email protected]>

* restore cyclonedx dependency

Signed-off-by: Alex Goodman <[email protected]>

---------

Signed-off-by: manifestori <[email protected]>
Signed-off-by: Alex Goodman <[email protected]>
Signed-off-by: Alex Goodman <[email protected]>
Co-authored-by: manifestori <[email protected]>

README.md

@@ -577,6 +577,13 @@ linux-kernel:
    # SYFT_LINUX_KERNEL_CATALOG_MODULES env var
    catalog-modules: true
 
+python:
+   # when running across entries in requirements.txt that do not specify a specific version 
+   # (e.g. "sqlalchemy >= 1.0.0, <= 2.0.0, != 3.0.0, <= 3.0.0"), attempt to guess what the version could
+   # be based on the version requirements specified (e.g. "1.0.0"). When enabled the lowest expressible version 
+   # when given an arbitrary constraint will be used (even if that version may not be available/published).
+   guess-unpinned-requirements: false
+
 # cataloging file contents is exposed through the power-user subcommand
 file-contents:
   cataloger:

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/anchore/clio v0.0.0-20230602170917-e747e60c4aa0
 	github.com/anchore/go-logger v0.0.0-20230531193951-db5ae83e7dbe
 	github.com/anchore/stereoscope v0.0.0-20230724160817-d515761c6ca2
+	github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46
 	github.com/charmbracelet/bubbletea v0.24.2
 	github.com/charmbracelet/lipgloss v0.7.1
 	github.com/dave/jennifer v1.6.1
@@ -90,6 +91,7 @@ require (
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/anchore/fangs v0.0.0-20230531202914-48a718c6b4ba // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
+	github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/becheran/wildmatch-go v1.0.0 // indirect
 	github.com/charmbracelet/bubbles v0.16.1 // indirect

go.sum

@@ -114,6 +114,10 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 h1:vmXNl+HDfqqXgr0uY1UgK1GAhps8nbAAtqHNBcgyf+4=
+github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46/go.mod h1:olhPNdiiAAMiSujemd1O/sc6GcyePr23f/6uGKtthNg=
+github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 h1:rcEG5HI490FF0a7zuvxOxen52ddygCfNVjP0XOCMl+M=
+github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492/go.mod h1:9Beu8XsUNNfzml7WBf3QmyPToP1wm1Gj/Vc5UJKqTzU=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
@@ -171,6 +175,7 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -585,6 +590,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saferwall/pe v1.4.4 h1:Ml++7/2/Z1iKwV4zCsd1nIqTEAdUQKAetwbbcCarhOg=
@@ -602,6 +608,7 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -675,6 +682,7 @@ github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/vbatts/go-mtree v0.5.3 h1:S/jYlfG8rZ+a0bhZd+RANXejy7M4Js8fq9U+XoWTd5w=
 github.com/vbatts/go-mtree v0.5.3/go.mod h1:eXsdoPMdL2jcJx6HweWi9lYQxBsTp4lNhqqAjgkZUg8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=

internal/config/application.go

@@ -20,6 +20,7 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger"
 	golangCataloger "github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
+	pythonCataloger "github.com/anchore/syft/syft/pkg/cataloger/python"
 )
 
 var (
@@ -52,6 +53,7 @@ type Application struct {
 	Package                pkg                `yaml:"package" json:"package" mapstructure:"package"`
 	Golang                 golang             `yaml:"golang" json:"golang" mapstructure:"golang"`
 	LinuxKernel            linuxKernel        `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"`
+	Python                 python             `yaml:"python" json:"python" mapstructure:"python"`
 	Attest                 attest             `yaml:"attest" json:"attest" mapstructure:"attest"`
 	FileMetadata           FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification     fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
@@ -85,6 +87,9 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 		LinuxKernel: kernel.LinuxCatalogerConfig{
 			CatalogModules: cfg.LinuxKernel.CatalogModules,
 		},
+		Python: pythonCataloger.CatalogerConfig{
+			GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
+		},
 	}
 }
 

internal/config/python.go

@@ -0,0 +1,13 @@
+package config
+
+import (
+	"github.com/spf13/viper"
+)
+
+type python struct {
+	GuessUnpinnedRequirements bool `json:"guess-unpinned-requirements" yaml:"guess-unpinned-requirements" mapstructure:"guess-unpinned-requirements"`
+}
+
+func (cfg python) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("python.guess-unpinned-requirements", false)
+}

schema/json/schema-10.0.0.json

@@ -1626,10 +1626,7 @@
       "type": "object",
       "required": [
         "name",
-        "extras",
-        "versionConstraint",
-        "url",
-        "markers"
+        "versionConstraint"
       ]
     },
     "RDescriptionFileMetadata": {

syft/pkg/cataloger/cataloger.go

@@ -46,7 +46,7 @@ func ImageCatalogers(cfg Config) []pkg.Cataloger {
 		binary.NewCataloger(),
 		deb.NewDpkgdbCataloger(),
 		dotnet.NewDotnetPortableExecutableCataloger(),
-		golang.NewGoModuleBinaryCataloger(cfg.Go()),
+		golang.NewGoModuleBinaryCataloger(cfg.Golang),
 		java.NewJavaCataloger(cfg.Java()),
 		java.NewNativeImageCataloger(),
 		javascript.NewPackageCataloger(),
@@ -74,8 +74,8 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
 		dotnet.NewDotnetPortableExecutableCataloger(),
 		elixir.NewMixLockCataloger(),
 		erlang.NewRebarLockCataloger(),
-		golang.NewGoModFileCataloger(cfg.Go()),
-		golang.NewGoModuleBinaryCataloger(cfg.Go()),
+		golang.NewGoModFileCataloger(cfg.Golang),
+		golang.NewGoModuleBinaryCataloger(cfg.Golang),
 		haskell.NewHackageCataloger(),
 		java.NewJavaCataloger(cfg.Java()),
 		java.NewJavaGradleLockfileCataloger(),
@@ -85,7 +85,7 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
 		nix.NewStoreCataloger(),
 		php.NewComposerLockCataloger(),
 		portage.NewPortageCataloger(),
-		python.NewPythonIndexCataloger(),
+		python.NewPythonIndexCataloger(cfg.Python),
 		python.NewPythonPackageCataloger(),
 		rpm.NewFileCataloger(),
 		rpm.NewRpmDBCataloger(),
@@ -110,21 +110,21 @@ func AllCatalogers(cfg Config) []pkg.Cataloger {
 		dotnet.NewDotnetPortableExecutableCataloger(),
 		elixir.NewMixLockCataloger(),
 		erlang.NewRebarLockCataloger(),
-		golang.NewGoModFileCataloger(cfg.Go()),
-		golang.NewGoModuleBinaryCataloger(cfg.Go()),
+		golang.NewGoModFileCataloger(cfg.Golang),
+		golang.NewGoModuleBinaryCataloger(cfg.Golang),
 		haskell.NewHackageCataloger(),
 		java.NewJavaCataloger(cfg.Java()),
 		java.NewJavaGradleLockfileCataloger(),
 		java.NewJavaPomCataloger(),
 		java.NewNativeImageCataloger(),
 		javascript.NewLockCataloger(),
 		javascript.NewPackageCataloger(),
-		kernel.NewLinuxKernelCataloger(cfg.Kernel()),
+		kernel.NewLinuxKernelCataloger(cfg.LinuxKernel),
 		nix.NewStoreCataloger(),
 		php.NewComposerInstalledCataloger(),
 		php.NewComposerLockCataloger(),
 		portage.NewPortageCataloger(),
-		python.NewPythonIndexCataloger(),
+		python.NewPythonIndexCataloger(cfg.Python),
 		python.NewPythonPackageCataloger(),
 		r.NewPackageCataloger(),
 		rpm.NewFileCataloger(),

syft/pkg/cataloger/config.go

@@ -4,6 +4,7 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
 	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
+	"github.com/anchore/syft/syft/pkg/cataloger/python"
 )
 
 // TODO: these field naming vs helper function naming schemes are inconsistent.
@@ -12,6 +13,7 @@ type Config struct {
 	Search      SearchConfig
 	Golang      golang.GoCatalogerOpts
 	LinuxKernel kernel.LinuxCatalogerConfig
+	Python      python.CatalogerConfig
 	Catalogers  []string
 	Parallelism int
 }
@@ -21,6 +23,7 @@ func DefaultConfig() Config {
 		Search:      DefaultSearchConfig(),
 		Parallelism: 1,
 		LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
+		Python:      python.DefaultCatalogerConfig(),
 	}
 }
 
@@ -30,11 +33,3 @@ func (c Config) Java() java.Config {
 		SearchIndexedArchives:   c.Search.IncludeIndexedArchives,
 	}
 }
-
-func (c Config) Go() golang.GoCatalogerOpts {
-	return c.Golang
-}
-
-func (c Config) Kernel() kernel.LinuxCatalogerConfig {
-	return c.LinuxKernel
-}

syft/pkg/cataloger/python/cataloger.go

@@ -6,10 +6,21 @@ import (
 
 const eggInfoGlob = "**/*.egg-info"
 
+type CatalogerConfig struct {
+	GuessUnpinnedRequirements bool
+}
+
+func DefaultCatalogerConfig() CatalogerConfig {
+	return CatalogerConfig{
+		GuessUnpinnedRequirements: false,
+	}
+}
+
 // NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
-func NewPythonIndexCataloger() *generic.Cataloger {
+func NewPythonIndexCataloger(cfg CatalogerConfig) *generic.Cataloger {
+	rqp := newRequirementsParser(cfg)
 	return generic.NewCataloger("python-index-cataloger").
-		WithParserByGlobs(parseRequirementsTxt, "**/*requirements*.txt").
+		WithParserByGlobs(rqp.parseRequirementsTxt, "**/*requirements*.txt").
 		WithParserByGlobs(parsePoetryLock, "**/poetry.lock").
 		WithParserByGlobs(parsePipfileLock, "**/Pipfile.lock").
 		WithParserByGlobs(parseSetup, "**/setup.py")

syft/pkg/cataloger/python/cataloger_test.go

@@ -263,7 +263,7 @@ func Test_IndexCataloger_Globs(t *testing.T) {
 			pkgtest.NewCatalogTester().
 				FromDirectory(t, test.fixture).
 				ExpectsResolverContentQueries(test.expected).
-				TestCataloger(t, NewPythonIndexCataloger())
+				TestCataloger(t, NewPythonIndexCataloger(DefaultCatalogerConfig()))
 		})
 	}
 }