From ad0427ed91caf1698b0afb38799b6d96ace0be3c Mon Sep 17 00:00:00 2001 From: caffix Date: Sat, 14 Sep 2024 15:36:30 -0400 Subject: [PATCH] updated to OAM v0.7.0 --- cmd/amass/emails.go | 13 ++- cmd/amass/main.go | 85 +++++++++++++++-- cmd/amass/subs.go | 226 ++++++++++++++++++++++++++++++++++++++++---- cmd/amass/track.go | 6 +- go.mod | 30 +++--- go.sum | 42 ++++---- viz/viz.go | 116 ++++++----------------- viz/viz_test.go | 69 -------------- 8 files changed, 363 insertions(+), 224 deletions(-) delete mode 100644 viz/viz_test.go diff --git a/cmd/amass/emails.go b/cmd/amass/emails.go index 8fc87dcbc..bb9037af2 100644 --- a/cmd/amass/emails.go +++ b/cmd/amass/emails.go @@ -1,4 +1,4 @@ -// Copyright © by Jeff Foley 2024. All rights reserved. +// Copyright © by Jeff Foley 2017-2024. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. // SPDX-License-Identifier: Apache-2.0 @@ -6,7 +6,6 @@ package main import ( "bytes" - "context" "flag" "io" "os" @@ -15,8 +14,8 @@ import ( "github.com/caffix/stringset" "github.com/fatih/color" + assetdb "github.com/owasp-amass/asset-db" "github.com/owasp-amass/config/config" - "github.com/owasp-amass/engine/graph" oam "github.com/owasp-amass/open-asset-model" "github.com/owasp-amass/open-asset-model/contact" "github.com/owasp-amass/open-asset-model/domain" @@ -119,7 +118,7 @@ func runEmailsCommand(clArgs []string) { showEmails(&args, db) } -func showEmails(args *emailsArgs, db *graph.Graph) { +func showEmails(args *emailsArgs, db *assetdb.AssetDB) { var err error var outfile *os.File domains := args.Domains.Slice() @@ -138,7 +137,7 @@ func showEmails(args *emailsArgs, db *graph.Graph) { _, _ = outfile.Seek(0, 0) } - addrs := getAddresses(context.Background(), domains, db) + addrs := getAddresses(db, domains) if len(addrs) == 0 { r.Println("No email addresses were discovered") return @@ -149,7 +148,7 @@ func showEmails(args *emailsArgs, db *graph.Graph) { } } -func getAddresses(ctx context.Context, domains []string, g *graph.Graph) []string { +func getAddresses(db *assetdb.AssetDB, domains []string) []string { if len(domains) == 0 { return nil } @@ -163,7 +162,7 @@ func getAddresses(ctx context.Context, domains []string, g *graph.Graph) []strin fqdns = append(fqdns, &domain.FQDN{Name: d}) } - assets, err := g.DB.FindByScope(fqdns, qtime) + assets, err := db.FindByScope(fqdns, qtime) if err != nil { return nil } diff --git a/cmd/amass/main.go b/cmd/amass/main.go index 2e5a75e50..729363c9e 100644 --- a/cmd/amass/main.go +++ b/cmd/amass/main.go @@ -25,10 +25,12 @@ package main import ( "bufio" "bytes" + "embed" "flag" "fmt" "io" "log/slog" + "math/rand" "net" "net/netip" "os" @@ -39,14 +41,22 @@ import ( "github.com/caffix/stringset" "github.com/fatih/color" + "github.com/glebarez/sqlite" "github.com/owasp-amass/amass/v4/format" + assetdb "github.com/owasp-amass/asset-db" + db "github.com/owasp-amass/asset-db" + pgmigrations "github.com/owasp-amass/asset-db/migrations/postgres" + sqlitemigrations "github.com/owasp-amass/asset-db/migrations/sqlite3" + "github.com/owasp-amass/asset-db/repository" "github.com/owasp-amass/config/config" - "github.com/owasp-amass/engine/graph" et "github.com/owasp-amass/engine/types" "github.com/owasp-amass/open-asset-model/domain" oamnet "github.com/owasp-amass/open-asset-model/network" + migrate "github.com/rubenv/sql-migrate" slogcommon "github.com/samber/slog-common" slogsyslog "github.com/samber/slog-syslog/v2" + "gorm.io/driver/postgres" + "gorm.io/gorm" ) const ( @@ -153,29 +163,88 @@ func createOutputDirectory(cfg *config.Config) { } } -func openGraphDatabase(cfg *config.Config) *graph.Graph { +func openGraphDatabase(cfg *config.Config) *assetdb.AssetDB { // Add the local database settings to the configuration cfg.GraphDBs = append(cfg.GraphDBs, cfg.LocalDatabaseSettings(cfg.GraphDBs)) for _, db := range cfg.GraphDBs { if db.Primary { - var g *graph.Graph + var dbase *assetdb.AssetDB if db.System == "local" { - g = graph.NewGraph(db.System, filepath.Join(config.OutputDirectory(cfg.Dir), "amass.sqlite"), db.Options) + dbase = NewGraph(db.System, filepath.Join(config.OutputDirectory(cfg.Dir), "amass.sqlite"), db.Options) } else { connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s", db.Host, db.Port, db.Username, db.Password, db.DBName) - g = graph.NewGraph(db.System, connStr, db.Options) + dbase = NewGraph(db.System, connStr, db.Options) } - if g != nil { - return g + if dbase != nil { + return dbase } break } } - return graph.NewGraph("memory", "", "") + return NewGraph("memory", "", "") +} + +func NewGraph(system, path string, options string) *assetdb.AssetDB { + var dsn string + var dbtype repository.DBType + + switch system { + case "memory": + dbtype = repository.SQLite + dsn = fmt.Sprintf("file:sqlite%d?mode=memory&cache=shared", rand.Int31n(100)) + case "local": + dbtype = repository.SQLite + dsn = path + case "postgres": + dbtype = repository.Postgres + dsn = path + default: + return nil + } + + store := db.New(dbtype, dsn) + if store == nil { + return nil + } + + var name string + var fs embed.FS + var database gorm.Dialector + switch dbtype { + case repository.SQLite: + name = "sqlite3" + fs = sqlitemigrations.Migrations() + database = sqlite.Open(dsn) + case repository.Postgres: + name = "postgres" + fs = pgmigrations.Migrations() + database = postgres.Open(dsn) + } + + sql, err := gorm.Open(database, &gorm.Config{}) + if err != nil { + return nil + } + + migrationsSource := migrate.EmbedFileSystemMigrationSource{ + FileSystem: fs, + Root: "/", + } + + sqlDb, err := sql.DB() + if err != nil { + panic(err) + } + + _, err = migrate.Exec(sqlDb, name, migrationsSource, migrate.Up) + if err != nil { + panic(err) + } + return store } func getWordList(reader io.Reader) ([]string, error) { diff --git a/cmd/amass/subs.go b/cmd/amass/subs.go index 50548caf1..430ca2aba 100644 --- a/cmd/amass/subs.go +++ b/cmd/amass/subs.go @@ -1,4 +1,4 @@ -// Copyright © by Jeff Foley 2024. All rights reserved. +// Copyright © by Jeff Foley 2017-2024. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. // SPDX-License-Identifier: Apache-2.0 @@ -6,11 +6,12 @@ package main import ( "bytes" - "context" + "errors" "flag" "fmt" "io" "net" + "net/netip" "os" "path" "strings" @@ -18,12 +19,12 @@ import ( "github.com/caffix/stringset" "github.com/fatih/color" + assetdb "github.com/owasp-amass/asset-db" "github.com/owasp-amass/config/config" - "github.com/owasp-amass/engine/graph" oam "github.com/owasp-amass/open-asset-model" "github.com/owasp-amass/open-asset-model/domain" "github.com/owasp-amass/open-asset-model/network" - "github.com/owasp-amass/open-asset-model/whois" + oamreg "github.com/owasp-amass/open-asset-model/registration" ) const subsUsageMsg = "subs [options] -d domain" @@ -155,7 +156,7 @@ func runSubsCommand(clArgs []string) { showData(&args, asninfo, db) } -func showData(args *subsArgs, asninfo bool, db *graph.Graph) { +func showData(args *subsArgs, asninfo bool, db *assetdb.AssetDB) { var total int var err error var outfile *os.File @@ -184,9 +185,9 @@ func showData(args *subsArgs, asninfo bool, db *graph.Graph) { } } - names := getNames(context.Background(), domains, asninfo, db) + names := getNames(db, domains, asninfo) if len(names) != 0 && (asninfo || args.Options.IPv4 || args.Options.IPv6) { - names = addAddresses(context.Background(), db, names, asninfo, cache) + names = addAddresses(db, names, asninfo, cache) } asns := make(map[int]*ASNSummaryData) @@ -245,7 +246,7 @@ func showData(args *subsArgs, asninfo bool, db *graph.Graph) { } } -func getNames(ctx context.Context, domains []string, asninfo bool, g *graph.Graph) []*Output { +func getNames(db *assetdb.AssetDB, domains []string, asninfo bool) []*Output { if len(domains) == 0 { return nil } @@ -259,7 +260,7 @@ func getNames(ctx context.Context, domains []string, asninfo bool, g *graph.Grap fqdns = append(fqdns, &domain.FQDN{Name: d}) } - assets, err := g.DB.FindByScope(fqdns, qtime) + assets, err := db.FindByScope(fqdns, qtime) if err != nil { return nil } @@ -274,7 +275,7 @@ func getNames(ctx context.Context, domains []string, asninfo bool, g *graph.Grap return names } -func addAddresses(ctx context.Context, g *graph.Graph, names []*Output, asninfo bool, cache *ASNCache) []*Output { +func addAddresses(db *assetdb.AssetDB, names []*Output, asninfo bool, cache *ASNCache) []*Output { var namestrs []string lookup := make(outLookup, len(names)) for _, n := range names { @@ -283,7 +284,7 @@ func addAddresses(ctx context.Context, g *graph.Graph, names []*Output, asninfo } qtime := time.Time{} - if pairs, err := g.NamesToAddrs(ctx, qtime, namestrs...); err == nil { + if pairs, err := namesToAddrs(db, qtime, namestrs...); err == nil { for _, p := range pairs { addr := p.Addr.Address.String() @@ -354,9 +355,9 @@ func addInfrastructureInfo(lookup outLookup, cache *ASNCache) []*Output { return output } -func fillCache(cache *ASNCache, db *graph.Graph) error { +func fillCache(cache *ASNCache, db *assetdb.AssetDB) error { start := time.Now().Add(-730 * time.Hour) - assets, err := db.DB.FindByType(oam.AutonomousSystem, start) + assets, err := db.FindByType(oam.AutonomousSystem, start) if err != nil { return err } @@ -368,14 +369,14 @@ func fillCache(cache *ASNCache, db *graph.Graph) error { } var desc string - rels, err := db.DB.OutgoingRelations(a, start, "registration") + rels, err := db.OutgoingRelations(a, start, "registration") if err != nil || len(rels) == 0 { continue } for _, rel := range rels { - if asset, err := db.DB.FindById(rel.ToAsset.ID, start); err == nil && asset != nil { - if autnum, ok := asset.Asset.(*whois.AutnumRecord); ok && autnum != nil { + if asset, err := db.FindById(rel.ToAsset.ID, start); err == nil && asset != nil { + if autnum, ok := asset.Asset.(*oamreg.AutnumRecord); ok && autnum != nil { desc = autnum.Handle + " - " + autnum.Name break } @@ -385,7 +386,7 @@ func fillCache(cache *ASNCache, db *graph.Graph) error { continue } - for _, prefix := range db.ReadASPrefixes(context.Background(), as.Number, start) { + for _, prefix := range readASPrefixes(db, as.Number, start) { first, cidr, err := net.ParseCIDR(prefix) if err != nil { continue @@ -404,3 +405,194 @@ func fillCache(cache *ASNCache, db *graph.Graph) error { } return nil } + +func readASPrefixes(db *assetdb.AssetDB, asn int, since time.Time) []string { + var prefixes []string + + assets, err := db.FindByContent(&network.AutonomousSystem{Number: asn}, since) + if err != nil || len(assets) == 0 { + return prefixes + } + + if rels, err := db.OutgoingRelations(assets[0], since, "announces"); err == nil && len(rels) > 0 { + for _, rel := range rels { + if a, err := db.FindById(rel.ToAsset.ID, since); err != nil { + continue + } else if netblock, ok := a.Asset.(*network.Netblock); ok { + prefixes = append(prefixes, netblock.Cidr.String()) + } + } + } + return prefixes +} + +type NameAddrPair struct { + FQDN *domain.FQDN + Addr *network.IPAddress +} + +func namesToAddrs(db *assetdb.AssetDB, since time.Time, names ...string) ([]*NameAddrPair, error) { + nameAddrMap := make(map[string]*stringset.Set, len(names)) + defer func() { + for _, ss := range nameAddrMap { + ss.Close() + } + }() + + remaining := stringset.New() + defer remaining.Close() + remaining.InsertMany(names...) + + // get the IPs associated with SRV, NS, and MX records + sel := "SELECT srvs.content->>'name' AS name,ips.content->>'address' AS addr " + from := "FROM ((((assets AS fqdns INNER JOIN relations AS r1 ON fqdns.id = r1.from_asset_id) " + from2 := "INNER JOIN assets AS srvs ON r1.to_asset_id = srvs.id) INNER JOIN relations AS r2 ON srvs.id =" + from3 := " r2.from_asset_id) INNER JOIN assets AS ips ON r2.to_asset_id = ips.id)" + where := " WHERE fqdns.type = 'FQDN' AND srvs.type = 'FQDN' AND ips.type = 'IPAddress'" + where2 := " AND r1.type IN ('srv_record','ns_record','mx_record') AND r2.type IN ('a_record','aaaa_record')" + query := sel + from + from2 + from3 + where + where2 + if !since.IsZero() { + query += " AND r1.last_seen > '" + since.Format("2006-01-02 15:04:05") + + "' AND r2.last_seen > '" + since.Format("2006-01-02 15:04:05") + "'" + } + query += " AND fqdns.content->>'name' in ('" + strings.Join(remaining.Slice(), "','") + "')" + + var results []struct { + Name string `gorm:"column:name"` + Addr string `gorm:"column:addr"` + } + + if err := db.RawQuery(query, &results); err == nil && len(results) > 0 { + for _, res := range results { + if !remaining.Has(res.Name) { + continue + } + remaining.Remove(res.Name) + if _, found := nameAddrMap[res.Name]; !found { + nameAddrMap[res.Name] = stringset.New() + } + nameAddrMap[res.Name].Insert(res.Addr) + } + } + + if remaining.Len() == 0 { + return generatePairsFromAddrMap(nameAddrMap) + } + + from = "(relations inner join assets on relations.from_asset_id = assets.id)" + where = " where assets.type = 'FQDN' and relations.type in ('a_record','aaaa_record')" + likeset := " and assets.content->>'name' in ('" + strings.Join(remaining.Slice(), "','") + "')" + query = from + where + likeset + if !since.IsZero() { + query += " and relations.last_seen > '" + since.Format("2006-01-02 15:04:05") + "'" + } + + rels, err := db.RelationQuery(query) + if err != nil { + return nil, err + } + for _, rel := range rels { + if f, ok := rel.FromAsset.Asset.(*domain.FQDN); ok { + if _, found := nameAddrMap[f.Name]; !found { + nameAddrMap[f.Name] = stringset.New() + } + if a, ok := rel.ToAsset.Asset.(*network.IPAddress); ok { + nameAddrMap[f.Name].Insert(a.Address.String()) + remaining.Remove(f.Name) + } + } + } + + if remaining.Len() == 0 { + return generatePairsFromAddrMap(nameAddrMap) + } + + // get the FQDNs that have CNAME records + from = "(assets inner join relations on assets.id = relations.from_asset_id)" + where = " where assets.type = 'FQDN' and relations.type = 'cname_record'" + likeset = " and assets.content->>'name' in ('" + strings.Join(remaining.Slice(), "','") + "')" + query = from + where + likeset + if !since.IsZero() { + query += " and relations.last_seen > '" + since.Format("2006-01-02 15:04:05") + "'" + } + + assets, err := db.AssetQuery(query) + if err != nil { + return nil, err + } + + var cnames []string + for _, a := range assets { + if f, ok := a.Asset.(*domain.FQDN); ok { + cnames = append(cnames, f.Name) + } + } + + // get to the end of the CNAME alias chains + for _, name := range cnames { + var results []struct { + Name string `gorm:"column:name"` + Addr string `gorm:"column:addr"` + } + + if err := db.RawQuery(cnameQuery(name, since), &results); err == nil && len(results) > 0 { + remaining.Remove(name) + + for _, res := range results { + if _, found := nameAddrMap[name]; !found { + nameAddrMap[name] = stringset.New() + } + nameAddrMap[name].Insert(res.Addr) + } + } + } + + return generatePairsFromAddrMap(nameAddrMap) +} + +func cnameQuery(name string, since time.Time) string { + query := "WITH RECURSIVE traverse_cname(fqdn) AS ( VALUES('" + name + "')" + query += " UNION SELECT cnames.content->>'name' FROM ((assets AS fqdns" + query += " INNER JOIN relations ON fqdns.id = relations.from_asset_id)" + query += " INNER JOIN assets AS cnames ON relations.to_asset_id = cnames.id), traverse_cname" + query += " WHERE fqdns.type = 'FQDN' AND cnames.type = 'FQDN'" + if !since.IsZero() { + query += " and relations.last_seen > '" + since.Format("2006-01-02 15:04:05") + "'" + } + query += " AND relations.type = 'cname_record' AND fqdns.content->>'name' = traverse_cname.fqdn)" + query += " SELECT fqdns.content->>'name' AS name, ips.content->>'address' AS addr" + query += " FROM ((assets AS fqdns INNER JOIN relations ON fqdns.id = relations.from_asset_id)" + query += " INNER JOIN assets AS ips ON relations.to_asset_id = ips.id)" + query += " WHERE fqdns.type = 'FQDN' AND ips.type = 'IPAddress'" + if !since.IsZero() { + query += " and relations.last_seen > '" + since.Format("2006-01-02 15:04:05") + "'" + } + query += " AND relations.type IN ('a_record', 'aaaa_record') AND " + return query + "fqdns.content->>'name' IN (SELECT fqdn FROM traverse_cname)" +} + +func generatePairsFromAddrMap(addrMap map[string]*stringset.Set) ([]*NameAddrPair, error) { + var pairs []*NameAddrPair + + for name, set := range addrMap { + for _, addr := range set.Slice() { + if ip, err := netip.ParseAddr(addr); err == nil { + address := &network.IPAddress{Address: ip} + if ip.Is4() { + address.Type = "IPv4" + } else if ip.Is6() { + address.Type = "IPv6" + } + pairs = append(pairs, &NameAddrPair{ + FQDN: &domain.FQDN{Name: name}, + Addr: address, + }) + } + } + } + + if len(pairs) == 0 { + return nil, errors.New("no addresses were discovered") + } + return pairs, nil +} diff --git a/cmd/amass/track.go b/cmd/amass/track.go index 306b2ab77..ee094faaa 100644 --- a/cmd/amass/track.go +++ b/cmd/amass/track.go @@ -14,8 +14,8 @@ import ( "github.com/caffix/stringset" "github.com/fatih/color" + assetdb "github.com/owasp-amass/asset-db" "github.com/owasp-amass/config/config" - "github.com/owasp-amass/engine/graph" oam "github.com/owasp-amass/open-asset-model" "github.com/owasp-amass/open-asset-model/domain" ) @@ -130,7 +130,7 @@ func runTrackCommand(clArgs []string) { } } -func getNewNames(domains []string, since time.Time, g *graph.Graph) []string { +func getNewNames(domains []string, since time.Time, db *assetdb.AssetDB) []string { if len(domains) == 0 { return []string{} } @@ -144,7 +144,7 @@ func getNewNames(domains []string, since time.Time, g *graph.Graph) []string { since = since.UTC() } - assets, err := g.DB.FindByScope(fqdns, since) + assets, err := db.FindByScope(fqdns, since) if err != nil { return []string{} } diff --git a/go.mod b/go.mod index 714724a74..b29ef5a69 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/owasp-amass/amass/v4 -go 1.21.4 +go 1.23.1 require ( github.com/PuerkitoBio/goquery v1.9.2 @@ -8,17 +8,21 @@ require ( github.com/cheggaaa/pb/v3 v3.1.5 github.com/fatih/color v1.17.0 github.com/geziyor/geziyor v0.0.0-20240812061556-229b8ca83ac1 + github.com/glebarez/sqlite v1.11.0 github.com/miekg/dns v1.1.62 - github.com/owasp-amass/asset-db v0.6.0 - github.com/owasp-amass/config v0.6.0 - github.com/owasp-amass/engine v0.0.1 - github.com/owasp-amass/open-asset-model v0.6.0 + github.com/owasp-amass/asset-db v0.7.0 + github.com/owasp-amass/config v0.7.0 + github.com/owasp-amass/engine v0.0.2 + github.com/owasp-amass/open-asset-model v0.7.0 github.com/owasp-amass/resolve v0.8.1 + github.com/rubenv/sql-migrate v1.7.0 github.com/samber/slog-common v0.17.1 github.com/samber/slog-syslog/v2 v2.5.0 github.com/stretchr/testify v1.9.0 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 github.com/yl2chen/cidranger v1.0.2 + gorm.io/driver/postgres v1.5.9 + gorm.io/gorm v1.25.12 ) require ( @@ -40,7 +44,6 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect - github.com/glebarez/sqlite v1.11.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-kit/kit v0.13.0 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect @@ -55,8 +58,8 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -75,13 +78,12 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rubenv/sql-migrate v1.7.0 // indirect github.com/samber/lo v1.47.0 // indirect github.com/temoto/robotstxt v1.1.2 // indirect go.uber.org/ratelimit v0.3.1 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/mod v0.20.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect @@ -89,12 +91,10 @@ require ( golang.org/x/tools v0.24.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/datatypes v1.2.1 // indirect + gorm.io/datatypes v1.2.2 // indirect gorm.io/driver/mysql v1.5.7 // indirect - gorm.io/driver/postgres v1.5.9 // indirect - gorm.io/gorm v1.25.11 // indirect modernc.org/libc v1.60.1 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.32.0 // indirect + modernc.org/sqlite v1.33.1 // indirect ) diff --git a/go.sum b/go.sum index b94669371..7fafb533c 100644 --- a/go.sum +++ b/go.sum @@ -998,10 +998,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -1133,14 +1133,14 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= -github.com/owasp-amass/asset-db v0.6.0 h1:UIaWAtNdtQ2AZZHUQw/4cZN73VquhXKbV2nILiescLQ= -github.com/owasp-amass/asset-db v0.6.0/go.mod h1:MdJQWLhi2DG+/vCQX3QTPZrIr/UP+egYlLeAmBqJJSE= -github.com/owasp-amass/config v0.6.0 h1:XpMvEFAf7MbMz0TfiI92edrpWQaPOuPquQInTNN+ERo= -github.com/owasp-amass/config v0.6.0/go.mod h1:AlPiMXpynSE8cpT3C30zPC/8aBvurxREacCQyrL4KqA= -github.com/owasp-amass/engine v0.0.1 h1:7OoSXRyv8S+f0UJWaq76EVB+DgRTWr0fQbcoJKI/+74= -github.com/owasp-amass/engine v0.0.1/go.mod h1:SU2QpQAlFCcXOjtH33wc8xhqpTkRYkpTksVGfCfjoo0= -github.com/owasp-amass/open-asset-model v0.6.0 h1:mfuUKMbcRhdzC1Li8dt7FfBwsOPKjoLi0tozuzRaB+g= -github.com/owasp-amass/open-asset-model v0.6.0/go.mod h1:DOX+SiD6PZBroSMnsILAmpf0SHi6TVpqjV4uNfBeg7g= +github.com/owasp-amass/asset-db v0.7.0 h1:r/f1ukjxjIx9tqXD4BQLLLVcuaPR8JB8B1SR7jD8nK4= +github.com/owasp-amass/asset-db v0.7.0/go.mod h1:wmhf8J8Afy8jPyZeF7HGQ5XBG+Qswg+Qec0+tajT7lQ= +github.com/owasp-amass/config v0.7.0 h1:1OTa7ifnMczEtG1GmFMfjTIwPMWqc3QhjvIjWt4ZOXo= +github.com/owasp-amass/config v0.7.0/go.mod h1:oUQEGKorjuNPMcMBU2abOxJqjy3NWqR15u0v4upPGDI= +github.com/owasp-amass/engine v0.0.2 h1:+Xc9GsxnVUM65COUGblIYilioYMC8dNpGmCJm5d6tvY= +github.com/owasp-amass/engine v0.0.2/go.mod h1:qQ0uX+c7C1UDDUQOoC7glY0OYNSXG/rezK44e6lGUn8= +github.com/owasp-amass/open-asset-model v0.7.0 h1:1Iv4Jtn4OUgWwSLnFAacwVZohECIBmcKRxjovVehQ9A= +github.com/owasp-amass/open-asset-model v0.7.0/go.mod h1:DOX+SiD6PZBroSMnsILAmpf0SHi6TVpqjV4uNfBeg7g= github.com/owasp-amass/resolve v0.8.1 h1:CM92zgnLj80pIuDdlbABQzu2G3th2RFbbS7QtyFOTfA= github.com/owasp-amass/resolve v0.8.1/go.mod h1:3rT2jEDEzFvKl/bACBpHTfj94M4ait5VmEAbXILJNbs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -1362,8 +1362,9 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1503,8 +1504,9 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2121,8 +2123,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v1.2.1 h1:r+g0bk4LPCW2v4+Ls7aeNgGme7JYdNDQ2VtvlNUfBh0= -gorm.io/datatypes v1.2.1/go.mod h1:hYK6OTb/1x+m96PgoZZq10UXJ6RvEBb9kRDQ2yyhzGs= +gorm.io/datatypes v1.2.2 h1:sdn7ZmG4l7JWtMDUb3L98f2Ym7CO5F8mZLlrQJMfF9g= +gorm.io/datatypes v1.2.2/go.mod h1:f4BsLcFAX67szSv8svwLRjklArSHAvHLeE3pXAS5DZI= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= @@ -2132,8 +2134,8 @@ gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= -gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -2188,8 +2190,8 @@ modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= -modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= -modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= diff --git a/viz/viz.go b/viz/viz.go index 1d755c0f8..705933f21 100644 --- a/viz/viz.go +++ b/viz/viz.go @@ -5,21 +5,15 @@ package viz import ( - "strconv" "strings" "time" + assetdb "github.com/owasp-amass/asset-db" "github.com/owasp-amass/asset-db/types" - "github.com/owasp-amass/engine/graph" oam "github.com/owasp-amass/open-asset-model" "github.com/owasp-amass/open-asset-model/contact" "github.com/owasp-amass/open-asset-model/domain" - "github.com/owasp-amass/open-asset-model/network" - "github.com/owasp-amass/open-asset-model/org" - "github.com/owasp-amass/open-asset-model/people" - oamcert "github.com/owasp-amass/open-asset-model/tls_certificate" - "github.com/owasp-amass/open-asset-model/url" - "github.com/owasp-amass/open-asset-model/whois" + oamreg "github.com/owasp-amass/open-asset-model/registration" ) // Edge represents an Amass graph edge in the viz package. @@ -38,7 +32,7 @@ type Node struct { } // VizData returns the current state of the Graph as viz package Nodes and Edges. -func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) { +func VizData(domains []string, since time.Time, db *assetdb.AssetDB) ([]Node, []Edge) { if len(domains) == 0 { return []Node{}, []Edge{} } @@ -52,7 +46,7 @@ func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) since = since.UTC() } - next, err := g.DB.FindByScope(fqdns, since) + next, err := db.FindByScope(fqdns, since) if err != nil { return []Node{}, []Edge{} } @@ -107,19 +101,21 @@ func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) out = true case oam.SocketAddress: case oam.ContactRecord: - fallthrough + out = true case oam.EmailAddress: - fallthrough + out = true case oam.Location: out = true case oam.Phone: + out = true case oam.Fingerprint: case oam.Organization: out = true case oam.Person: + out = true case oam.TLSCertificate: case oam.URL: - fallthrough + out = true case oam.DomainRecord: out = true case oam.Source: @@ -127,10 +123,10 @@ func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) } // Obtain relations to additional assets in the graph if out { - if rels, err := g.DB.OutgoingRelations(a, since, outRels...); err == nil && len(rels) > 0 { + if rels, err := db.OutgoingRelations(a, since, outRels...); err == nil && len(rels) > 0 { fromID := id for _, rel := range rels { - if to, err := g.DB.FindById(rel.ToAsset.ID, since); err == nil { + if to, err := db.FindById(rel.ToAsset.ID, since); err == nil { toID := idx n2 := newNode(toID, to) if n2 == nil { @@ -157,10 +153,10 @@ func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) } } if in { - if rels, err := g.DB.IncomingRelations(a, since, inRels...); err == nil && len(rels) > 0 { + if rels, err := db.IncomingRelations(a, since, inRels...); err == nil && len(rels) > 0 { toID := id for _, rel := range rels { - if from, err := g.DB.FindById(rel.FromAsset.ID, since); err == nil { + if from, err := db.FindById(rel.FromAsset.ID, since); err == nil { fromID := idx n2 := newNode(fromID, from) if n2 == nil { @@ -194,85 +190,35 @@ func VizData(domains []string, since time.Time, g *graph.Graph) ([]Node, []Edge) } func newNode(idx int, a *types.Asset) *Node { - var name, atype, title string + key := a.Asset.Key() + if key == "" { + return nil + } + + atype := string(a.Asset.AssetType()) + if atype == string(oam.Source) { + return nil + } switch v := a.Asset.(type) { - case *domain.FQDN: - name = v.Name - atype = string(oam.FQDN) - title = atype + ": " + name - case *network.IPAddress: - name = v.Address.String() - atype = string(oam.IPAddress) - title = atype + ": " + name - case *network.AutonomousSystem: - name = strconv.Itoa(v.Number) - atype = string(oam.AutonomousSystem) - title = atype + ": AS" + name - case *whois.AutnumRecord: - name = v.Handle + " - " + v.Name - atype = string(oam.AutnumRecord) - title = atype + ": " + name - case *network.Netblock: - name = v.Cidr.String() - atype = string(oam.Netblock) - title = atype + ": " + name - case *network.SocketAddress: - name = v.Address.String() - atype = string(oam.SocketAddress) - title = atype + ": " + name + case *oamreg.AutnumRecord: + key = v.Handle + " - " + key case *contact.ContactRecord: - name = v.DiscoveredAt - atype = string(oam.ContactRecord) - title = atype + ": " + name - case *contact.EmailAddress: - name = v.Address - atype = string(oam.EmailAddress) - title = atype + ": " + name + key = "Found->" + key case *contact.Location: - name = v.Address - atype = string(oam.Location) - title = atype + ": " + name - case *contact.Phone: - name = v.Raw - atype = string(oam.Phone) - title = atype + ": " + name - /*case *fingerprint.Fingerprint: - name = v.Value - atype = string(oam.Fingerprint) - title = atype + ": " + name*/ - case *org.Organization: - name = v.Name - atype = string(oam.Organization) - title = atype + ": " + name - case *people.Person: - name = v.FullName - atype = string(oam.Person) - title = atype + ": " + name - case *oamcert.TLSCertificate: - name = v.SerialNumber - atype = string(oam.TLSCertificate) - title = atype + ": " + name - case *url.URL: - name = v.Raw - atype = string(oam.URL) - title = atype + ": " + name - case *whois.DomainRecord: - name = v.Domain - atype = string(oam.DomainRecord) - title = atype + ": " + name - /*case *source.Source: - name = v.Name - atype = string(oam.Source) - title = atype + ": " + name*/ + parts := []string{v.BuildingNumber, v.StreetName, v.City, v.Province, v.PostalCode} + key = strings.Join(parts, " ") + case *oamreg.DomainRecord: + key = "WHOIS: " + key default: return nil } + title := atype + ": " + key return &Node{ ID: idx, Type: atype, - Label: name, + Label: key, Title: title, } } diff --git a/viz/viz_test.go b/viz/viz_test.go deleted file mode 100644 index 3829faf51..000000000 --- a/viz/viz_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright © by Jeff Foley 2017-2024. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -// SPDX-License-Identifier: Apache-2.0 - -package viz - -import ( - "context" - "testing" - "time" - - "github.com/owasp-amass/engine/graph" -) - -func TestViz(t *testing.T) { - g := graph.NewGraph("memory", "", "") - - tt := []struct { - fqdn string - addr string - }{ - {fqdn: "dev.example.domain", addr: "127.0.0.1"}, - } - - for _, tc := range tt { - t.Run("Testing VizData...", func(t *testing.T) { - _, err := g.UpsertA(context.Background(), tc.fqdn, tc.addr) - - if err != nil { - t.Errorf("Error inserting A record.\n%v", err) - } - gotNode, gotEdge := VizData([]string{"example.domain"}, time.Time{}, g) - if gotNode == nil { - t.Errorf("Failed to obtain node.\n%v", gotNode) - } - if gotEdge == nil { - t.Errorf("Failed to obtain edge.\n%v", gotEdge) - } - }) - } -} - -func testEdges() []Edge { - return []Edge{ - { - From: 0, - To: 1, - Label: "a_record", - Title: "a_record", - }, - } -} - -func testNodes() []Node { - return []Node{ - { - ID: 0, - Type: "FQDN", - Label: "owasp.org", - Title: "FQDN: owasp.org", - }, - { - ID: 1, - Type: "IPAddress", - Label: "205.251.199.98", - Title: "IPAddress: 205.251.199.98", - }, - } -}