final changes before the v4.0 release

cmd/amass/enum.go

@@ -59,21 +59,17 @@ type enumArgs struct {
 	Trusted           *stringset.Set
 	Timeout           int
 	Options           struct {
-		Active          bool
-		Alterations     bool
-		BruteForcing    bool
-		DemoMode        bool
-		IPs             bool
-		IPv4            bool
-		IPv6            bool
-		ListSources     bool
-		NoAlts          bool
-		NoColor         bool
-		NoLocalDatabase bool
-		NoRecursive     bool
-		Passive         bool
-		Silent          bool
-		Verbose         bool
+		Active       bool
+		Alterations  bool
+		BruteForcing bool
+		DemoMode     bool
+		ListSources  bool
+		NoAlts       bool
+		NoColor      bool
+		NoRecursive  bool
+		Passive      bool
+		Silent       bool
+		Verbose      bool
 	}
 	Filepaths struct {
 		AllFilePrefix    string
@@ -119,21 +115,14 @@ func defineEnumArgumentFlags(enumFlags *flag.FlagSet, args *enumArgs) {
 }
 
 func defineEnumOptionFlags(enumFlags *flag.FlagSet, args *enumArgs) {
-	var placeholder bool
 	enumFlags.BoolVar(&args.Options.Active, "active", false, "Attempt zone transfers and certificate name grabs")
 	enumFlags.BoolVar(&args.Options.BruteForcing, "brute", false, "Execute brute forcing after searches")
 	enumFlags.BoolVar(&args.Options.DemoMode, "demo", false, "Censor output to make it suitable for demonstrations")
-	enumFlags.BoolVar(&args.Options.IPs, "ip", false, "Show the IP addresses for discovered names")
-	enumFlags.BoolVar(&args.Options.IPv4, "ipv4", false, "Show the IPv4 addresses for discovered names")
-	enumFlags.BoolVar(&args.Options.IPv6, "ipv6", false, "Show the IPv6 addresses for discovered names")
 	enumFlags.BoolVar(&args.Options.ListSources, "list", false, "Print the names of all available data sources")
 	enumFlags.BoolVar(&args.Options.Alterations, "alts", false, "Enable generation of altered names")
-	enumFlags.BoolVar(&args.Options.NoAlts, "noalts", true, "Deprecated flag to be removed in version 4.0")
 	enumFlags.BoolVar(&args.Options.NoColor, "nocolor", false, "Disable colorized output")
-	enumFlags.BoolVar(&placeholder, "nolocaldb", false, "Deprecated feature to be removed in version 4.0")
 	enumFlags.BoolVar(&args.Options.NoRecursive, "norecursive", false, "Turn off recursive brute forcing")
 	enumFlags.BoolVar(&args.Options.Passive, "passive", false, "Disable DNS resolution of names and dependent features")
-	enumFlags.BoolVar(&placeholder, "share", false, "Deprecated feature to be removed in version 4.0")
 	enumFlags.BoolVar(&args.Options.Silent, "silent", false, "Disable all output during execution")
 	enumFlags.BoolVar(&args.Options.Verbose, "v", false, "Output status / debug / troubleshooting info")
 }
@@ -148,7 +137,6 @@ func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) {
 	enumFlags.Var(&args.Filepaths.Domains, "df", "Path to a file providing root domain names")
 	enumFlags.StringVar(&args.Filepaths.ExcludedSrcs, "ef", "", "Path to a file providing data sources to exclude")
 	enumFlags.StringVar(&args.Filepaths.IncludedSrcs, "if", "", "Path to a file providing data sources to include")
-	enumFlags.StringVar(&args.Filepaths.JSONOutput, "json", "", "Path to the JSON output file")
 	enumFlags.StringVar(&args.Filepaths.LogFile, "log", "", "Path to the log file where errors will be written")
 	enumFlags.Var(&args.Filepaths.Names, "nf", "Path to a file providing already known subdomain names (from other tools/sources)")
 	enumFlags.Var(&args.Filepaths.Resolvers, "rf", "Path to a file providing untrusted DNS resolvers")
@@ -346,10 +334,6 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) {
 		return nil, &args
 	}
 	// Some input validation
-	if cfg.Passive && (args.Options.IPs || args.Options.IPv4 || args.Options.IPv6) {
-		r.Fprintln(color.Error, "IP addresses cannot be provided without DNS resolution")
-		os.Exit(1)
-	}
 	if !cfg.Active && len(args.Ports) > 0 {
 		r.Fprintln(color.Error, "Ports can only be scanned in the active mode")
 		os.Exit(1)
@@ -367,7 +351,6 @@ func printOutput(e *enum.Enumeration, args *enumArgs, output chan string, wg *sy
 	var total int
 	// Print all the output returned by the enumeration
 	for out := range output {
-		//fmt.Fprintf(color.Output, "%s%s\n", green(name), yellow(ips))
 		fmt.Fprintf(color.Output, "%s\n", out)
 		total++
 	}

cmd/amass/io.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"strconv"
 	"time"
@@ -49,7 +50,7 @@ func NewOutput(ctx context.Context, g *netmap.Graph, e *enum.Enumeration, filter
 				if to, err := g.DB.FindById(rel.ToAsset.ID, e.Config.CollectionStartTime.UTC()); err == nil {
 					tostr := extractAssetName(to, e.Config.CollectionStartTime.UTC())
 
-					output = append(output, fromstr+yellow(" --> ")+magenta(rel.Type)+yellow(" --> ")+tostr)
+					output = append(output, fmt.Sprintf("%s %s %s %s %s", fromstr, "-->", magenta(rel.Type), "-->", tostr))
 					filter.Insert(lineid)
 				}
 			}

datasrcs/scripting/dns_test.go

@@ -47,7 +47,7 @@ func TestResolve(t *testing.T) {
 	sys.Config().AddDomain("owasp.org")
 	script.Input() <- &requests.DNSRequest{Domain: "owasp.org"}
 
-	timer := time.NewTimer(15 * time.Second)
+	timer := time.NewTimer(2 * time.Minute)
 	defer timer.Stop()
 loop:
 	for _, name := range expected {

datasrcs/scripting/http.go

@@ -189,15 +189,6 @@ func (s *Script) scrape(L *lua.LState) int {
 }
 
 func (s *Script) req(ctx context.Context, url, data string, hdr http.Header, auth *http.BasicAuth) (*http.Response, error) {
-	cfg := s.sys.Config()
-	// Check for cached responses first
-	dsc := cfg.GetDataSourceConfig(s.String())
-	if dsc != nil && dsc.TTL > 0 {
-		if r, err := s.getCachedResponse(ctx, url+data, dsc.TTL); err == nil {
-			return r, nil
-		}
-	}
-
 	method := "GET"
 	if data != "" {
 		method = "POST"
@@ -215,11 +206,11 @@ func (s *Script) req(ctx context.Context, url, data string, hdr http.Header, aut
 		Auth:   auth,
 	})
 	if err != nil {
+		cfg := s.sys.Config()
+
 		if cfg.Verbose {
 			cfg.Log.Printf("%s: %s: %v", s.String(), url, err)
 		}
-	} else if dsc != nil && dsc.TTL > 0 && resp != nil && resp.StatusCode >= 200 && resp.StatusCode < 400 {
-		_ = s.setCachedResponse(ctx, url+data, resp)
 	}
 	return resp, err
 }

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/geziyor/geziyor v0.0.0-20230315135110-a242b58aaa65
 	github.com/miekg/dns v1.1.55
 	github.com/owasp-amass/asset-db v0.3.0
-	github.com/owasp-amass/config v0.1.0
+	github.com/owasp-amass/config v0.1.1
 	github.com/owasp-amass/open-asset-model v0.1.0-alpha.0.20230622033932-2f80ca929692
 	github.com/owasp-amass/resolve v0.6.20
 	github.com/stretchr/testify v1.8.2
@@ -29,10 +29,11 @@ require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chromedp/cdproto v0.0.0-20230319112347-6603f2c23d36 // indirect
+	github.com/chromedp/cdproto v0.0.0-20230716001748-3ed7c525ec8b // indirect
 	github.com/chromedp/chromedp v0.9.1 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -46,34 +47,34 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/gobwas/ws v1.2.1 // indirect
 	github.com/golang/glog v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.4.1 // indirect
+	github.com/jackc/pgx/v5 v5.4.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/owasp-amass/amass/v3 v3.23.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rubenv/sql-migrate v1.5.1 // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
-	go.uber.org/ratelimit v0.2.0 // indirect
+	go.uber.org/ratelimit v0.3.0 // indirect
 	golang.org/x/crypto v0.11.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
@@ -87,8 +88,8 @@ require (
 	gorm.io/driver/postgres v1.5.2 // indirect
 	gorm.io/driver/sqlite v1.5.2 // indirect
 	gorm.io/gorm v1.25.2 // indirect
-	modernc.org/libc v1.22.5 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/sqlite v1.23.1 // indirect
+	modernc.org/libc v1.24.1 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.6.0 // indirect
+	modernc.org/sqlite v1.24.0 // indirect
 )