Skip to content

Commit

Permalink
initial changes for client-side logging to support syslog
Browse files Browse the repository at this point in the history
  • Loading branch information
@caffix
caffix committed Feb 28, 2024
1 parent b386cee commit ce3bd8e
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 106 deletions.
188 changes: 82 additions & 106 deletions cmd/amass/enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,15 @@
package main

import (
"bufio"
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"log/slog"
"os"
"os/signal"
"path/filepath"
"regexp"
"strings"
"time"

"github.com/caffix/stringset"
Expand Down Expand Up @@ -147,17 +145,13 @@ func runEnumCommand(clArgs []string) {
}
createOutputDirectory(cfg)

rLog, wLog := io.Pipe()
dir := config.OutputDirectory(cfg.Dir)
// Setup logging so that messages can be written to the file and used by the program
cfg.Log = log.New(wLog, "", log.Lmicroseconds)
logfile := filepath.Join(dir, "amass.log")
// Setup logging
var logfile string
if args.Filepaths.LogFile != "" {
logfile = args.Filepaths.LogFile
}

// Start handling the log messages
go writeLogsAndMessages(rLog, logfile, args.Options.Verbose)
l := selectLogger(dir, logfile)

// Create the client that will provide a connection to the engine
url := "http://localhost:4000/graphql"
Expand Down Expand Up @@ -214,7 +208,7 @@ func runEnumCommand(clArgs []string) {
}
}
case message := <-messages:
cfg.Log.Print(message)
writeLogMessage(l, message)
case <-done:
return
}
Expand Down Expand Up @@ -322,62 +316,78 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) {
return cfg, &args
}

func writeLogsAndMessages(logs *io.PipeReader, logfile string, verbose bool) {
wildcard := regexp.MustCompile("DNS wildcard")
queries := regexp.MustCompile("Querying")

var filePtr *os.File
if logfile != "" {
var err error
func writeLogMessage(l *slog.Logger, logstr string) {
j := make(map[string]interface{})
if err := json.Unmarshal([]byte(logstr), &j); err != nil {
return
}
// check that this is a log message sent over the subscription channel
if p, found := j["payload"]; !found {
return
} else if pmap, ok := p.(map[string]interface{}); !ok {
return
} else if d, found := pmap["data"]; !found {
return
} else if dmap, ok := d.(map[string]interface{}); !ok {
return
} else if lm, found := dmap["logMessages"]; !found {
return
} else if lmmap, ok := lm.(map[string]interface{}); !ok {
return
} else {
// set our json map to the log message within the channel data
j = lmmap
}
delete(j, slog.TimeKey)
delete(j, slog.SourceKey)

filePtr, err = os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
r.Fprintf(color.Error, "Failed to open the log file: %v\n", err)
} else {
defer func() {
_ = filePtr.Sync()
_ = filePtr.Close()
}()
_ = filePtr.Truncate(0)
_, _ = filePtr.Seek(0, 0)
}
var level slog.Level
if val, found := j[slog.LevelKey]; !found {
return
} else if str, ok := val.(string); !ok {
return
} else if level.UnmarshalText([]byte(str)) != nil {
return
}
delete(j, slog.LevelKey)

scanner := bufio.NewScanner(logs)
for scanner.Scan() {
line := scanner.Text()
if err := scanner.Err(); err != nil {
fmt.Fprintf(color.Error, "Error reading the Amass logs: %v\n", err)
break
}
var msg string
if val, found := j[slog.MessageKey]; !found {
return
} else if str, ok := val.(string); !ok {
return
} else {
msg = str
}
delete(j, slog.MessageKey)

if filePtr != nil {
fmt.Fprintln(filePtr, line)
}
// Remove the timestamp
parts := strings.Split(line, " ")
line = strings.Join(parts[1:], " ")
// Check for Amass DNS wildcard messages
if verbose && wildcard.FindString(line) != "" {
fgR.Fprintln(color.Error, line)
}
// Let the user know when data sources are being queried
if verbose && queries.FindString(line) != "" {
fgY.Fprintln(color.Error, line)
}
var pc uintptr
ctx := context.Background()
r := slog.NewRecord(time.Now(), level, msg, pc)
if l.Handler().Enabled(ctx, level) {
l.Handler().Handle(ctx, r)
}
}

// Obtain parameters from provided input files
func processEnumInputFiles(args *enumArgs) error {
if args.Options.BruteForcing {
if len(args.Filepaths.BruteWordlist) > 0 {
for _, f := range args.Filepaths.BruteWordlist {
list, err := config.GetListFromFile(f)
getList := func(fp []string, name string, s *stringset.Set) error {
for _, p := range fp {
if p != "" {
list, err := config.GetListFromFile(p)
if err != nil {
return fmt.Errorf("failed to parse the brute force wordlist file: %v", err)
return fmt.Errorf("failed to parse the %s file: %v", name, err)
}
args.BruteWordList.InsertMany(list...)
s.InsertMany(list...)
}
}
return nil
}

if args.Options.BruteForcing {
if len(args.Filepaths.BruteWordlist) > 0 {
if err := getList(args.Filepaths.BruteWordlist, "brute force wordlist", args.BruteWordList); err != nil {
return err
}
} else {
if f, err := resources.GetResourceFile("namelist.txt"); err == nil {
Expand All @@ -389,12 +399,8 @@ func processEnumInputFiles(args *enumArgs) error {
}
if !args.Options.NoAlts {
if len(args.Filepaths.AltWordlist) > 0 {
for _, f := range args.Filepaths.AltWordlist {
list, err := config.GetListFromFile(f)
if err != nil {
return fmt.Errorf("failed to parse the alterations wordlist file: %v", err)
}
args.AltWordList.InsertMany(list...)
if err := getList(args.Filepaths.AltWordlist, "alterations wordlist", args.AltWordList); err != nil {
return err
}
} else {
if f, err := resources.GetResourceFile("alterations.txt"); err == nil {
Expand All @@ -404,53 +410,23 @@ func processEnumInputFiles(args *enumArgs) error {
}
}
}
if args.Filepaths.Blacklist != "" {
list, err := config.GetListFromFile(args.Filepaths.Blacklist)
if err != nil {
return fmt.Errorf("failed to parse the blacklist file: %v", err)
}
args.Blacklist.InsertMany(list...)
if err := getList([]string{args.Filepaths.Blacklist}, "blacklist", args.Blacklist); err != nil {
return err
}
if args.Filepaths.ExcludedSrcs != "" {
list, err := config.GetListFromFile(args.Filepaths.ExcludedSrcs)
if err != nil {
return fmt.Errorf("failed to parse the exclude file: %v", err)
}
args.Excluded.InsertMany(list...)
if err := getList([]string{args.Filepaths.ExcludedSrcs}, "exclude", args.Excluded); err != nil {
return err
}
if args.Filepaths.IncludedSrcs != "" {
list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs)
if err != nil {
return fmt.Errorf("failed to parse the include file: %v", err)
}
args.Included.InsertMany(list...)
if err := getList([]string{args.Filepaths.IncludedSrcs}, "include", args.Included); err != nil {
return err
}
if len(args.Filepaths.Names) > 0 {
for _, f := range args.Filepaths.Names {
list, err := config.GetListFromFile(f)
if err != nil {
return fmt.Errorf("failed to parse the subdomain names file: %v", err)
}
args.Names.InsertMany(list...)
}
if err := getList(args.Filepaths.Names, "subdomain names", args.Names); err != nil {
return err
}
if len(args.Filepaths.Domains) > 0 {
for _, f := range args.Filepaths.Domains {
list, err := config.GetListFromFile(f)
if err != nil {
return fmt.Errorf("failed to parse the domain names file: %v", err)
}
args.Domains.InsertMany(list...)
}
if err := getList(args.Filepaths.Domains, "domain names", args.Domains); err != nil {
return err
}
if len(args.Filepaths.Resolvers) > 0 {
for _, f := range args.Filepaths.Resolvers {
list, err := config.GetListFromFile(f)
if err != nil {
return fmt.Errorf("failed to parse the esolver file: %v", err)
}
args.Resolvers.InsertMany(list...)
}
if err := getList(args.Filepaths.Resolvers, "resolver", args.Resolvers); err != nil {
return err
}
return nil
}
Expand Down
71 changes: 71 additions & 0 deletions cmd/amass/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"flag"
"fmt"
"io"
"log/slog"
"net"
"net/netip"
"net/url"
Expand All @@ -48,6 +49,8 @@ import (
et "github.com/owasp-amass/engine/types"
"github.com/owasp-amass/open-asset-model/domain"
oamnet "github.com/owasp-amass/open-asset-model/network"
slogcommon "github.com/samber/slog-common"
slogsyslog "github.com/samber/slog-syslog/v2"
)

const (
Expand Down Expand Up @@ -307,3 +310,71 @@ func convertScopeToAssets(scope *config.Scope) []*et.Asset {

return assets
}

func selectLogger(dir, logfile string) *slog.Logger {
if logfile == "" {
if l := setupSyslogLogger(); l != nil {
return l
}
}
return setupFileLogger(dir, logfile)
}

func setupFileLogger(dir, logfile string) *slog.Logger {
if dir != "" {
if err := os.MkdirAll(dir, 0640); err != nil {
fmt.Fprintf(os.Stderr, "Failed to create the log directory: %v", err)
}
}

p := filepath.Join(dir, fmt.Sprintf("amass_client_%s.log", time.Now().Format("2006-01-02T15:04:05")))
if logfile != "" {
p = logfile
}

f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open log file: %v", err)
}

return slog.New(slog.NewJSONHandler(f, nil))
}

func setupSyslogLogger() *slog.Logger {
port := os.Getenv("SYSLOG_PORT")
host := strings.ToLower(os.Getenv("SYSLOG_HOST"))
transport := strings.ToLower(os.Getenv("SYSLOG_TRANSPORT"))

if host == "" {
return nil
}
if port == "" {
port = "514"
}
if transport == "" {
transport = "udp"
}

writer, err := net.Dial(transport, net.JoinHostPort(host, port))
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to create the connection to the log server: %v", err)
return nil
}

return slog.New(slogsyslog.Option{
Level: slog.LevelInfo,
Converter: syslogConverter,
Writer: writer,
}.NewSyslogHandler())
}

func syslogConverter(addSource bool, replaceAttr func(groups []string, a slog.Attr) slog.Attr, loggerAttr []slog.Attr, groups []string, record *slog.Record) map[string]any {
attrs := slogcommon.AppendRecordAttrsToAttrs(loggerAttr, groups, record)
attrs = slogcommon.ReplaceAttrs(replaceAttr, []string{}, attrs...)

return map[string]any{
"level": record.Level.String(),
"message": record.Message,
"attrs": slogcommon.AttrsToMap(attrs...),
}
}

0 comments on commit ce3bd8e

Please sign in to comment.