Skip to content

Commit

Permalink
initial changes for the integration of asset-db and v4.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@caffix
caffix committed Jul 1, 2023
1 parent 4151628 commit 7e7c348
Show file tree
Hide file tree
Showing 38 changed files with 340 additions and 3,268 deletions.
209 changes: 30 additions & 179 deletions cmd/amass/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,20 @@ package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"os"
"strconv"

"github.com/caffix/netmap"
"github.com/caffix/stringset"
"github.com/fatih/color"
"github.com/owasp-amass/amass/v3/config"
"github.com/owasp-amass/amass/v3/datasrcs"
"github.com/owasp-amass/amass/v3/format"
"github.com/owasp-amass/amass/v3/requests"
"github.com/owasp-amass/amass/v3/systems"
oam "github.com/owasp-amass/open-asset-model"
"github.com/owasp-amass/open-asset-model/network"
)

const (
Expand All @@ -33,23 +31,20 @@ type dbArgs struct {
Domains *stringset.Set
Enum int
Options struct {
DemoMode bool
IPs bool
IPv4 bool
IPv6 bool
ListEnumerations bool
ASNTableSummary bool
DiscoveredNames bool
NoColor bool
ShowAll bool
Silent bool
Sources bool
DemoMode bool
IPs bool
IPv4 bool
IPv6 bool
ASNTableSummary bool
DiscoveredNames bool
NoColor bool
ShowAll bool
Silent bool
}
Filepaths struct {
ConfigFile string
Directory string
Domains string
JSONOutput string
TermOut string
}
}
Expand All @@ -67,13 +62,10 @@ func runDBCommand(clArgs []string) {
dbCommand.BoolVar(&help1, "h", false, "Show the program usage message")
dbCommand.BoolVar(&help2, "help", false, "Show the program usage message")
dbCommand.Var(args.Domains, "d", "Domain names separated by commas (can be used multiple times)")
dbCommand.IntVar(&args.Enum, "enum", 0, "Identify an enumeration via an index from the listing")
dbCommand.BoolVar(&args.Options.DemoMode, "demo", false, "Censor output to make it suitable for demonstrations")
dbCommand.BoolVar(&args.Options.IPs, "ip", false, "Show the IP addresses for discovered names")
dbCommand.BoolVar(&args.Options.IPv4, "ipv4", false, "Show the IPv4 addresses for discovered names")
dbCommand.BoolVar(&args.Options.IPv6, "ipv6", false, "Show the IPv6 addresses for discovered names")
dbCommand.BoolVar(&args.Options.ListEnumerations, "list", false, "Numbered list of enums filtered on provided domains")
dbCommand.BoolVar(&args.Options.Sources, "src", false, "Print data sources for the discovered names")
dbCommand.BoolVar(&args.Options.ASNTableSummary, "summary", false, "Print Just ASN Table Summary")
dbCommand.BoolVar(&args.Options.DiscoveredNames, "names", false, "Print Just Discovered Names")
dbCommand.BoolVar(&args.Options.NoColor, "nocolor", false, "Disable colorized output")
Expand All @@ -82,7 +74,6 @@ func runDBCommand(clArgs []string) {
dbCommand.StringVar(&args.Filepaths.ConfigFile, "config", "", "Path to the INI configuration file. Additional details below")
dbCommand.StringVar(&args.Filepaths.Directory, "dir", "", "Path to the directory containing the graph database")
dbCommand.StringVar(&args.Filepaths.Domains, "df", "", "Path to a file providing root domain names")
dbCommand.StringVar(&args.Filepaths.JSONOutput, "json", "", "Path to the JSON output file")
dbCommand.StringVar(&args.Filepaths.TermOut, "o", "", "Path to the text file containing terminal stdout/stderr")

if len(clArgs) < 1 {
Expand Down Expand Up @@ -127,90 +118,33 @@ func runDBCommand(clArgs []string) {
os.Exit(1)
}

srcs := datasrcs.GetAllSources(&systems.LocalSystem{Cfg: cfg})
initializeSourceTags(srcs)
for _, src := range srcs {
_ = src.Stop()
}

db := openGraphDatabase(args.Filepaths.Directory, cfg)
if db == nil {
r.Fprintln(color.Error, "Failed to connect with the database")
os.Exit(1)
}
defer db.Close()
// Create the in-memory graph database for events that have information in scope
memDB, err := memGraphForScope(context.Background(), args.Domains.Slice(), db)
if err != nil {
r.Fprintln(color.Error, err.Error())
os.Exit(1)
}
defer memDB.Close()
// Get all the UUIDs for events that have information in scope
uuids := memDB.EventList(context.Background())
if len(uuids) == 0 {
r.Fprintln(color.Error, "Failed to find the domains of interest in the database")
os.Exit(1)
}
if args.Options.ListEnumerations {
listEvents(uuids, memDB)
return
}
if args.Options.ShowAll || args.Filepaths.JSONOutput != "" {

if args.Options.ShowAll {
args.Options.DiscoveredNames = true
args.Options.ASNTableSummary = true
}
if !args.Options.DiscoveredNames && !args.Options.ASNTableSummary {
commandUsage(dbUsageMsg, dbCommand, dbBuf)
return
}
// Put the events in chronological order
uuids, _, _ = orderedEvents(context.Background(), uuids, memDB)
if len(uuids) == 0 {
r.Fprintln(color.Error, "Failed to sort the events")
os.Exit(1)
}
// Select the enumeration that the user specified
if args.Enum > 0 && len(uuids) >= args.Enum {
idx := len(uuids) - args.Enum

uuids = []string{uuids[idx]}
}

var asninfo bool
if args.Options.ASNTableSummary {
asninfo = true
}

showEventData(&args, uuids, asninfo, memDB)
}

func listEvents(uuids []string, db *netmap.Graph) {
events, earliest, latest := orderedEvents(context.Background(), uuids, db)
// Check if the user has requested the list of enumerations
for pos, idx := 0, len(events)-1; idx >= 0; idx-- {
if pos != 0 {
g.Println()
}

g.Printf("%d) %s -> %s: ", pos+1, earliest[idx].Format(timeFormat), latest[idx].Format(timeFormat))
// Print out the scope for this enumeration
for x, domain := range db.EventDomains(context.Background(), events[idx]) {
if x != 0 {
g.Print(", ")
}
g.Print(domain)
}
g.Println()
pos++
}
showEventData(&args, asninfo, db)
}

func showEventData(args *dbArgs, uuids []string, asninfo bool, db *netmap.Graph) {
func showEventData(args *dbArgs, asninfo bool, db *netmap.Graph) {
var total int
var err error
var outfile *os.File
var discovered []*requests.Output
domains := args.Domains.Slice()

if args.Filepaths.TermOut != "" {
Expand All @@ -231,14 +165,13 @@ func showEventData(args *dbArgs, uuids []string, asninfo bool, db *netmap.Graph)
if asninfo {
cache = requests.NewASNCache()
if err := fillCache(cache, db); err != nil {
r.Println("Failed to populate the ASN cache")
r.Printf("Failed to populate the ASN cache: %v\n", err)
return
}
}

tags := make(map[string]int)
asns := make(map[int]*format.ASNSummaryData)
for _, out := range getEventOutput(context.Background(), uuids, asninfo, db, cache) {
for _, out := range getEventOutput(context.Background(), domains, asninfo, db, cache) {
if len(domains) > 0 && !domainNameInScope(out.Name, domains) {
continue
}
Expand All @@ -247,28 +180,23 @@ func showEventData(args *dbArgs, uuids []string, asninfo bool, db *netmap.Graph)
if l := len(out.Addresses); (args.Options.IPs || args.Options.IPv4 || args.Options.IPv6) && l == 0 {
continue
} else if l > 0 {
format.UpdateSummaryData(out, tags, asns)
format.UpdateSummaryData(out, asns)
}

total++
source, name, ips := format.OutputLineParts(out, args.Options.Sources,
args.Options.IPs || args.Options.IPv4 || args.Options.IPv6, args.Options.DemoMode)
name, ips := format.OutputLineParts(out, args.Options.IPs || args.Options.IPv4 || args.Options.IPv6, args.Options.DemoMode)
if ips != "" {
ips = " " + ips
}

if args.Options.DiscoveredNames {
var written bool
if outfile != nil {
fmt.Fprintf(outfile, "%s%s%s\n", source, name, ips)
written = true
}
if args.Filepaths.JSONOutput != "" {
discovered = append(discovered, out)
fmt.Fprintf(outfile, "%s%s\n", name, ips)
written = true
}
if !written {
fmt.Fprintf(color.Output, "%s%s%s\n", blue(source), green(name), yellow(ips))
fmt.Fprintf(color.Output, "%s%s\n", green(name), yellow(ips))
}
}
}
Expand All @@ -277,9 +205,7 @@ func showEventData(args *dbArgs, uuids []string, asninfo bool, db *netmap.Graph)
r.Println("No names were discovered")
return
}
if args.Filepaths.JSONOutput != "" {
writeJSON(args, uuids, discovered, db)
} else if args.Options.ASNTableSummary {
if args.Options.ASNTableSummary {
var out io.Writer
status := color.NoColor

Expand All @@ -292,102 +218,29 @@ func showEventData(args *dbArgs, uuids []string, asninfo bool, db *netmap.Graph)
out = color.Output
}

format.FprintEnumerationSummary(out, total, tags, asns, args.Options.DemoMode)
format.FprintEnumerationSummary(out, total, asns, args.Options.DemoMode)
color.NoColor = status
}
}

type jsonEvent struct {
UUID string `json:"uuid"`
Start string `json:"start"`
Finish string `json:"finish"`
}

type jsonDomain struct {
Domain string `json:"domain"`
Total int `json:"total"`
Names []*requests.Output `json:"names"`
}

type jsonOutput struct {
Events []*jsonEvent `json:"events"`
Domains []*jsonDomain `json:"domains"`
}

func writeJSON(args *dbArgs, uuids []string, assets []*requests.Output, db *netmap.Graph) {
var output jsonOutput

// Add the event data to the JSON
events, earliest, latest := orderedEvents(context.Background(), uuids, db)
for i, uuid := range events {
output.Events = append(output.Events, &jsonEvent{
UUID: uuid,
Start: earliest[i].Format(timeFormat),
Finish: latest[i].Format(timeFormat),
})
}
// Add the asset specific data
for _, asset := range assets {
var found bool
var d *jsonDomain

for _, domain := range output.Domains {
if domain.Domain == asset.Domain {
found = true
d = domain
break
}
}
if !found {
d = &jsonDomain{Domain: asset.Domain}

output.Domains = append(output.Domains, d)
}

d.Total++
d.Names = append(d.Names, asset)
}

var jsonptr *os.File
var err error

// Write to STDOUT and not a file if named "-"
if args.Filepaths.JSONOutput == "-" {
jsonptr = os.Stdout
} else {
jsonptr, err = os.OpenFile(args.Filepaths.JSONOutput, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
r.Fprintf(color.Error, "Failed to open the JSON output file: %v\n", err)
return
}
}

// Remove previously stored data and encode the JSON
_ = jsonptr.Truncate(0)
_, _ = jsonptr.Seek(0, 0)
_ = json.NewEncoder(jsonptr).Encode(output)
_ = jsonptr.Sync()
_ = jsonptr.Close()
}

func fillCache(cache *requests.ASNCache, db *netmap.Graph) error {
aslist, err := db.AllNodesOfType(context.Background(), netmap.TypeAS)
assets, err := db.DB.FindByType(oam.ASN)
if err != nil {
return err
}

for _, as := range aslist {
asn, err := strconv.Atoi(as.(string))
if err != nil {
for _, a := range assets {
as, ok := a.Asset.(network.AutonomousSystem)
if !ok {
continue
}

desc := db.ReadASDescription(context.Background(), asn)
desc := db.ReadASDescription(context.Background(), as.Number)
if desc == "" {
continue
}

for _, prefix := range db.ReadASPrefixes(context.Background(), asn) {
for _, prefix := range db.ReadASPrefixes(context.Background(), as.Number) {
first, cidr, err := net.ParseCIDR(prefix)
if err != nil {
continue
Expand All @@ -398,11 +251,9 @@ func fillCache(cache *requests.ASNCache, db *netmap.Graph) error {

cache.Update(&requests.ASNRequest{
Address: first.String(),
ASN: asn,
ASN: as.Number,
Prefix: cidr.String(),
Description: desc,
Tag: requests.RIR,
Source: "RIR",
})
}
}
Expand Down
Loading

0 comments on commit 7e7c348

Please sign in to comment.