diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp index f42f1c5a..53b84c68 100644 --- a/.snapshots/TestHelp +++ b/.snapshots/TestHelp @@ -16,6 +16,7 @@ Application Options: sitemap.xml --header=
... Custom headers -f, --ignore-fragments Ignore URL fragments + --json Output results in JSON -r, --max-redirections= Maximum number of redirections (default: 64) -t, --timeout= Timeout for HTTP requests in diff --git a/.snapshots/TestMarshalJSONPageResult b/.snapshots/TestMarshalJSONPageResult new file mode 100644 index 00000000..bcda3c57 --- /dev/null +++ b/.snapshots/TestMarshalJSONPageResult @@ -0,0 +1 @@ +{"url":"http://foo.com","links":[{"url":"http://foo.com/bar","error":"baz"}]} diff --git a/arguments.go b/arguments.go index fadd7b90..7a9ac342 100644 --- a/arguments.go +++ b/arguments.go @@ -18,6 +18,7 @@ type arguments struct { FollowSitemapXML bool `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml"` RawHeaders []string `long:"header" value-name:"
..." description:"Custom headers"` IgnoreFragments bool `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"` + JSONOutput bool `long:"json" description:"Output results in JSON"` MaxRedirections int `short:"r" long:"max-redirections" value-name:"" default:"64" description:"Maximum number of redirections"` Timeout int `short:"t" long:"timeout" value-name:"" default:"10" description:"Timeout for HTTP requests in seconds"` Verbose bool `short:"v" long:"verbose" description:"Show successful results too"` diff --git a/arguments_test.go b/arguments_test.go index 2ba123e4..778b34cf 100644 --- a/arguments_test.go +++ b/arguments_test.go @@ -30,6 +30,7 @@ func TestGetArguments(t *testing.T) { {"-v", "-f", "https://foo.com"}, {"-v", "--ignore-fragments", "https://foo.com"}, {"--one-page-only", "https://foo.com"}, + {"--json", "https://foo.com"}, {"-h"}, {"--help"}, {"--version"}, diff --git a/command.go b/command.go index 3889b46d..5d77204e 100644 --- a/command.go +++ b/command.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "errors" "fmt" "io" @@ -101,6 +102,10 @@ func (c *command) runWithError(ss []string) (bool, error) { go checker.Check(p) + if args.JSONOutput { + return c.printResultsInJSON(checker.Results()) + } + formatter := newPageResultFormatter(args.Verbose, c.terminal) ok := true @@ -117,13 +122,35 @@ func (c *command) runWithError(ss []string) (bool, error) { return ok, nil } -func (c command) print(xs ...interface{}) { +func (c *command) printResultsInJSON(rc <-chan *pageResult) (bool, error) { + rs := []*jsonPageResult{} + ok := true + + for r := range rc { + if !r.OK() { + rs = append(rs, newJSONPageResult(r)) + ok = false + } + } + + bs, err := json.Marshal(rs) + + if err != nil { + return false, err + } + + c.print(string(bs)) + + return ok, nil +} + +func (c *command) print(xs ...interface{}) { if _, err := fmt.Fprintln(c.stdout, strings.TrimSpace(fmt.Sprint(xs...))); err != nil { panic(err) } } -func (c command) printError(xs ...interface{}) { +func (c *command) printError(xs ...interface{}) { s := fmt.Sprint(xs...) if c.terminal { diff --git a/command_test.go b/command_test.go index ed7be8ca..3c6eb5b9 100644 --- a/command_test.go +++ b/command_test.go @@ -220,3 +220,40 @@ func TestCommandShowVersion(t *testing.T) { assert.Nil(t, err) assert.True(t, r.MatchString(strings.TrimSpace(b.String()))) } + +func TestCommandFailToRunWithJSONOutput(t *testing.T) { + b := &bytes.Buffer{} + + ok := newTestCommandWithStdout( + b, + func(u *url.URL) (*fakeHTTPResponse, error) { + if u.String() == "http://foo.com" { + return newFakeHTTPResponse( + 200, + "http://foo.com", + "text/html", + []byte(``), + ), nil + } + + return nil, errors.New("foo") + }, + ).Run([]string{"--json", "http://foo.com"}) + + assert.False(t, ok) + assert.Greater(t, b.Len(), 0) +} + +func TestCommandDoNotIncludeSuccessfulPageInJSONOutput(t *testing.T) { + b := &bytes.Buffer{} + + ok := newTestCommandWithStdout( + b, + func(u *url.URL) (*fakeHTTPResponse, error) { + return newFakeHTTPResponse(200, "", "text/html", nil), nil + }, + ).Run([]string{"--json", "http://foo.com"}) + + assert.True(t, ok) + assert.Equal(t, strings.TrimSpace(b.String()), "[]") +} diff --git a/configuration.go b/configuration.go index aa63c8d9..289141bf 100644 --- a/configuration.go +++ b/configuration.go @@ -3,7 +3,7 @@ package main import "time" const ( - version = "2.0.6" + version = "2.1.0" agentName = "muffet" concurrency = 1024 tcpTimeout = 5 * time.Second diff --git a/json_page_result.go b/json_page_result.go new file mode 100644 index 00000000..cc6fd367 --- /dev/null +++ b/json_page_result.go @@ -0,0 +1,21 @@ +package main + +type jsonPageResult struct { + URL string `json:"url"` + Links []*jsonLinkResult `json:"links"` +} + +type jsonLinkResult struct { + URL string `json:"url"` + Error string `json:"error"` +} + +func newJSONPageResult(r *pageResult) *jsonPageResult { + ls := make([]*jsonLinkResult, 0, len(r.ErrorLinkResults)) + + for _, r := range r.ErrorLinkResults { + ls = append(ls, &jsonLinkResult{r.URL, r.Error.Error()}) + } + + return &jsonPageResult{r.URL, ls} +} diff --git a/json_page_result_test.go b/json_page_result_test.go new file mode 100644 index 00000000..cbadc069 --- /dev/null +++ b/json_page_result_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/bradleyjkemp/cupaloy" + "github.com/stretchr/testify/assert" +) + +func TestMarshalJSONPageResult(t *testing.T) { + bs, err := json.Marshal(newJSONPageResult( + &pageResult{ + "http://foo.com", + []*successLinkResult{ + {"http://foo.com/foo", 200}, + }, + []*errorLinkResult{ + {"http://foo.com/bar", errors.New("baz")}, + }, + })) + assert.Nil(t, err) + cupaloy.SnapshotT(t, bs) +}