JSON output (#117)

* Add JSONOutput option * Do json output * Do not include successful pages * Bump version
raviqqe · Sep 27, 2020 · afaee41 · afaee41
1 parent 4ef17e1
commit afaee41
Show file tree

Hide file tree

Showing 9 changed files with 117 additions and 3 deletions.
diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp
@@ -16,6 +16,7 @@ Application Options:
                                             sitemap.xml
       --header=<header>...                  Custom headers
   -f, --ignore-fragments                    Ignore URL fragments
+      --json                                Output results in JSON
   -r, --max-redirections=<count>            Maximum number of redirections
                                             (default: 64)
   -t, --timeout=<seconds>                   Timeout for HTTP requests in

diff --git a/.snapshots/TestMarshalJSONPageResult b/.snapshots/TestMarshalJSONPageResult
@@ -0,0 +1 @@
+{"url":"http://foo.com","links":[{"url":"http://foo.com/bar","error":"baz"}]}
diff --git a/arguments.go b/arguments.go
@@ -18,6 +18,7 @@ type arguments struct {
 	FollowSitemapXML      bool     `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml"`
 	RawHeaders            []string `long:"header" value-name:"<header>..." description:"Custom headers"`
 	IgnoreFragments       bool     `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
+	JSONOutput            bool     `long:"json" description:"Output results in JSON"`
 	MaxRedirections       int      `short:"r" long:"max-redirections" value-name:"<count>" default:"64" description:"Maximum number of redirections"`
 	Timeout               int      `short:"t" long:"timeout" value-name:"<seconds>" default:"10" description:"Timeout for HTTP requests in seconds"`
 	Verbose               bool     `short:"v" long:"verbose" description:"Show successful results too"`

diff --git a/arguments_test.go b/arguments_test.go
@@ -30,6 +30,7 @@ func TestGetArguments(t *testing.T) {
 		{"-v", "-f", "https://foo.com"},
 		{"-v", "--ignore-fragments", "https://foo.com"},
 		{"--one-page-only", "https://foo.com"},
+		{"--json", "https://foo.com"},
 		{"-h"},
 		{"--help"},
 		{"--version"},

diff --git a/command.go b/command.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -101,6 +102,10 @@ func (c *command) runWithError(ss []string) (bool, error) {
 
 	go checker.Check(p)
 
+	if args.JSONOutput {
+		return c.printResultsInJSON(checker.Results())
+	}
+
 	formatter := newPageResultFormatter(args.Verbose, c.terminal)
 	ok := true
 
@@ -117,13 +122,35 @@ func (c *command) runWithError(ss []string) (bool, error) {
 	return ok, nil
 }
 
-func (c command) print(xs ...interface{}) {
+func (c *command) printResultsInJSON(rc <-chan *pageResult) (bool, error) {
+	rs := []*jsonPageResult{}
+	ok := true
+
+	for r := range rc {
+		if !r.OK() {
+			rs = append(rs, newJSONPageResult(r))
+			ok = false
+		}
+	}
+
+	bs, err := json.Marshal(rs)
+
+	if err != nil {
+		return false, err
+	}
+
+	c.print(string(bs))
+
+	return ok, nil
+}
+
+func (c *command) print(xs ...interface{}) {
 	if _, err := fmt.Fprintln(c.stdout, strings.TrimSpace(fmt.Sprint(xs...))); err != nil {
 		panic(err)
 	}
 }
 
-func (c command) printError(xs ...interface{}) {
+func (c *command) printError(xs ...interface{}) {
 	s := fmt.Sprint(xs...)
 
 	if c.terminal {

diff --git a/command_test.go b/command_test.go
@@ -220,3 +220,40 @@ func TestCommandShowVersion(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, r.MatchString(strings.TrimSpace(b.String())))
 }
+
+func TestCommandFailToRunWithJSONOutput(t *testing.T) {
+	b := &bytes.Buffer{}
+
+	ok := newTestCommandWithStdout(
+		b,
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			if u.String() == "http://foo.com" {
+				return newFakeHTTPResponse(
+					200,
+					"http://foo.com",
+					"text/html",
+					[]byte(`<html><body><a href="/foo" /></body></html>`),
+				), nil
+			}
+
+			return nil, errors.New("foo")
+		},
+	).Run([]string{"--json", "http://foo.com"})
+
+	assert.False(t, ok)
+	assert.Greater(t, b.Len(), 0)
+}
+
+func TestCommandDoNotIncludeSuccessfulPageInJSONOutput(t *testing.T) {
+	b := &bytes.Buffer{}
+
+	ok := newTestCommandWithStdout(
+		b,
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			return newFakeHTTPResponse(200, "", "text/html", nil), nil
+		},
+	).Run([]string{"--json", "http://foo.com"})
+
+	assert.True(t, ok)
+	assert.Equal(t, strings.TrimSpace(b.String()), "[]")
+}
diff --git a/configuration.go b/configuration.go
@@ -3,7 +3,7 @@ package main
 import "time"
 
 const (
-	version     = "2.0.6"
+	version     = "2.1.0"
 	agentName   = "muffet"
 	concurrency = 1024
 	tcpTimeout  = 5 * time.Second

diff --git a/json_page_result.go b/json_page_result.go
@@ -0,0 +1,21 @@
+package main
+
+type jsonPageResult struct {
+	URL   string            `json:"url"`
+	Links []*jsonLinkResult `json:"links"`
+}
+
+type jsonLinkResult struct {
+	URL   string `json:"url"`
+	Error string `json:"error"`
+}
+
+func newJSONPageResult(r *pageResult) *jsonPageResult {
+	ls := make([]*jsonLinkResult, 0, len(r.ErrorLinkResults))
+
+	for _, r := range r.ErrorLinkResults {
+		ls = append(ls, &jsonLinkResult{r.URL, r.Error.Error()})
+	}
+
+	return &jsonPageResult{r.URL, ls}
+}
diff --git a/json_page_result_test.go b/json_page_result_test.go
@@ -0,0 +1,25 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"github.com/bradleyjkemp/cupaloy"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarshalJSONPageResult(t *testing.T) {
+	bs, err := json.Marshal(newJSONPageResult(
+		&pageResult{
+			"http://foo.com",
+			[]*successLinkResult{
+				{"http://foo.com/foo", 200},
+			},
+			[]*errorLinkResult{
+				{"http://foo.com/bar", errors.New("baz")},
+			},
+		}))
+	assert.Nil(t, err)
+	cupaloy.SnapshotT(t, bs)
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"url":"http://foo.com","links":[{"url":"http://foo.com/bar","error":"baz"}]}