diff --git a/arguments.go b/arguments.go index 3fee5168..c537acdf 100644 --- a/arguments.go +++ b/arguments.go @@ -45,7 +45,6 @@ func getArguments(ss []string) (*arguments, error) { args.URL = ss[0] rs, err := compileRegexps(args.RawExcludedPatterns) - if err != nil { return nil, err } @@ -53,7 +52,6 @@ func getArguments(ss []string) (*arguments, error) { args.ExcludedPatterns = rs hs, err := parseHeaders(args.RawHeaders) - if err != nil { return nil, err } diff --git a/checker.go b/checker.go index 5dda9380..f4c854a5 100644 --- a/checker.go +++ b/checker.go @@ -61,10 +61,7 @@ func (c *checker) checkPage(p *page) { ec <- &errorLinkResult{u, err} } - if !c.onePageOnly && - p != nil && - c.linkValidator.Validate(p.URL()) && - !c.donePages.Add(p.URL().String()) { + if !c.onePageOnly && p != nil && c.linkValidator.Validate(p.URL()) { c.addPage(p) } }(u) @@ -91,5 +88,7 @@ func (c *checker) checkPage(p *page) { } func (c *checker) addPage(p *page) { - c.daemonManager.Add(func() { c.checkPage(p) }) + if !c.donePages.Add(p.URL().String()) { + c.daemonManager.Add(func() { c.checkPage(p) }) + } } diff --git a/checker_test.go b/checker_test.go index 49048846..c3f58333 100644 --- a/checker_test.go +++ b/checker_test.go @@ -92,3 +92,23 @@ func TestCheckerFailToCheckPage(t *testing.T) { assert.False(t, (<-c.Results()).OK()) } + +func TestCheckerDoNotCheckSamePageTwice(t *testing.T) { + c := newTestChecker( + newFakeHTTPClient( + func(u *url.URL) (*fakeHTTPResponse, error) { + return newFakeHTTPResponse(200, "http://foo.com", "text/html", nil), nil + }, + ), + ) + + go c.Check(newTestPage(t, nil, map[string]error{"http://foo.com": nil})) + + i := 0 + + for range c.Results() { + i++ + } + + assert.Equal(t, 1, i) +} diff --git a/command.go b/command.go index 11147ef0..3111a605 100644 --- a/command.go +++ b/command.go @@ -22,7 +22,6 @@ func newCommand(stdout, stderr io.Writer, terminal bool, f httpClientFactory) *c func (c *command) Run(args []string) bool { ok, err := c.runWithError(args) - if err != nil { c.printError(err) } diff --git a/configuration.go b/configuration.go index 3c164c29..61f767ec 100644 --- a/configuration.go +++ b/configuration.go @@ -3,7 +3,7 @@ package main import "time" const ( - version = "2.0.0" + version = "2.0.1" agentName = "muffet" concurrency = 1024 tcpTimeout = time.Minute diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go index 6620056f..f8bcfecf 100644 --- a/fasthttp_http_client_factory.go +++ b/fasthttp_http_client_factory.go @@ -24,7 +24,8 @@ func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient { }, Dial: func(addr string) (net.Conn, error) { return fasthttp.DialTimeout(addr, tcpTimeout) - }}, + }, + }, o.MaxRedirections, o.Timeout, ) diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go index c0592e33..eccda110 100644 --- a/robots_txt_fetcher.go +++ b/robots_txt_fetcher.go @@ -19,7 +19,6 @@ func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) { u := *uu u.Path = "robots.txt" r, err := f.client.Get(&u, nil) - if err != nil { return nil, fmt.Errorf("failed to fetch robots.txt: %v", err) } diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go index a34fc267..0ce313d0 100644 --- a/sitemap_fetcher.go +++ b/sitemap_fetcher.go @@ -21,7 +21,6 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) { u.Path = "sitemap.xml" r, err := f.client.Get(&u, nil) - if err != nil { return nil, fmt.Errorf("failed to GET sitemap.xml: %v", err) }