Merge branch 'add-response'

Author: osvik

README.md

@@ -40,45 +40,48 @@ If you installed from source do:
 
 The urls file, by default `urls.csv` must have all the urls you want to check. You can use a text file with 1 url per line or a csv file with the urls on the first column and without headers.
 
-## Checking the urls
+You can use [ecounter](https://github.com/greenpeace/ecounter) to create a urls file from a sitemap.xml file.
 
-To check all urls in `urls.csv` with all the checks use the command:
+## Http info about a list of urls
 
+If you want to obtain information about http status codes, mime-types, file sizes and redirect urls of any urls, you can use `-http`.
+
+You must use this check in a separate command like:
 ```
-./check-my-pages -urls=urls.csv -http -analytics -canonical -redirects -linkpattern -cssjspattern -mediapattern
+./check-my-pages -urls=urls.csv -http -miliseconds=100
+```
+because check-my-pages will stop after executing `-http`
+
+This check creates a file named `httpResponses.csv` with 5 fields: 
+1. initial url
+2. http status code
+3. mime type
+4. file size *(adds -1 if the file size is unknown)*
+5. final url
+
+## Checking html urls
+
+To do all the checks in `urls.csv` (html urls) with all the checks use the command:
+
+```
+./check-my-pages -urls=urls.csv -analytics -canonical -linkpattern -cssjspattern -mediapattern
 ```
 
 This repository includes a few testing urls in the file `urls.csv`. Please replace them by your own.
 
 It will create a couple of files, one per check the script is doing:
-* `httpResponses.csv` - Stores the **http response** codes for the URL. 200 means everything is OK.
 * `analytics.csv` - Reports **google analytics** tracking ID
 * `canonicals.csv` - Reports the **canonical url** for every url
-* `redirects.csv` - Reports the requested URL and the final URL. This will be useful to test the **redirects** in the main site.
 * `linkpattern.csv` - Reports on links that include a regular expression pattern. Useful to track **links** to specific **dead sites**. The default pattern can be set by the `-pattern` option.
 * `cssjspattern.csv` - Reports **css and js** urls that include a regular expression pattern. To detect dead css and js urls in large sites. The pattern can also be defined with the option `-pattern` (described bellow)
 * `mediapattern.csv` - Reports **media** links. Images, videos, audios, iframes and objects. Also use `-pattern` to define the urls pattern.
 
 ## Optional command line configurations
 
-`-miliseconds=100` - Sets a delay of 100 miliseconds between requests.
+`-miliseconds=100` - Sets a delay of 100 miliseconds between requests (the default value)
 
 `-pattern='https?://(\w|-)+.greenpeace.org/espana/.+'` - Changes the search link pattern to the regular expression.
 
-## Information about other urls
-
-If you want to obtain information about non-html files, like for example images, it's better to use `-fileinfo`.
-
-You must use this check in a separate command like:
-
-```
-./check-my-pages -urls=urls.csv -fileinfo -miliseconds=100
-```
-
-because check-my-pages will stop after executing `-fileinfo`
-
-This check creates a file named `fileInfo.csv` with 4 fields: url, http status code, mime type and file size (adds -1 if the file size is unknown).
-
 ## Remove the report files
 
 To remove the files created by **check-my-pages**:

check-my-pages.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
 	"regexp"
 	"time"
@@ -16,10 +15,8 @@ func main() {
 	isHelp := flag.Bool("help", false, "Help")
 	urlsFileName := flag.String("urls", "urls.csv", "Name of the csv file with the urs in the first column")
 	isHTTP := flag.Bool("http", false, "Http response codes")
-	isRedirects := flag.Bool("redirects", false, "Redirects response codes")
 	isAnalytics := flag.Bool("analytics", false, "Correct analytics tag in the html")
 	isCanonical := flag.Bool("canonical", false, "Canonical URLS in the ")
-	isFileInfo := flag.Bool("fileinfo", false, "Specific calls for files")
 	isLinkpattern := flag.Bool("linkpattern", false, "Link Pattern")
 	isCSSJsPattern := flag.Bool("cssjspattern", false, "CSS and JS Pattern")
 	isMediaPattern := flag.Bool("mediapattern", false, "Image, object and iframe Pattern")
@@ -44,19 +41,22 @@ func main() {
 
 	if *isHTTP == true {
 
-		httpResponses, httpErr := os.OpenFile("httpResponses.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-		if httpErr != nil {
-			panic(httpErr)
+		isHTTPfile, isHTTPErr := os.OpenFile("httpResponses.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+		if isHTTPErr != nil {
+			panic(isHTTPErr)
 		}
-		defer httpResponses.Close()
+		defer isHTTPfile.Close()
 
-		c.OnResponse(func(r *colly.Response) {
-			lineResponse := fmt.Sprintf("%s,%v\n", r.Request.URL.String(), r.StatusCode)
-			if _, err := httpResponses.WriteString(lineResponse); err != nil {
+		var lineHTTP string
+		for _, v := range allUrls {
+			lineHTTP = getHTTPinfoAsCsvline(v)
+			if _, err := isHTTPfile.WriteString(lineHTTP); err != nil {
 				panic(err)
 			}
+			time.Sleep(time.Millisecond * time.Duration(*waitMiliseconds))
+		}
 
-		})
+		os.Exit(0)
 	}
 
 	if *isAnalytics == true {
@@ -94,26 +94,6 @@ func main() {
 		})
 	}
 
-	if *isFileInfo == true {
-
-		fileInfofile, fileInfofileErr := os.OpenFile("fileInfo.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-		if fileInfofileErr != nil {
-			panic(fileInfofileErr)
-		}
-		defer fileInfofile.Close()
-
-		var lineFileInfo string
-		for _, v := range allUrls {
-			lineFileInfo = fileInfo(v)
-			if _, err := fileInfofile.WriteString(lineFileInfo); err != nil {
-				panic(err)
-			}
-			time.Sleep(time.Millisecond * time.Duration(*waitMiliseconds))
-		}
-
-		os.Exit(0)
-	}
-
 	if *isLinkpattern == true {
 
 		linkpattern, linkpatternErr := os.OpenFile("linkpattern.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
@@ -234,39 +214,14 @@ func main() {
 
 	}
 
-	if *isRedirects == true {
-
-		redirects, redirectsErr := os.OpenFile("redirects.csv", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-		if redirectsErr != nil {
-			panic(redirects)
-		}
-		defer redirects.Close()
-
-		c.OnRequest(func(r *colly.Request) {
-			response, error := http.Get(r.URL.String())
-			if error != nil {
-				fmt.Printf("=> %v\n", error.Error())
-			} else {
-				finalURL := response.Request.URL.String()
-				lineCanonical := fmt.Sprintf("%s,%s\n", r.URL.String(), finalURL)
-				if _, err := redirects.WriteString(lineCanonical); err != nil {
-					panic(err)
-				}
-			}
-
-		})
-	}
-
 	if *isClear == true {
 
 		os.Remove("httpResponses.csv")
 		os.Remove("analytics.csv")
 		os.Remove("canonicals.csv")
-		os.Remove("redirects.csv")
 		os.Remove("linkpattern.csv")
 		os.Remove("cssjspattern.csv")
 		os.Remove("mediapattern.csv")
-		os.Remove("fileInfo.csv")
 		os.Exit(0)
 	}
 

file.go

@@ -5,8 +5,8 @@ import (
 	"net/http"
 )
 
-// Obtains statuscode, content-type and content lenght from a specific url
-func fileInfo(url string) string {
+// getHTTPinfoAsCsvline Obtains statuscode, content-type, content lenght and final URL from a specific http get request
+func getHTTPinfoAsCsvline(url string) string {
 	response, error := http.Get(url)
 	if error != nil {
 		return fmt.Sprintf("%s,%s,,\n", url, error.Error())
@@ -15,5 +15,6 @@ func fileInfo(url string) string {
 	statusCode := response.StatusCode
 	contentType := headers["Content-Type"][0]
 	contentLength := response.ContentLength
-	return fmt.Sprintf("%s,%d,%s,%d\n", url, statusCode, contentType, contentLength)
+	finalURL := response.Request.URL.String()
+	return fmt.Sprintf("%s,%d,%s,%d,%s\n", url, statusCode, contentType, contentLength, finalURL)
 }

help.go

@@ -16,29 +16,25 @@ check-my-pages is a scrapping script. It checks each url in a list and creates r
 
 EXAMPLES:
 
-./check-my-pages -urls=urls.csv -http -analytics -canonical -redirects -linkpattern -cssjspattern -mediapattern
+./check-my-pages -urls=urls.csv -http -miliseconds=100
 
-./check-my-pages -urls=urls.csv -fileinfo -miliseconds=100
+./check-my-pages -urls=urls.csv -analytics -canonical -linkpattern -cssjspattern -mediapattern
 
 
 CHECKS:
 
--http : Gets the http response code. If it's 200 it should be OK.
+-http : Gets the http response code, mime-type, file size and final url. It must be used separately from the other checks.
 
 -analytics : Gets the first Google Analytics account.
 
 -canonical : Gets the canonical URL for the url.
 
--redirects : Gets info about redirects and final URLs.
-
 -linkpattern : Gets links that match the regular expression pattern.
 
 -cssjspattern : Gets CSS and JS URLs that match the regular expression pattern.
 
 -mediapattern : Gets urls from images, videos, audios, iframes and objects that match the regular expression pattern
 
--fileinfo : Speciall check more suitable for non-html pages (for example images). It needs to be used alone as the example above, without other checks.
-
 
 OPTIONS:
 
@@ -48,6 +44,10 @@ OPTIONS:
 
 -miliseconds=100 : Sets a delay of 100 miliseconds between requests.
 
+OTHER:
+
+-clear : Deletes all the files with the reports
+
 
 FILES WITH THE REPORTS:
 
@@ -57,8 +57,6 @@ FILES WITH THE REPORTS:
 
 - canonicals.csv : Reports the canonical url for every url
 
-- redirects.csv : Reports the requested URL and the final URL. This will be useful to test the redirects in the main site.
-
 - linkpattern.csv : Reports on links that include a regular expression pattern. Useful to track links to specific dead sites. The default pattern can be set by the -pattern option.
 
 - cssjspattern.csv : Reports css and js urls that include a regular expression pattern. To detect dead css and js urls in large sites. The pattern can also be defined with the option -pattern (described bellow)