Merge pull request #12 from circleci/retest

Add retest.

Author: technomancy

README.md

@@ -10,9 +10,12 @@ Add `[circleci/circleci.test "0.2.0"]` to your dev dependencies.
 
 Run specific test namespaces with `lein run -m circleci.test my.first.test.ns my.second.test.ns`.
 
+Run all loaded test namespaces from a repl with `(circleci.test/run-tests)` or
+pass in a list of symbols to just test a few namespaces.
+
 Run individiual tests from a repl with `(circleci.test/test-var #'my.test.ns/my-test)`.
 
-Alternatively you can add a Leiningen alias to test your whole test directory:
+It's more convenient to a Leiningen alias to test your whole test directory:
 
 ```clj
 :aliases {"test" ["run" "-m" "circleci.test/dir" :project/test-paths]}
@@ -40,7 +43,7 @@ selector functions in `dev-resources/circleci_test/config.clj`:
              :default (complement :flaky)}}
 ```
 
-If you have the `circleci.test/dir` alias set up as per above, you can pass
+If you have the Leiningen `:aliases` set up as per above, you can pass
 in a test selector as a command-line argument:
 
     $ lein test :acceptance
@@ -55,16 +58,26 @@ putting this in your `dev-resources/circleci_test/config.clj` file:
 ```clj
 (require '[circleci.test.report.junit :as junit])
 
-{:reporters [(circleci.test.report/clojure-test-reporter)
-             (junit/reporter "target/test-results")]}
+{:test-results-dir "target/test-results"
+ :reporters [circleci.test.report/clojure-test-reporter
+             junit/reporter]}
 ```
 
-Unlike `clojure.test`, you can use more than one reporter at a time. The
-argument to the `junit/reporter` function indicates which directory you
-want your test output in. It's best to ignore this directory in your version
-control system configuration.
+Unlike `clojure.test`, you can use more than one reporter at a time. It's
+recommended to add your `:test-results-dir` directory to the ignored list of
+your version control system.
+
+If you use the junit test reporter, you can run `circleci.test.retest/-main` to
+re-run only the set of tests which previously failed. Adding an alias in
+`project.clj` is recommended:
+
+```clj
+:aliases {"test" ["run" "-m" "circleci.test/dir" :project/test-paths]
+          "retest" ["run" "-m" "circleci.test.retest"]}
+```
 
 ### Running tests from a repl
+
 Use `circleci.test/test-var` to run a single test fn:
 ```clojure
 (circleci.test/test-var #'my.test.ns/my-test)
@@ -82,7 +95,10 @@ running all tests that exist on disk.
 
 ### Global fixtures
 
-In addition to `:once` and `:each` fixtures from `clojure.test`, you can define global fixtures that are only run once for the entire test run, no matter how many namespaces you run. This should be declared in your config file rather in any one individual test file though; otherwise it may not get loaded:
+In addition to `:once` and `:each` fixtures from `clojure.test`, you can define
+global fixtures that are only run once for the entire test run, no matter how
+many namespaces you run. This should be declared in your config file rather in
+any one individual test file though; otherwise it may not get loaded:
 
 ```clj
 {:global-fixture (fn [f]
@@ -164,13 +180,16 @@ and will not work with code that assumes it has control of that class.
 ## Design goals
 
 #### Compatibility with tests written using `clojure.test`
+
 Don't force people to re-write tests just to use a different test runner.
 
-Maintain compatibility with the `clojure.test` assertion expansions such as [slingshot.test](https://github.com/scgilardi/slingshot/blob/release/src/slingshot/test.clj)
+Maintain compatibility with the `clojure.test` assertion expansions such as
+[slingshot.test](https://github.com/scgilardi/slingshot/blob/release/src/slingshot/test.clj)
 
 #### Extensible test reporting
-Must be possible to use more than one test reporter at a time, for instance it's desirable to produce Junit XML and console output during a test run.
 
+Must be possible to use more than one test reporter at a time, for instance it's
+desirable to produce Junit XML and console output during a test run.
 
 ## License
 

dev-resources/circleci_test/config.clj

@@ -5,9 +5,9 @@
 
 {:selectors {:all (constantly true)
              :default (complement :failing)}
- :reporters [(clojure-test-reporter)
-             (junit/reporter (or (System/getenv "CIRCLE_TEST_REPORTS")
-                                 "test-results"))]
+ :test-results-dir (or (System/getenv "CIRCLE_TEST_REPORTS")
+                       "test-results")
+ :reporters [clojure-test-reporter junit/reporter]
  :global-fixture (fn [f]
                    (assert (not *inside-global*))
                    (binding [*inside-global* true]

project.clj

@@ -6,5 +6,6 @@
   :pedantic? :abort
   :dependencies [[org.clojure/clojure "1.8.0"]
                  [org.clojure/data.xml "0.0.8"]]
-  :aliases {"test" ["run" "-m" "circleci.test/dir" :project/test-paths]}
+  :aliases {"test" ["run" "-m" "circleci.test/dir" :project/test-paths]
+            "retest" ["run" "-m" "circleci.test.retest"]}
   :profiles {:uberjar {:aot :all}})

src/circleci/test.clj

@@ -1,7 +1,8 @@
 (ns circleci.test
   (:require [circleci.test.report :as report]
             [clojure.test :as test]
-            [clojure.java.io :as io]))
+            [clojure.java.io :as io])
+  (:import (clojure.lang LineNumberingPushbackReader)))
 
 ;; Once fixtures should be run exactly once regardless of which entry-point
 ;; into circleci.test is used
@@ -23,10 +24,14 @@
 ;; dummy once-fixture.
 (def ^:dynamic *once-fixtures* {})
 
-(defn- read-config! []
-  (if-let [r (io/resource "circleci_test/config.clj")]
-    (load-reader (clojure.lang.LineNumberingPushbackReader. (io/reader r)))
-    {}))
+(def ^:private default-config {:test-results-dir "test-results"
+                               :reporters [report/clojure-test-reporter]})
+
+(defn read-config! []
+  (let [config (if-let [r (io/resource "circleci_test/config.clj")]
+                 (load-reader (LineNumberingPushbackReader. (io/reader r)))
+                 {})]
+    (merge default-config config)))
 
 (defn- make-once-fixture-fn
   [ns]
@@ -53,6 +58,11 @@
 
 ;; Running tests; low-level fns
 
+(defn- get-reporters [config]
+  (or report/*reporters*
+      (for [make-reporter (:reporters config)]
+        (make-reporter config))))
+
 (defn- nanos->seconds
   [nanos]
   (/ nanos 1e9))
@@ -105,7 +115,7 @@
    ;; entrypoints into the test runner
    (binding [test/test-var (partial test-var* config)
              test/report report/report
-             report/*reporters* (:reporters config report/*reporters*)]
+             report/*reporters* (get-reporters config)]
      (test-var* config v))))
 
 
@@ -129,7 +139,7 @@
   ([ns selector config]
    (binding [test/*report-counters* (ref test/*initial-report-counters*)
              test/report report/report
-             report/*reporters* (:reporters config report/*reporters*)]
+             report/*reporters* (get-reporters config)]
      (let [ns-obj (the-ns ns)
            global-fixture-fn (make-global-fixture config)
            once-fixture-fn (once-fixtures ns-obj)]

src/circleci/test/report.clj

@@ -81,15 +81,14 @@
 
   (end-test-var [this m]))
 
-(defn clojure-test-reporter
-  []
+(defn clojure-test-reporter [_config]
   (->ClojureDotTestReporter))
 
 (def ^:dynamic *reporters*
   "The sequence of currently configured test reporters. Consumers of the
   circleci.test library should not bind this var, set the :reporters key in
   test runner config instead."
-  [(clojure-test-reporter)])
+  nil)
 
 ;; Test result reporting
 (defmulti

src/circleci/test/report/junit.clj

@@ -112,5 +112,5 @@
 (defn reporter
   "Create a TestReporter that writes Junit compatible XML reports to
   `out-dir`."
-  [out-dir]
-  (->JunitReporter out-dir (atom []) (atom [])))
+  [config]
+  (->JunitReporter (:test-results-dir config) (atom []) (atom [])))

src/circleci/test/retest.clj

@@ -0,0 +1,35 @@
+(ns circleci.test.retest
+  (:require [clojure.data.xml :as xml]
+            [clojure.java.io :as io]
+            [circleci.test :as test]))
+
+(defn- failed-tests-from [root]
+  (let [suite (:name (:attrs root))]
+    (for [test-case (:content root)
+          :when (some #(= :failure (:tag %)) (:content test-case))]
+      (symbol suite (:name (:attrs test-case))))))
+
+(def ^:private missing-junit-msg
+  "Previous test report files not found. Is the junit reporter configured?")
+
+(defn- failed-tests [report-dir]
+  (when-not (.exists (io/file report-dir))
+    (throw (Exception. missing-junit-msg)))
+  (for [report-file (.listFiles (io/file report-dir))
+        test (failed-tests-from (xml/parse (io/reader report-file)))]
+    test))
+
+(defn- make-selector [tests]
+  (comp (set (map name tests)) str :name))
+
+(defn -main []
+  (let [{:keys [test-results-dir] :as config} (test/read-config!)]
+    (when-not (.exists (io/file test-results-dir))
+      (binding [*out* *err*]
+        (println missing-junit-msg)
+        (System/exit 1)))
+    (if-let [failed (seq (failed-tests test-results-dir))]
+      (doseq [[test-ns tests] (group-by (comp symbol namespace) failed)]
+        (require test-ns)
+        (test/test-ns test-ns (make-selector tests) config))
+      (println "No failed tests."))))

test/circleci/test/report/test_junit.clj

@@ -49,5 +49,5 @@
   ;; `lein test` assumes tests return a map.
   (let [tempdir (Files/createTempDirectory "circleci.test.report.test-junit" (into-array FileAttribute []))]
     (-> tempdir .toFile .deleteOnExit)
-    (binding [circleci.test.report/*reporters* [(junit/reporter (str tempdir))]]
+    (binding [circleci.test.report/*reporters* [(junit/reporter {:test-results-dir (str tempdir)})]]
       (is (map? (circleci.test/run-tests 'circleci.test-config))))))