Remove unsafe dereference operations

src/main/kotlin/io/thelandscape/krawler/crawler/Krawler.kt

@@ -52,35 +52,38 @@ import java.util.concurrent.atomic.AtomicInteger
  *
  */
 abstract class Krawler(val config: KrawlConfig = KrawlConfig(),
-                       private var krawlHistory: KrawlHistoryIf? = null,
-                       private var krawlQueues: List<KrawlQueueIf>? = null,
+                       krawlHistory: KrawlHistoryIf? = null,
+                       krawlQueues: List<KrawlQueueIf>? = null,
                        robotsConfig: RobotsConfig? = null,
                        private val requestProvider: RequestProviderIf = Requests(config),
                        private val job: Job = Job()) {
 
     private val logger: Logger = LogManager.getLogger()
+    private val krawlHistory: KrawlHistoryIf
+    private val krawlQueues: List<KrawlQueueIf>
 
     // Map of start URL -> int id to track branches of a crawl
     private val rootPageIds: MutableMap<String, Int> = mutableMapOf()
     // Current Max ID
     private val maximumUsedId: AtomicInteger = AtomicInteger(0)
 
     init {
-        if (krawlHistory == null || krawlQueues == null) {
+        this.krawlHistory = if (krawlHistory == null) {
             val hsqlConnection = HSQLConnection(config.persistentCrawl, config.crawlDirectory)
+            KrawlHistoryHSQLDao(hsqlConnection.hsqlSession)
+        } else {
+            krawlHistory
+        }
 
-            if (krawlHistory == null)
-                krawlHistory = KrawlHistoryHSQLDao(hsqlConnection.hsqlSession)
-
+        this.krawlQueues = if (krawlQueues == null) {
             // This is safe because we don't have any KrawlHistoryIf implementations other than HSQL
-            val histDao: KrawlHistoryHSQLDao = krawlHistory as KrawlHistoryHSQLDao
-
-            if (krawlQueues == null)
-                // TODO: Dynamic number of queues? Why 10?
-                krawlQueues = (0 until 10).map {
-                    KrawlQueueHSQLDao("queue$it", hsqlConnection.hsqlSession, histDao)
-                }
-
+            val histDao: KrawlHistoryHSQLDao = this.krawlHistory as KrawlHistoryHSQLDao
+            // TODO: Dynamic number of queues? Why 10?
+            (0 until 10).map {
+                KrawlQueueHSQLDao("queue$it", this.krawlHistory.session, histDao)
+            }
+        } else {
+            krawlQueues
         }
 
         job.invokeOnCompletion {
@@ -89,7 +92,7 @@ abstract class Krawler(val config: KrawlConfig = KrawlConfig(),
         }
     }
 
-    private val scheduledQueue: ScheduledQueue = ScheduledQueue(krawlQueues!!, config, job)
+    private val scheduledQueue: ScheduledQueue = ScheduledQueue(this.krawlQueues, config, job)
 
     /**
      * Handle robots.txt
@@ -254,7 +257,7 @@ abstract class Krawler(val config: KrawlConfig = KrawlConfig(),
 
         onCrawlStart()
         val urls: Channel<KrawlQueueEntry> = scheduledQueue.krawlQueueEntryChannel
-		repeat(krawlQueues!!.size) {
+		repeat(krawlQueues.size) {
 		    GlobalScope.launch(Dispatchers.Default) {
     		    val actions: ReceiveChannel<KrawlAction> = produceKrawlActions(urls)
 	    		doCrawl(actions)
@@ -368,12 +371,12 @@ abstract class Krawler(val config: KrawlConfig = KrawlConfig(),
 
         // Do a history check
         val history: KrawlHistoryEntry =
-                if (krawlHistory!!.hasBeenSeen(krawlUrl)) { // If it has been seen
+                if (krawlHistory.hasBeenSeen(krawlUrl)) { // If it has been seen
                     onRepeatVisit(krawlUrl, parent)
                     logger.debug("History says no")
                     return@async KrawlAction.Noop
                 } else {
-                    krawlHistory!!.insert(krawlUrl)
+                    krawlHistory.insert(krawlUrl)
                 }
 
         val visit = shouldVisit(krawlUrl)

src/main/kotlin/io/thelandscape/krawler/http/Requests.kt

@@ -80,13 +80,15 @@ internal class HistoryTrackingRedirectStrategy: DefaultRedirectStrategy() {
 private val pcm: PoolingHttpClientConnectionManager = PoolingHttpClientConnectionManager()
 
 class Requests(private val krawlConfig: KrawlConfig,
-               private var httpClient: CloseableHttpClient? = null) : RequestProviderIf {
+               httpClient: CloseableHttpClient? = null) : RequestProviderIf {
 
 	private val logger: Logger = LogManager.getLogger()
+    private val httpClient: CloseableHttpClient
 
     init {
-        if (httpClient == null) {
-            val requestConfig = RequestConfig.custom()
+        this.httpClient =
+            if (httpClient == null) {
+                val requestConfig = RequestConfig.custom()
                     .setCookieSpec(CookieSpecs.STANDARD)
                     .setExpectContinueEnabled(false)
                     .setContentCompressionEnabled(krawlConfig.allowContentCompression)
@@ -96,23 +98,25 @@ class Requests(private val krawlConfig: KrawlConfig,
                     .setSocketTimeout(krawlConfig.socketTimeout)
                     .build()
 
-            val trustStrat = TrustStrategy { _: Array<X509Certificate>, _: String -> true }
+                val trustStrat = TrustStrategy { _: Array<X509Certificate>, _: String -> true }
 
-            val sslContext = SSLContextBuilder.create()
+                val sslContext = SSLContextBuilder.create()
                     .loadTrustMaterial(null, trustStrat)
                     .build()
 
-            val redirectStrategy = HistoryTrackingRedirectStrategy()
+                val redirectStrategy = HistoryTrackingRedirectStrategy()
 
-            httpClient = HttpClients.custom()
+                HttpClients.custom()
                     .setDefaultRequestConfig(requestConfig)
                     .setSSLContext(sslContext)
                     .setSSLHostnameVerifier(NoopHostnameVerifier())
                     .setRedirectStrategy(redirectStrategy)
                     .setUserAgent(krawlConfig.userAgent)
                     .setConnectionManager(pcm)
                     .build()
-        }
+            } else {
+                httpClient
+            }
     }
 
     /** Fetch the robots.txt file from a domain
@@ -186,7 +190,7 @@ class Requests(private val krawlConfig: KrawlConfig,
         }
 
         val resp: RequestResponse = try {
-            val response: HttpResponse? = httpClient!!.execute(req, httpContext)
+            val response: HttpResponse? = httpClient.execute(req, httpContext)
             if (response == null) ErrorResponse(url) else retFun(url, response, httpContext)
         } catch (e: Throwable) {
             ErrorResponse(url, e.toString())

src/main/kotlin/io/thelandscape/krawler/robots/RoboMinder.kt

@@ -90,6 +90,7 @@ class RoboMinder(private val userAgent: String,
             rules.put(url.hierarchicalPart, process(resp))
         }
 
-        return rules.getIfPresent(url.hierarchicalPart)!!.invoke(withoutGetParams)
+        val rule = rules.getIfPresent(url.hierarchicalPart) ?: return true
+        return rule(withoutGetParams)
     }
 }

src/main/kotlin/io/thelandscape/krawler/robots/RobotsTxt.kt

@@ -28,8 +28,7 @@ class RobotsTxt(url: KrawlUrl, response: HttpResponse, context: HttpClientContex
 
     // TODO: Improve handling: https://en.wikipedia.org/wiki/Robots_exclusion_standard#About_the_standard
 
-    var disallowRules: Map<String, MutableSet<String>> = mapOf()
-        private set
+    val disallowRules: Map<String, MutableSet<String>>
 
     // Do all the parsing in a single pass in here
     init {
@@ -47,14 +46,12 @@ class RobotsTxt(url: KrawlUrl, response: HttpResponse, context: HttpClientContex
             if (key == "user-agent") {
                 userAgent = value
                 continue
-            }
-
-            if (key == "disallow") {
-                if (userAgent in rules)
-                    rules[userAgent]!!.add(value)
-                else
+            } else if (key == "disallow") {
+                if (userAgent in rules) {
+                    rules.getOrPut(userAgent, { mutableSetOf() }).add(value)
+                } else {
                     rules[userAgent] = mutableSetOf(value)
-
+                }
                 continue
             }
         }

src/test/kotlin/io/thelandscape/KrawlQueueDaoTest.kt

@@ -64,4 +64,4 @@ class KrawlQueueDaoTest {
         val res = queueDao.deleteByAge(LocalDateTime.now().minusDays(1))
         assertEquals(2, res)
     }
-}
+}