#162: release automation and documentation update

.github/workflows/build-sbt.yml

@@ -39,7 +39,11 @@ jobs:
     name: SBT Spark ${{matrix.spark}} on Scala ${{matrix.scala}}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
       - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 #v8.1.0
       - name: Setup Scala
         uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c #v14

.github/workflows/jacoco_check.yml

@@ -44,7 +44,11 @@ jobs:
             changed: 80.0
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
       - name: Setup Scala
         uses: olafurpg/setup-scala@v10
         with:

.github/workflows/license_check.yml

@@ -26,7 +26,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
       - name: Setup Scala
         uses: olafurpg/setup-scala@v10
         with:

.github/workflows/release.yml

@@ -0,0 +1,41 @@
+#
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Release
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c #v14
+        with:
+          java-version: "adopt@1.8"
+
+      - run: sbt ci-release
+        env:
+          PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+          PGP_SECRET: ${{ secrets.PGP_SECRET }}
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}

README.md

@@ -1,6 +1,6 @@
 # About Atum
 
-[![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/atum_2.11/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/atum_2.11/)
+[![Maven Central](https://img.shields.io/maven-central/v/za.co.absa.atum-service/atum-model_2.12)](https://central.sonatype.com/artifact/za.co.absa/atum_2.12/)
 
 Atum is a data completeness and accuracy library for Apache Spark.
 

RELEASE.md

@@ -0,0 +1,23 @@
+# Release
+
+Releases of this library are currently handled by [sbt-ci-release](https://github.com/sbt/sbt-ci-release).
+Please see its documentation for more details about how it works if you are interested to know more.
+
+The actual deployments are triggered manually by the maintainers of this repository, using `workflow_dispatch` event
+trigger.
+
+Once changes from a PR were reviewed and merged into the master branch, follow these steps:
+1. Create a new Git Tag at the desired commit on `master`, and push the tag to the repository. For example,
+   if you want to release a version 0.4.0 (note that we are using [Semantic Versioning](https://semver.org/)):
+
+    ```shell
+    git tag -a v0.4.0 -m "v0.4.0"
+    git push origin v0.4.0
+    ```
+
+2. In GitHub UI, go to the repository's **Actions** -> **Release** -> **Run workflow**, and under **Use workflow from**
+   use **Tags** and find the tag you created in the previous step.
+
+   > **Important note**: don't run the workflow against the master branch, but against the tag.
+   > `sbt-ci-release` plugin won't be able to correctly find tag, and it will think that you are trying
+   > to do a snapshot release, not an actual release that should be synchronized with Maven Central.

atum/src/main/scala/za/co/absa/atum/utils/SparkTestBase.scala

@@ -34,6 +34,8 @@ trait SparkTestBase {
     .config("spark.driver.host", "127.0.0.1")
     .config("spark.ui.enabled", "false")
     .config("spark.testing.memory", 1024*1024*1024) // otherwise may fail based on local machine settings
+    // Spark 3.2+ fires listeners async by default; sync needed for _INFO file tests
+    .config("spark.sql.queryExecutionListeners.async", value = false)
     .getOrCreate()
 
 }

atum/src/test/scala/za/co/absa/atum/ControlMeasureBaseTestSuite.scala

@@ -36,7 +36,7 @@ trait ControlMeasureBaseTestSuite {
       .replaceFirst("""(?<="informationDate"\s?:\s?")(\d{2}-\d{2}-\d{4})""", testingDate)
       .replaceAll("""(?<="processStartTime"\s?:\s?")([-+: \d]+)""", testingDateTime1)
       .replaceAll("""(?<="processEndTime"\s?:\s?")([-+: \d]+)""", testingDateTime2)
-      .replaceAll("""(?<="version"\s?:\s?")([-\d\.A-z]+)""", testingVersion)
+      .replaceAll("""(?<="version"\s?:\s?")([-+\d\.A-z]+)""", testingVersion)
       .replaceAll("""(?<="software"\s?:\s?")([\d\.A-z_]+)""", testingSoftware)
       .replaceAll("\r\n", "\n") // Windows guard
   }

build.sbt

@@ -27,6 +27,8 @@ lazy val scala213 = "2.13.14"
 ThisBuild / scalaVersion := scala211  // default version
 ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213)
 
+ThisBuild / versionScheme := Some("early-semver")
+
 lazy val printSparkScalaVersion = taskKey[Unit]("Print Spark and Scala versions that Atum is being built for.")
 ThisBuild / printSparkScalaVersion := {
   val log = streams.value.log

project/build.properties

@@ -13,4 +13,4 @@
 # limitations under the License.
 #
 
-sbt.version = 1.9.9
+sbt.version=1.11.5

project/plugins.sbt

@@ -13,12 +13,12 @@
  * limitations under the License.
  */
 
+addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2")
+
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
 
 addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0")
 
-addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") // formerly known as com.jsuereth:sbt-pgp
-
 // sbt-jacoco - workaround related dependencies required to download
 lazy val ow2Version = "9.5"
 lazy val jacocoVersion = "0.8.10-absa.1"

publish.sbt

@@ -14,10 +14,6 @@
  * limitations under the License.
  */
 
-// to successfully publish to Sonatype OSS (using sbt publishSigned), setup outline at
-// https://www.scala-sbt.org/1.x/docs/Using-Sonatype.html is expected
-
-ThisBuild / organizationName := "ABSA Group Limited"
 ThisBuild / organizationHomepage := Some(url("https://www.absa.africa"))
 ThisBuild / scmInfo := Some(
   ScmInfo(
@@ -39,21 +35,18 @@ ThisBuild / developers := List(
     name  = "Ruslan Iushchenko",
     email = "ruslan.iushchenko@absa.africa",
     url   = url("https://github.com/yruslan")
+  ),
+  Developer(
+    id    = "lsulak",
+    name  = "Ladislav Sulak",
+    email = "ladislav.sulak@absa.africa",
+    url   = url("https://github.com/lsulak")
   )
 )
 
 ThisBuild / homepage := Some(url("https://github.com/AbsaOSS/atum"))
 ThisBuild / description := "Dynamic data completeness and accuracy at enterprise scale in Apache Spark"
+
+ThisBuild / organizationName := "ABSA Group Limited"
 ThisBuild / startYear := Some(2018)
 ThisBuild / licenses += "Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt")
-
-ThisBuild / pomIncludeRepository := { _ => false }
-ThisBuild / publishTo := {
-  val nexus = "https://oss.sonatype.org/"
-  if (isSnapshot.value) {
-    Some("snapshots" at s"${nexus}content/repositories/snapshots")
-  } else {
-    Some("releases" at s"${nexus}service/local/staging/deploy/maven2")
-  }
-}
-ThisBuild / publishMavenStyle := true