diff --git a/.github/workflows/build-sbt.yml b/.github/workflows/build-sbt.yml index b9bcfdf..9512e17 100644 --- a/.github/workflows/build-sbt.yml +++ b/.github/workflows/build-sbt.yml @@ -39,7 +39,11 @@ jobs: name: SBT Spark ${{matrix.spark}} on Scala ${{matrix.scala}} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 #v8.1.0 - name: Setup Scala uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c #v14 diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index f5babf2..a267a09 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -44,7 +44,11 @@ jobs: changed: 80.0 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Scala uses: olafurpg/setup-scala@v10 with: diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index ddddf93..e6f20fe 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -26,7 +26,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Scala uses: olafurpg/setup-scala@v10 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..15b8504 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +# +# Copyright 2018 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Release +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Scala + uses: olafurpg/setup-scala@32ffa16635ff8f19cc21ea253a987f0fdf29844c #v14 + with: + java-version: "adopt@1.8" + + - run: sbt ci-release + env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} diff --git a/README.md b/README.md index 4aaded9..d8ddf86 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # About Atum -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/za.co.absa/atum_2.11/badge.svg)](https://maven-badges.herokuapp.com/maven-central/za.co.absa/atum_2.11/) +[![Maven Central](https://img.shields.io/maven-central/v/za.co.absa.atum-service/atum-model_2.12)](https://central.sonatype.com/artifact/za.co.absa/atum_2.12/) Atum is a data completeness and accuracy library for Apache Spark. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..c7f467e --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,23 @@ +# Release + +Releases of this library are currently handled by [sbt-ci-release](https://github.com/sbt/sbt-ci-release). +Please see its documentation for more details about how it works if you are interested to know more. + +The actual deployments are triggered manually by the maintainers of this repository, using `workflow_dispatch` event +trigger. + +Once changes from a PR were reviewed and merged into the master branch, follow these steps: +1. Create a new Git Tag at the desired commit on `master`, and push the tag to the repository. For example, + if you want to release a version 0.4.0 (note that we are using [Semantic Versioning](https://semver.org/)): + + ```shell + git tag -a v0.4.0 -m "v0.4.0" + git push origin v0.4.0 + ``` + +2. In GitHub UI, go to the repository's **Actions** -> **Release** -> **Run workflow**, and under **Use workflow from** + use **Tags** and find the tag you created in the previous step. + + > **Important note**: don't run the workflow against the master branch, but against the tag. + > `sbt-ci-release` plugin won't be able to correctly find tag, and it will think that you are trying + > to do a snapshot release, not an actual release that should be synchronized with Maven Central. diff --git a/atum/src/main/scala/za/co/absa/atum/utils/SparkTestBase.scala b/atum/src/main/scala/za/co/absa/atum/utils/SparkTestBase.scala index ef5fc94..22e8609 100644 --- a/atum/src/main/scala/za/co/absa/atum/utils/SparkTestBase.scala +++ b/atum/src/main/scala/za/co/absa/atum/utils/SparkTestBase.scala @@ -34,6 +34,8 @@ trait SparkTestBase { .config("spark.driver.host", "127.0.0.1") .config("spark.ui.enabled", "false") .config("spark.testing.memory", 1024*1024*1024) // otherwise may fail based on local machine settings + // Spark 3.2+ fires listeners async by default; sync needed for _INFO file tests + .config("spark.sql.queryExecutionListeners.async", value = false) .getOrCreate() } diff --git a/atum/src/test/scala/za/co/absa/atum/ControlMeasureBaseTestSuite.scala b/atum/src/test/scala/za/co/absa/atum/ControlMeasureBaseTestSuite.scala index 353609a..1d26fcc 100644 --- a/atum/src/test/scala/za/co/absa/atum/ControlMeasureBaseTestSuite.scala +++ b/atum/src/test/scala/za/co/absa/atum/ControlMeasureBaseTestSuite.scala @@ -36,7 +36,7 @@ trait ControlMeasureBaseTestSuite { .replaceFirst("""(?<="informationDate"\s?:\s?")(\d{2}-\d{2}-\d{4})""", testingDate) .replaceAll("""(?<="processStartTime"\s?:\s?")([-+: \d]+)""", testingDateTime1) .replaceAll("""(?<="processEndTime"\s?:\s?")([-+: \d]+)""", testingDateTime2) - .replaceAll("""(?<="version"\s?:\s?")([-\d\.A-z]+)""", testingVersion) + .replaceAll("""(?<="version"\s?:\s?")([-+\d\.A-z]+)""", testingVersion) .replaceAll("""(?<="software"\s?:\s?")([\d\.A-z_]+)""", testingSoftware) .replaceAll("\r\n", "\n") // Windows guard } diff --git a/build.sbt b/build.sbt index dc08112..9b6d21d 100644 --- a/build.sbt +++ b/build.sbt @@ -27,6 +27,8 @@ lazy val scala213 = "2.13.14" ThisBuild / scalaVersion := scala211 // default version ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213) +ThisBuild / versionScheme := Some("early-semver") + lazy val printSparkScalaVersion = taskKey[Unit]("Print Spark and Scala versions that Atum is being built for.") ThisBuild / printSparkScalaVersion := { val log = streams.value.log diff --git a/project/build.properties b/project/build.properties index 683dc3d..0cef0a5 100644 --- a/project/build.properties +++ b/project/build.properties @@ -13,4 +13,4 @@ # limitations under the License. # -sbt.version = 1.9.9 +sbt.version=1.11.5 diff --git a/project/plugins.sbt b/project/plugins.sbt index e3040b0..d71c440 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -13,12 +13,12 @@ * limitations under the License. */ +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") + addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0") -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") // formerly known as com.jsuereth:sbt-pgp - // sbt-jacoco - workaround related dependencies required to download lazy val ow2Version = "9.5" lazy val jacocoVersion = "0.8.10-absa.1" diff --git a/publish.sbt b/publish.sbt index 4a48b06..b1f2dce 100644 --- a/publish.sbt +++ b/publish.sbt @@ -14,10 +14,6 @@ * limitations under the License. */ -// to successfully publish to Sonatype OSS (using sbt publishSigned), setup outline at -// https://www.scala-sbt.org/1.x/docs/Using-Sonatype.html is expected - -ThisBuild / organizationName := "ABSA Group Limited" ThisBuild / organizationHomepage := Some(url("https://www.absa.africa")) ThisBuild / scmInfo := Some( ScmInfo( @@ -39,21 +35,18 @@ ThisBuild / developers := List( name = "Ruslan Iushchenko", email = "ruslan.iushchenko@absa.africa", url = url("https://github.com/yruslan") + ), + Developer( + id = "lsulak", + name = "Ladislav Sulak", + email = "ladislav.sulak@absa.africa", + url = url("https://github.com/lsulak") ) ) ThisBuild / homepage := Some(url("https://github.com/AbsaOSS/atum")) ThisBuild / description := "Dynamic data completeness and accuracy at enterprise scale in Apache Spark" + +ThisBuild / organizationName := "ABSA Group Limited" ThisBuild / startYear := Some(2018) ThisBuild / licenses += "Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.txt") - -ThisBuild / pomIncludeRepository := { _ => false } -ThisBuild / publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) { - Some("snapshots" at s"${nexus}content/repositories/snapshots") - } else { - Some("releases" at s"${nexus}service/local/staging/deploy/maven2") - } -} -ThisBuild / publishMavenStyle := true diff --git a/version.sbt b/version.sbt deleted file mode 100644 index 8a6218b..0000000 --- a/version.sbt +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2018 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -ThisBuild / version := "3.10.1-SNAPSHOT"