Merge pull request #1350 from hyperledger/develop

Hyperledger Iroha v1.0.0 beta-2

Author: bakhtin

.jenkinsci/artifacts.groovy

@@ -0,0 +1,44 @@
+#!/usr/bin/env groovy
+
+def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.co.jp']) {
+  def baseUploadPath = 'files'
+  def filePathsConverted = []
+  agentType = sh(script: 'uname', returnStdout: true).trim()
+  uploadPath = baseUploadPath + uploadPath
+  filePaths.each {
+    fp = sh(script: "ls -d ${it} | tr '\n' ','", returnStdout: true).trim()
+    filePathsConverted.addAll(fp.split(','))
+  }
+  def shaSumBinary = 'sha256sum'
+  def md5SumBinary = 'md5sum'
+  if (agentType == 'Darwin') {
+    shaSumBinary = 'shasum -a 256'
+    md5SumBinary = 'md5 -r'
+  }
+  sh "> \$(pwd)/batch.txt"
+  filePathsConverted.each {
+    sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
+    sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
+    sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
+    sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
+    sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
+  }
+  // mkdirs recursively
+  uploadPath = uploadPath.split('/')
+  def p = ''
+  sh "> \$(pwd)/mkdirs.txt"
+  uploadPath.each {
+    p += "/${it}"
+    sh("echo -mkdir $p >> \$(pwd)/mkdirs.txt")
+  }
+
+  sshagent(['jenkins-artifact']) {
+    sh "ssh-agent"
+    artifactServers.each {
+      sh "sftp -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
+      sh "sftp -b \$(pwd)/batch.txt jenkins@${it}"
+    }
+  }
+}
+
+return this

.jenkinsci/bindings.groovy

@@ -1,27 +1,74 @@
 #!/usr/bin/env groovy
 
-def doBindings() {
-  def cmake_options = ""
-  if (params.JavaBindings) {
-    cmake_options += " -DSWIG_JAVA=ON "
-  }
-  if (params.PythonBindings) {
-    cmake_options += " -DSWIG_PYTHON=ON "
-  }
-  // In case language specific options were not set,
-  // build for each language
-  if (!params.JavaBindings && !params.PythonBindings) {
-    cmake_options += " -DSWIG_JAVA=ON -DSWIG_PYTHON=ON "
-  }
+def doJavaBindings(buildType=Release) {
+  def currentPath = sh(script: "pwd", returnStdout: true).trim()
+  def commit = env.GIT_COMMIT
+  def artifactsPath = sprintf('%1$s/java-bindings-%2$s-%3$s-%4$s.zip', 
+    [currentPath, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
   sh """
     cmake \
       -H. \
       -Bbuild \
-      -DCMAKE_BUILD_TYPE=Release \
-      ${cmake_options}
+      -DCMAKE_BUILD_TYPE=$buildType \
+      -DSWIG_JAVA=ON
+  """
+  sh "cd build; make -j${params.PARALLELISM} irohajava"
+  sh "zip -j $artifactsPath build/shared_model/bindings/*.java build/shared_model/bindings/libirohajava.so"
+  sh "cp $artifactsPath /tmp/bindings-artifact"
+  return artifactsPath
+}
+
+def doPythonBindings(buildType=Release) {
+  def currentPath = sh(script: "pwd", returnStdout: true).trim()
+  def commit = env.GIT_COMMIT
+  def supportPython2 = "OFF"
+  def artifactsPath = sprintf('%1$s/python-bindings-%2$s-%3$s-%4$s-%5$s.zip', 
+    [currentPath, env.PBVersion, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
+  // do not use preinstalled libed25519
+  sh "rm -rf /usr/local/include/ed25519*; unlink /usr/local/lib/libed25519.so; rm -f /usr/local/lib/libed25519.so.1.2.2"
+  if (env.PBVersion == "python2") { supportPython2 = "ON" }
+  sh """
+    cmake \
+      -H. \
+      -Bbuild \
+      -DCMAKE_BUILD_TYPE=$buildType \
+      -DSWIG_PYTHON=ON \
+      -DSUPPORT_PYTHON2=$supportPython2
   """
   sh "cmake --build build --target python_tests"
-  sh "cd build; make -j${params.PARALLELISM} irohajava irohapy"
+  sh "cd build; make -j${params.PARALLELISM} irohapy"
+  sh "protoc --proto_path=schema --python_out=build/shared_model/bindings block.proto primitive.proto commands.proto queries.proto responses.proto endpoint.proto"
+  sh "${env.PBVersion} -m grpc_tools.protoc --proto_path=schema --python_out=build/shared_model/bindings --grpc_python_out=build/shared_model/bindings endpoint.proto yac.proto ordering.proto loader.proto"
+  sh "zip -j $artifactsPath build/shared_model/bindings/*.py build/shared_model/bindings/*.so"
+  sh "cp $artifactsPath /tmp/bindings-artifact"
+  return artifactsPath
+}
+
+def doAndroidBindings(abiVersion) {
+  def currentPath = sh(script: "pwd", returnStdout: true).trim()
+  def commit = env.GIT_COMMIT
+  def artifactsPath = sprintf('%1$s/android-bindings-%2$s-%3$s-%4$s-%5$s-%6$s.zip', 
+    [currentPath, "\$PLATFORM", abiVersion, "\$BUILD_TYPE_A", sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
+  sh """
+    (cd /iroha; git init; git remote add origin https://github.com/hyperledger/iroha.git; \
+    git fetch --depth 1 origin develop; git checkout -t origin/develop)
+  """
+  sh """
+    . /entrypoint.sh; \
+    sed -i.bak "s~find_package(JNI REQUIRED)~SET(CMAKE_SWIG_FLAGS \\\${CMAKE_SWIG_FLAGS} -package \${PACKAGE})~" /iroha/shared_model/bindings/CMakeLists.txt; \
+    # TODO: might not be needed in the future
+    sed -i.bak "/target_include_directories(\\\${SWIG_MODULE_irohajava_REAL_NAME} PUBLIC/,+3d" /iroha/shared_model/bindings/CMakeLists.txt; \
+    sed -i.bak "s~swig_link_libraries(irohajava~swig_link_libraries(irohajava \"/protobuf/.build/lib\${PROTOBUF_LIB_NAME}.a\" \"\${NDK_PATH}/platforms/android-$abiVersion/\${ARCH}/usr/\${LIBP}/liblog.so\"~" /iroha/shared_model/bindings/CMakeLists.txt; \
+    sed -i.bak "s~find_library(protobuf_LIBRARY protobuf)~find_library(protobuf_LIBRARY \${PROTOBUF_LIB_NAME})~" /iroha/cmake/Modules/Findprotobuf.cmake; \
+    sed -i.bak "s~find_program(protoc_EXECUTABLE protoc~set(protoc_EXECUTABLE \"/protobuf/host_build/protoc\"~" /iroha/cmake/Modules/Findprotobuf.cmake; \
+    cmake -H/iroha/shared_model -B/iroha/shared_model/build -DCMAKE_SYSTEM_NAME=Android -DCMAKE_SYSTEM_VERSION=$abiVersion -DCMAKE_ANDROID_ARCH_ABI=\$PLATFORM \
+      -DANDROID_NDK=\$NDK_PATH -DCMAKE_ANDROID_STL_TYPE=c++_static -DCMAKE_BUILD_TYPE=\$BUILD_TYPE_A -DTESTING=OFF \
+      -DSHARED_MODEL_DISABLE_COMPATIBILITY=ON -DSWIG_JAVA=ON -DCMAKE_PREFIX_PATH=\$DEPS_DIR
+    """
+  sh "cmake --build /iroha/shared_model/build --target irohajava -- -j${params.PARALLELISM}"
+  sh "zip -j $artifactsPath /iroha/shared_model/build/bindings/*.java /iroha/shared_model/build/bindings/libirohajava.so"
+  sh "cp $artifactsPath /tmp/bindings-artifact"
+  return artifactsPath
 }
 
 return this

.jenkinsci/debug-build.groovy

@@ -1,106 +1,115 @@
 #!/usr/bin/env groovy
 
 def doDebugBuild(coverageEnabled=false) {
+  def dPullOrBuild = load ".jenkinsci/docker-pull-or-build.groovy"
+  def manifest = load ".jenkinsci/docker-manifest.groovy"
+  def pCommit = load ".jenkinsci/previous-commit.groovy"
   def parallelism = params.PARALLELISM
+  def platform = sh(script: 'uname -m', returnStdout: true).trim()
+  def previousCommit = pCommit.previousCommitOrCurrent()
   // params are always null unless job is started
   // this is the case for the FIRST build only.
   // So just set this to same value as default. 
   // This is a known bug. See https://issues.jenkins-ci.org/browse/JENKINS-41929
-  if (parallelism == null) {
+  if (!parallelism) {
     parallelism = 4
   }
-  if ("arm7" in env.NODE_NAME) {
+  if (env.NODE_NAME.contains('arm7')) {
     parallelism = 1
   }
   sh "docker network create ${env.IROHA_NETWORK}"
+  def iC = dPullOrBuild.dockerPullOrUpdate("${platform}-develop-build",
+                                           "${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
+                                           "${env.GIT_RAW_BASE_URL}/${previousCommit}/docker/develop/Dockerfile",
+                                           "${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
+                                           ['PARALLELISM': parallelism])
 
-  docker.image('postgres:9.5').run(""
+  if (GIT_LOCAL_BRANCH == 'develop' && manifest.manifestSupportEnabled()) {
+    manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop-build", 
+      ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build", 
+       "${DOCKER_REGISTRY_BASENAME}:armv7l-develop-build", 
+       "${DOCKER_REGISTRY_BASENAME}:aarch64-develop-build"])
+    manifest.manifestAnnotate("${DOCKER_REGISTRY_BASENAME}:develop-build",
+      [
+        [manifest: "${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build",
+         arch: 'amd64', os: 'linux', osfeatures: [], variant: ''],
+        [manifest: "${DOCKER_REGISTRY_BASENAME}:armv7l-develop-build",
+         arch: 'arm', os: 'linux', osfeatures: [], variant: 'v7'],
+        [manifest: "${DOCKER_REGISTRY_BASENAME}:aarch64-develop-build",
+         arch: 'arm64', os: 'linux', osfeatures: [], variant: '']
+      ])
+    withCredentials([usernamePassword(credentialsId: 'docker-hub-credentials', usernameVariable: 'login', passwordVariable: 'password')]) {
+      manifest.manifestPush("${DOCKER_REGISTRY_BASENAME}:develop-build", login, password)
+    }
+  }
+  docker.image('postgres:9.5').withRun(""
     + " -e POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
     + " -e POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
     + " --name ${env.IROHA_POSTGRES_HOST}"
-    + " --network=${env.IROHA_NETWORK}")
-
-  def platform = sh(script: 'uname -m', returnStdout: true).trim()
-  sh "curl -L -o /tmp/${env.GIT_COMMIT}/Dockerfile --create-dirs https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/develop/${platform}/Dockerfile"
-  // pull docker image in case we don't have one
-  // speeds up consequent image builds as we simply tag them
-  sh "docker pull ${DOCKER_BASE_IMAGE_DEVELOP}"
-  if (env.BRANCH_NAME == 'develop') {
-    iC = docker.build("hyperledger/iroha:${GIT_COMMIT}-${BUILD_NUMBER}", "--build-arg PARALLELISM=${parallelism} -f /tmp/${env.GIT_COMMIT}/Dockerfile /tmp/${env.GIT_COMMIT}")
-    docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
-      iC.push("${platform}-develop")
-    }
-  }
-  else {
-    iC = docker.build("hyperledger/iroha-workflow:${GIT_COMMIT}-${BUILD_NUMBER}", "-f /tmp/${env.GIT_COMMIT}/Dockerfile /tmp/${env.GIT_COMMIT} --build-arg PARALLELISM=${parallelism}")
-  }
-  iC.inside(""
-    + " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
-    + " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
-    + " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
-    + " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
-    + " --network=${env.IROHA_NETWORK}"
-    + " -v /var/jenkins/ccache:${CCACHE_DIR}") {
-
-    def scmVars = checkout scm
-    def cmakeOptions = ""
-    if ( coverageEnabled ) {
-      cmakeOptions = " -DCOVERAGE=ON "
-    }
-    env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
-    env.IROHA_HOME = "/opt/iroha"
-    env.IROHA_BUILD = "${env.IROHA_HOME}/build"
+    + " --network=${env.IROHA_NETWORK}") {
+      iC.inside(""
+      + " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
+      + " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+      + " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+      + " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+      + " --network=${env.IROHA_NETWORK}"
+      + " -v /var/jenkins/ccache:${CCACHE_DIR}"
+      + " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {
 
-    sh """
-      ccache --version
-      ccache --show-stats
-      ccache --zero-stats
-      ccache --max-size=5G
-    """  
-    sh """
-      cmake \
-        -DTESTING=ON \
-        -H. \
-        -Bbuild \
-        -DCMAKE_BUILD_TYPE=Debug \
-        -DIROHA_VERSION=${env.IROHA_VERSION} \
-        ${cmakeOptions}
-    """
-    sh "cmake --build build -- -j${parallelism}"
-    sh "ccache --show-stats"
-    if ( coverageEnabled ) {
-      sh "cmake --build build --target coverage.init.info"
-    }
-    def testExitCode = sh(script: 'cmake --build build --target test', returnStatus: true)
-    if (testExitCode != 0) {
-      currentBuild.result = "UNSTABLE"
-    }
-    if ( coverageEnabled ) {
-      sh "cmake --build build --target cppcheck"
-      // Sonar
-      if (env.CHANGE_ID != null) {
-        sh """
-          sonar-scanner \
-            -Dsonar.github.disableInlineComments \
-            -Dsonar.github.repository='hyperledger/iroha' \
-            -Dsonar.analysis.mode=preview \
-            -Dsonar.login=${SONAR_TOKEN} \
-            -Dsonar.projectVersion=${BUILD_TAG} \
-            -Dsonar.github.oauth=${SORABOT_TOKEN} \
-            -Dsonar.github.pullRequest=${CHANGE_ID}
-        """
+      def scmVars = checkout scm
+      def cmakeOptions = ""
+      if ( coverageEnabled ) {
+        cmakeOptions = " -DCOVERAGE=ON "
       }
+      env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
+      env.IROHA_HOME = "/opt/iroha"
+      env.IROHA_BUILD = "${env.IROHA_HOME}/build"
 
-      sh "cmake --build build --target coverage.info"
-      sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
-      cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
-    }
-
-    // TODO: replace with upload to artifactory server
-    // develop branch only
-    if ( env.BRANCH_NAME == "develop" ) {
-      //archive(includes: 'build/bin/,compile_commands.json')
+      sh """
+        ccache --version
+        ccache --show-stats
+        ccache --zero-stats
+        ccache --max-size=5G
+      """  
+      sh """
+        cmake \
+          -DTESTING=ON \
+          -H. \
+          -Bbuild \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DIROHA_VERSION=${env.IROHA_VERSION} \
+          ${cmakeOptions}
+      """
+      sh "cmake --build build -- -j${parallelism}"
+      sh "ccache --show-stats"
+      if ( coverageEnabled ) {
+        sh "cmake --build build --target coverage.init.info"
+      }
+      def testExitCode = sh(script: 'CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test', returnStatus: true)
+      if (testExitCode != 0) {
+        currentBuild.result = "UNSTABLE"
+      }
+      if ( coverageEnabled ) {
+        sh "cmake --build build --target cppcheck"
+        // Sonar
+        if (env.CHANGE_ID != null) {
+          sh """
+            sonar-scanner \
+              -Dsonar.github.disableInlineComments \
+              -Dsonar.github.repository='${DOCKER_REGISTRY_BASENAME}' \
+              -Dsonar.analysis.mode=preview \
+              -Dsonar.login=${SONAR_TOKEN} \
+              -Dsonar.projectVersion=${BUILD_TAG} \
+              -Dsonar.github.oauth=${SORABOT_TOKEN} \
+              -Dsonar.github.pullRequest=${CHANGE_ID}
+          """
+        }
+        sh "cmake --build build --target coverage.info"
+        sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
+        cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
+      }
     }
   }
 }
+
 return this

.jenkinsci/docker-manifest.groovy

@@ -0,0 +1,28 @@
+#!/usr/bin/env groovy
+
+def manifestSupportEnabled() {
+	def dockerVersion = sh(script: "docker -v", returnStdout: true).trim()
+	def experimentalEnabled = sh(script: "grep -i experimental ~/.docker/config.json", returnStatus: true)
+	return experimentalEnabled == 0 && dockerVersion ==~ /^Docker version 18.*$/
+
+}
+
+def manifestCreate(manifestListName, manifests) {
+	sh "docker manifest create ${manifestListName} ${manifests.join(' ')}"
+}
+
+def manifestAnnotate(manifestListName, manifestsWithFeatures) {
+	manifestsWithFeatures.each {
+		sh """
+			docker manifest annotate ${manifestListName} ${it['manifest']} --arch "${it['arch']}" \
+			--os "${it['os']}" --os-features "${it['osfeatures'].join(',')}" --variant "${it['variant']}"
+		"""
+	}
+}
+
+def manifestPush(manifestListName, dockerRegistryLogin, dockerRegistryPassword) {
+	sh "docker login -u '${dockerRegistryLogin}' -p '${dockerRegistryPassword}'"
+	sh "docker manifest push --purge ${manifestListName}"
+}
+
+return this