diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c85c31..a78ac69 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,5 @@ name: CI -on: [push] +on: [push, pull_request] jobs: test: strategy: diff --git a/base/deploy b/base/deploy index 6fbcd90..fb04f42 100755 --- a/base/deploy +++ b/base/deploy @@ -39,7 +39,7 @@ function archive_deploy() { rm /tmp/archive.tar.gz if [ -n "$EXPERIMENTAL_DIFF_ENABLED" ]; then if [ -d $CURRENT_DIR ]; then - recursive_diff $CURRENT_DIR $NEW_CODE_DIR > $DIFF_FILE || 2<&0 + recursive_diff $CURRENT_DIR $NEW_CODE_DIR >$DIFF_FILE || 2<&0 fi fi mkdir -p $CURRENT_DIR @@ -60,13 +60,35 @@ function pre_deploy() { } function post_deploy() { - if [ ! -f ${CURRENT_DIR}/Procfile ] && [ -f ${SOURCE_DIR}/default/Procfile ]; then - cp ${SOURCE_DIR}/default/Procfile ${CURRENT_DIR}/Procfile - touch ${APP_DIR}/.default_procfile + if [ ! -f ${CURRENT_DIR}/Procfile ]; then + tsuru_yaml="" + if [ -f ${CURRENT_DIR}/tsuru.yml ]; then + tsuru_yaml="${CURRENT_DIR}/tsuru.yml" + elif [ -f ${CURRENT_DIR}/tsuru.yaml ]; then + tsuru_yaml="${CURRENT_DIR}/tsuru.yaml" + fi + + use_default_procfile=1 + if [ -n "$tsuru_yaml" ]; then + # Look for 'processes:' and at least one '- name:' and 'command:' under it + if awk ' + $1 == "processes:" {in_proc=1; next} + in_proc && $1 == "-" && $2 == "name:" {name=1} + in_proc && $1 == "command:" {cmd=1} + name && cmd {found=1; exit} + END {exit !found} + ' "$tsuru_yaml"; then + use_default_procfile=0 + fi + fi + + if [ $use_default_procfile -eq 1 ] && [ -f ${SOURCE_DIR}/default/Procfile ]; then + cp ${SOURCE_DIR}/default/Procfile ${CURRENT_DIR}/Procfile + touch ${APP_DIR}/.default_procfile + fi fi } - source ${SOURCE_DIR}/base/rc/config source ${SOURCE_DIR}/base/rc/os_dependencies @@ -77,14 +99,15 @@ fi pre_deploy case $1 in - "archive") - archive_deploy $2 - shift; shift - ;; - *) - echo "FATAL: this deployment method is no longer supported" - exit 1 - ;; +"archive") + archive_deploy $2 + shift + shift + ;; +*) + echo "FATAL: this deployment method is no longer supported" + exit 1 + ;; esac post_deploy diff --git a/base/rc/os_dependencies b/base/rc/os_dependencies index d15f566..a1a0e7e 100644 --- a/base/rc/os_dependencies +++ b/base/rc/os_dependencies @@ -7,19 +7,19 @@ # removes all '#' and all characters afterwards; trim whitespaces; carriage return and empty lines function cat_without_hash_comments() { local file_path=${1} - cat ${file_path} \ - | sed "s/#.*$//g" \ - | sed "s/^[[:space:]]*//g" \ - | sed "s/[[:space:]]*$//g" \ - | sed "s/\r$//" \ - | sed "/^$/d" + cat ${file_path} | + sed "s/#.*$//g" | + sed "s/^[[:space:]]*//g" | + sed "s/[[:space:]]*$//g" | + sed "s/\r$//" | + sed "/^$/d" } function os_dependencies() { if [[ -f "${CURRENT_DIR}/repositories.apt" ]]; then echo_debug ">>> repositories.apt found: adding extra APT repositories" - while read repo ; do + while read repo; do add_repository ${repo} done < <(cat_without_hash_comments "${CURRENT_DIR}/repositories.apt") fi @@ -34,38 +34,54 @@ function os_dependencies() { function add_repository() { local repository=${1} + local repository_entries + local ppa + local fingerprint + local user + local filesum case ${repository} in - deb*) - local repository_entries=${*};; + deb*) + repository_entries=${*} + ;; - *) - local ppa=$(echo ${repository} | sed 's/ppa://') + *) + ppa=${repository/#ppa:/} + fingerprint=${2:-${ppa}} + user=$(echo "${ppa}" | awk -F / '{print $1}') - local repository_entries=$(cat <<-EOF -deb https://ppa.launchpadcontent.net/${ppa}/ubuntu ${UBUNTU_RELEASE} main -deb-src https://ppa.launchpadcontent.net/${ppa}/ubuntu ${UBUNTU_RELEASE} main + repository_entries=$( + cat <<-EOF +deb [signed-by=/usr/share/keyrings/${user}.gpg] https://ppa.launchpadcontent.net/${ppa}/ubuntu ${UBUNTU_RELEASE} main +deb-src [signed-by=/usr/share/keyrings/${user}.gpg] https://ppa.launchpadcontent.net/${ppa}/ubuntu ${UBUNTU_RELEASE} main EOF -) - local fingerprint=${2:-${ppa}} - get_ppa_key ${fingerprint};; + ) + + get_ppa_key "${fingerprint}" "${user}" + ;; esac - local filesum=$(echo ${repository_entries} | sha1sum | awk '{print $1}') - echo "${repository_entries}" | sudo -E tee >/dev/null 2>&1 "/etc/apt/sources.list.d/tsuru_${filesum}.list" + filesum=$(echo "${repository_entries}" | sha1sum | awk '{print $1}') + echo "${repository_entries}" | sudo -E tee "/etc/apt/sources.list.d/tsuru_${filesum}.list" >/dev/null 2>&1 } function get_ppa_key() { local ppa_or_fingerprint=${1} + local user=${2} + local fingerprint + local repo if [[ ${ppa_or_fingerprint} =~ ^0x.+ ]]; then - local fingerprint=${ppa_or_fingerprint} + fingerprint=${ppa_or_fingerprint} else - user=$(echo ${ppa_or_fingerprint} | awk -F / '{print $1}') - repo=$(echo ${ppa_or_fingerprint} | awk -F / '{print $2}') + repo=$(echo "${ppa_or_fingerprint}" | awk -F / '{print $2}') api_url="https://launchpad.net/api/1.0/~${user}/+archive/ubuntu/${repo}" - local fingerprint=$(curl -sS "${api_url}" | jq -r ".signing_key_fingerprint") + fingerprint=$(curl -sS "${api_url}" | jq -r ".signing_key_fingerprint") fi - sudo -E apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv ${fingerprint} + local gpg_tmp_dir + gpg_tmp_dir=$(mktemp -d /tmp/gpg-XXXXXX) + chmod 700 "${gpg_tmp_dir}" + sudo gpg --homedir "${gpg_tmp_dir}" --no-default-keyring --keyring /usr/share/keyrings/"${user}".gpg --keyserver keyserver.ubuntu.com --recv-keys "${fingerprint}" + rm -rf "${gpg_tmp_dir}" } diff --git a/test/Dockerfile b/test/Dockerfile index 145fe99..8ac7dab 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -7,22 +7,22 @@ ARG image FROM ${image} ARG bats_core_version=1.6.0 \ - bats_support_version=0.3.0 \ - bats_assert_version=master + bats_support_version=0.3.0 \ + bats_assert_version=master RUN set -eux \ - && curl -fsSL https://github.com/bats-core/bats-core/archive/refs/tags/v${bats_core_version}.tar.gz \ - | tar -xzf- -C /tmp \ - && sudo /tmp/bats-core-${bats_core_version}/install.sh /opt/bats \ - && sudo ln -s /opt/bats/bin/bats /usr/local/bin \ - && bats --version \ - && rm -rf /tmp/bats-core-${bats_core_version} \ - && curl -fsSL https://github.com/bats-core/bats-support/archive/refs/tags/v${bats_support_version}.tar.gz \ - | tar -xzf- -C /tmp \ - && sudo mv /tmp/bats-support-${bats_support_version} /opt/bats-support \ - && curl -fsSL https://github.com/bats-core/bats-assert/archive/refs/heads/${bats_assert_version}.tar.gz \ - | tar -xzf- -C /tmp \ - && sudo mv /tmp/bats-assert-${bats_assert_version} /opt/bats-assert + && curl -fsSL https://github.com/bats-core/bats-core/archive/refs/tags/v${bats_core_version}.tar.gz \ + | tar -xzf- -C /tmp \ + && sudo /tmp/bats-core-${bats_core_version}/install.sh /opt/bats \ + && sudo ln -s /opt/bats/bin/bats /usr/local/bin \ + && bats --version \ + && rm -rf /tmp/bats-core-${bats_core_version} \ + && curl -fsSL https://github.com/bats-core/bats-support/archive/refs/tags/v${bats_support_version}.tar.gz \ + | tar -xzf- -C /tmp \ + && sudo mv /tmp/bats-support-${bats_support_version} /opt/bats-support \ + && curl -fsSL https://github.com/bats-core/bats-assert/archive/refs/heads/${bats_assert_version}.tar.gz \ + | tar -xzf- -C /tmp \ + && sudo mv /tmp/bats-assert-${bats_assert_version} /opt/bats-assert WORKDIR /tests diff --git a/test/tests.bats b/test/tests.bats index 641d227..6551992 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -33,10 +33,76 @@ load '/opt/bats-assert/load.bash' [ ! -f /home/ubuntu/.bash_logout ] } +@test 'should copy default Procfile when tsuru.yml does not have processes defined' { + export APP_DIR=${BATS_TEST_TMPDIR}/app + export CURRENT_DIR=${APP_DIR}/current + export SOURCE_DIR=/var/lib/tsuru + + [ -r /var/lib/tsuru/base/deploy ] + sudo sed -i '/source \${SOURCE_DIR}\/base\/rc\/config/,$d' /var/lib/tsuru/base/deploy + source /var/lib/tsuru/base/deploy + + sudo mkdir -p ${SOURCE_DIR}/default + mkdir -p ${CURRENT_DIR} + + # Create tsuru.yml without processes + cat >${CURRENT_DIR}/tsuru.yml <<-EOF +healthcheck: + path: /healthcheck + method: GET + status: 200 + match: .*WORKING.* + allowed_failures: 5 + force_restart: true +EOF + + # Create default Procfile + echo "web: gunicorn app:app" | sudo tee ${SOURCE_DIR}/default/Procfile + + run post_deploy + assert_success + + # Should create Procfile from default + [ -f ${CURRENT_DIR}/Procfile ] + [ -f ${APP_DIR}/.default_procfile ] +} + +@test 'should not copy default Procfile when tsuru.yaml has processes defined' { + export APP_DIR=${BATS_TEST_TMPDIR}/app + export CURRENT_DIR=${APP_DIR}/current + export SOURCE_DIR=/var/lib/tsuru + + [ -r /var/lib/tsuru/base/deploy ] + sudo sed -i '/source \${SOURCE_DIR}\/base\/rc\/config/,$d' /var/lib/tsuru/base/deploy + source /var/lib/tsuru/base/deploy + + sudo mkdir -p ${SOURCE_DIR}/default + mkdir -p ${CURRENT_DIR} + + # Create tsuru.yml with processes + cat >${CURRENT_DIR}/tsuru.yaml <<-EOF +processes: + - name: web + command: python app.py + - name: worker + command: celery worker +EOF + + # Create default Procfile + echo "web: gunicorn app:app" | sudo tee ${SOURCE_DIR}/default/Procfile + + run post_deploy + assert_success + + # Should NOT create Procfile when tsuru.yml has processes + [ ! -f ${CURRENT_DIR}/Procfile ] + [ ! -f ${APP_DIR}/.default_procfile ] +} + @test 'reading requirements.apt w/ carriage return (CR+LF)' { - echo -e 'openssl\r' > ${BATS_TEST_TMPDIR}/requirements.apt - echo -e 'vim\r' >> ${BATS_TEST_TMPDIR}/requirements.apt - echo -e 'bash\r' >> ${BATS_TEST_TMPDIR}/requirements.apt + echo -e 'openssl\r' >${BATS_TEST_TMPDIR}/requirements.apt + echo -e 'vim\r' >>${BATS_TEST_TMPDIR}/requirements.apt + echo -e 'bash\r' >>${BATS_TEST_TMPDIR}/requirements.apt [ -r /var/lib/tsuru/base/rc/os_dependencies ] source /var/lib/tsuru/base/rc/os_dependencies @@ -46,7 +112,7 @@ load '/opt/bats-assert/load.bash' } @test 'reading requirements.apt with comments, whitespaces and empty line' { - cat > ${BATS_TEST_TMPDIR}/requirements.apt <<-EOF + cat >${BATS_TEST_TMPDIR}/requirements.apt <<-EOF # Installing openssl for X purpose openssl example # Installing exampe to mitigate Y @@ -63,7 +129,7 @@ EOF } @test 'install system package from requirements.apt' { - echo "neofetch" > ${BATS_TEST_TMPDIR}/requirements.apt + echo "neofetch" >"${BATS_TEST_TMPDIR}"/requirements.apt [ -r /var/lib/tsuru/base/rc/os_dependencies ] source /var/lib/tsuru/base/rc/os_dependencies @@ -78,15 +144,18 @@ EOF assert_output --regexp '^Neofetch [^ ]+$' } -@test 'adding extra APT repositories from repositories.apt' { +@test 'adding extra APT repositories from repositories.apt from deb' { export UBUNTU_RELEASE=$(source /etc/lsb-release && echo $DISTRIB_CODENAME) export CURRENT_DIR=${BATS_TEST_TMPDIR} - cat > ${CURRENT_DIR}/repositories.apt <<-EOF -deb https://packagecloud.io/tsuru/rc/ubuntu/ ${UBUNTU_RELEASE} main -deb-src https://packagecloud.io/tsuru/rc/ubuntu/ ${UBUNTU_RELEASE} main -ppa:tsuru/ppa -pogo-dev/stable 0x2445455D0F8FB8F9299E7A0AF2244A5C0D4D9B55 + if [ ! -d "/etc/apt/keyrings/" ]; then + sudo mkdir -p /etc/apt/keyrings/ + fi + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + cat >${CURRENT_DIR}/repositories.apt <<-EOF +deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${UBUNTU_RELEASE} stable EOF [ -r /var/lib/tsuru/base/rc/os_dependencies ] @@ -95,41 +164,97 @@ EOF run os_dependencies assert_success - expected_source_file="deb https://packagecloud.io/tsuru/rc/ubuntu/ ${UBUNTU_RELEASE} main" + expected_source_file="deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${UBUNTU_RELEASE} stable" expected_file="/etc/apt/sources.list.d/tsuru_$(echo ${expected_source_file} | sha1sum | awk '{print $1}').list" - [ -r ${expected_file} ] - run cat ${expected_file} + [ -r "${expected_file}" ] + run cat "${expected_file}" assert_output "${expected_source_file}" +} + +@test 'adding extra APT repositories from repositories.apt from deb-src' { + export UBUNTU_RELEASE=$(source /etc/lsb-release && echo $DISTRIB_CODENAME) + export CURRENT_DIR=${BATS_TEST_TMPDIR} + + if [ ! -d "/etc/apt/keyrings/" ]; then + sudo mkdir -p /etc/apt/keyrings/ + fi + sudo curl -fsSL https://packages.mozilla.org/apt/repo-signing-key.gpg -o /etc/apt/keyrings/packages.mozilla.org.asc + sudo chmod a+r /etc/apt/keyrings/packages.mozilla.org.asc + + cat >${CURRENT_DIR}/repositories.apt <<-EOF +deb-src [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main +EOF + + [ -r /var/lib/tsuru/base/rc/os_dependencies ] + source /var/lib/tsuru/base/rc/os_dependencies + + run os_dependencies + assert_success - expected_source_file="deb-src https://packagecloud.io/tsuru/rc/ubuntu/ ${UBUNTU_RELEASE} main" + expected_source_file="deb-src [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" expected_file="/etc/apt/sources.list.d/tsuru_$(echo ${expected_source_file} | sha1sum | awk '{print $1}').list" - [ -r ${expected_file} ] - run cat ${expected_file} + [ -r "${expected_file}" ] + run cat "${expected_file}" assert_output "${expected_source_file}" +} + +@test 'adding extra APT repositories from repositories.apt from ppa' { + export UBUNTU_RELEASE=$(source /etc/lsb-release && echo $DISTRIB_CODENAME) + export CURRENT_DIR=${BATS_TEST_TMPDIR} - expected_source_file=$(cat <<-EOF -deb https://ppa.launchpadcontent.net/tsuru/ppa/ubuntu ${UBUNTU_RELEASE} main -deb-src https://ppa.launchpadcontent.net/tsuru/ppa/ubuntu ${UBUNTU_RELEASE} main + cat >${CURRENT_DIR}/repositories.apt <<-EOF +ppa:graphics-drivers/ppa EOF -) - expected_file="/etc/apt/sources.list.d/tsuru_$(echo ${expected_source_file} | sha1sum | awk '{print $1}').list" - [ -r ${expected_file} ] - run cat ${expected_file} + + [ -r /var/lib/tsuru/base/rc/os_dependencies ] + source /var/lib/tsuru/base/rc/os_dependencies + + run os_dependencies + assert_success + + expected_source_file=$( + cat <<-EOF +deb [signed-by=/usr/share/keyrings/graphics-drivers.gpg] https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu ${UBUNTU_RELEASE} main +deb-src [signed-by=/usr/share/keyrings/graphics-drivers.gpg] https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu ${UBUNTU_RELEASE} main +EOF + ) + expected_file="/etc/apt/sources.list.d/tsuru_$(echo "${expected_source_file}" | sha1sum | awk '{print $1}').list" + + [ -r "${expected_file}" ] + run cat "${expected_file}" assert_output "${expected_source_file}" - run sudo apt-key finger 2>/dev/null - assert_output --partial 'B0DE 9C5D EBF4 8635 9EB2 55B0 3B01 53D0 383F 073D' - expected_source_file=$(cat <<-EOF -deb https://ppa.launchpadcontent.net/pogo-dev/stable/ubuntu ${UBUNTU_RELEASE} main -deb-src https://ppa.launchpadcontent.net/pogo-dev/stable/ubuntu ${UBUNTU_RELEASE} main + [ -r "/usr/share/keyrings/graphics-drivers.gpg" ] +} + +@test 'adding extra APT repositories from repositories.apt from ppa with fingerprint' { + export UBUNTU_RELEASE=$(source /etc/lsb-release && echo $DISTRIB_CODENAME) + export CURRENT_DIR=${BATS_TEST_TMPDIR} + + cat >${CURRENT_DIR}/repositories.apt <<-EOF +pogo-dev/daily 0x32D78285D050BE9F1B0030D0F0AE9179321F84C8 EOF -) - expected_file="/etc/apt/sources.list.d/tsuru_$(echo ${expected_source_file} | sha1sum | awk '{print $1}').list" - [ -r ${expected_file} ] - run cat ${expected_file} + + [ -r /var/lib/tsuru/base/rc/os_dependencies ] + source /var/lib/tsuru/base/rc/os_dependencies + + run os_dependencies + assert_success + + expected_source_file=$( + cat <<-EOF +deb [signed-by=/usr/share/keyrings/pogo-dev.gpg] https://ppa.launchpadcontent.net/pogo-dev/daily/ubuntu ${UBUNTU_RELEASE} main +deb-src [signed-by=/usr/share/keyrings/pogo-dev.gpg] https://ppa.launchpadcontent.net/pogo-dev/daily/ubuntu ${UBUNTU_RELEASE} main +EOF + ) + expected_file="/etc/apt/sources.list.d/tsuru_$(echo "${expected_source_file}" | sha1sum | awk '{print $1}').list" + [ -r "${expected_file}" ] + run cat "${expected_file}" assert_output "${expected_source_file}" - run sudo apt-key finger 2>/dev/null - assert_output --partial '2445 455D 0F8F B8F9 299E 7A0A F224 4A5C 0D4D 9B55' + + run gpg --no-default-keyring --keyring /usr/share/keyrings/pogo-dev.gpg --fingerprint + assert_success + assert_output --partial "32D7 8285 D050 BE9F 1B00 30D0 F0AE 9179 321F 84C8" } # vim: ft=bash