Merge pull request #2083 from nextcloud/devel

NC 32 and fixes

Author: theCalcaholic

.github/workflows/build-lxd.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Setup incus
         if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
         run: |
-          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
+          sudo apt-get install -y incus qemu-system incus-tools
           sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
           sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
           sudo incus admin init --auto
@@ -341,6 +341,7 @@ jobs:
 #          }
 
   test-dist-upgrade:
+    if: false
     needs:
       - determine-runner
     runs-on: ${{ needs.determine-runner.outputs.runner_label }}
@@ -355,7 +356,7 @@ jobs:
       - name: Setup incus
         if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
         run: |
-          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
+          sudo apt-get install -y incus qemu-system incus-tools
           sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
           sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
           sudo incus admin init --auto
@@ -364,11 +365,12 @@ jobs:
         with:
           ref: "v1.54.3"
       - name: Setup Firefox
+        if: ${{ runner.arch != 'ARM64' }}
         continue-on-error: true
         id: setup-firefox-browser-action
         uses: browser-actions/setup-firefox@latest
       - name: Setup Firefox from packages
-        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
+        if: ${{ runner.arch == 'ARM64' || steps.setup-firefox-browser-action.outcome == 'failure' }}
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends firefox
@@ -386,6 +388,7 @@ jobs:
           tar xf "geckodriver-"*"-$arch.tar.gz"
           sudo mv geckodriver /usr/local/bin/
           sudo chmod +x /usr/local/bin/geckodriver
+          echo "GECKODRIVER_PATH=/usr/local/bin/geckodriver" >> "$GITHUB_ENV"
           geckodriver -V
       - name: Setup Selenium
         run: |
@@ -419,6 +422,9 @@ jobs:
       - name: Activate and Test LXD Image
         working-directory: ./tests
         run: |
+          set -x
+          export GECKODRIVER_PATH="$GECKODRIVER_PATH"
+          export FF_BINARY_PATH="$(which firefox)"
           sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
           ../.venv/bin/python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
             echo "Activation test failed!"
@@ -444,6 +450,12 @@ jobs:
           }
           USE_INCUS="$USE_INCUS"  ../.venv/bin/python system_tests.py --non-interactive --skip-update-test || {
             echo "System test failed!"
+            echo "ncp.log: "
+            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
+            echo "================"
+            echo "nextcloud log: "
+            datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)"
+            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
             exit 1
           }
 
@@ -473,13 +485,13 @@ jobs:
           sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
           sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
           
-          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
-          then
-            echo "Nextcloud is up to date - skipping NC update test."
-          else
-            sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
-            sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
-          fi
+          #if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
+          #then
+          #  echo "Nextcloud is up to date - skipping NC update test."
+          #else
+          #  sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
+          #  sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
+          #fi
   
           sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
 
@@ -500,6 +512,8 @@ jobs:
       - name: Test LXD Image
         working-directory: ./tests
         run: |
+          export GECKODRIVER_PATH="$GECKODRIVER_PATH"
+          export FF_BINARY_PATH="$(which firefox)"
           ../.venv/bin/python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
             echo "Nextcloud test failed!"
             echo "Geckodriver logs:"
@@ -579,6 +593,9 @@ jobs:
       - name: Test LXD Image
         working-directory: ./tests
         run: |
+          set -x
+          export GECKODRIVER_PATH="$GECKODRIVER_PATH"
+          export FF_BINARY_PATH="$(which firefox)"
           sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
           ../.venv/bin/python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
             echo "Nextcloud test failed!"
@@ -619,16 +636,17 @@ jobs:
       - name: Setup incus
         if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
         run: |
-          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
+          sudo apt-get install -y incus qemu-system incus-tools
           sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
           sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
           sudo incus admin init --auto
       - name: Setup Firefox
+        if: ${{ runner.arch != 'ARM64' }}
         continue-on-error: true
         id: setup-firefox-browser-action
         uses: browser-actions/setup-firefox@latest
       - name: Setup Firefox from packages
-        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
+        if: ${{ runner.arch == 'ARM64' || steps.setup-firefox-browser-action.outcome == 'failure' }}
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends firefox
@@ -646,6 +664,7 @@ jobs:
           tar xf "geckodriver-"*"-$arch.tar.gz"
           sudo mv geckodriver /usr/local/bin/
           sudo chmod +x /usr/local/bin/geckodriver
+          echo "GECKODRIVER_PATH=/usr/local/bin/geckodriver" >> "$GITHUB_ENV"
           geckodriver -V
       - name: Setup Selenium
         run: |
@@ -682,6 +701,9 @@ jobs:
       - name: Test LXD Image
         working-directory: ./tests
         run: |
+          set -x
+          export GECKODRIVER_PATH="$GECKODRIVER_PATH"
+          export FF_BINARY_PATH="$(which firefox)"
           sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
           ../.venv/bin/python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
             echo "Activation test failed!"

.github/workflows/build-sd-images.yml

@@ -155,11 +155,12 @@ jobs:
         run: |
           set -ex
           mkdir -p output
-          wget -nv "${{ inputs.test_image_url }}" | pv -i 3 > "output/${ARTIFACT_FILE?}"
+          sudo apt-get install -y pv
+          wget -nv "${{ inputs.test_image_url }}" -O - | pv -n --bytes --rate --timer -i 3 > "output/${ARTIFACT_FILE?}"
       - name: Prepare test
         run: |
           set -x
-          mv output/${ARTIFACT_FILE?} ncp.img
+          mv "output/${ARTIFACT_FILE?}" ncp.img
           sudo apt-get install -y systemd-container
           python3 -m venv ./.venv
           . ./.venv/bin/activate

.github/workflows/publish-image.yml

@@ -43,7 +43,7 @@ jobs:
         run: |
           set -ex
           mkdir -p publish
-          mv artifacts/${{ inputs.artifact_file }} publish/
+          mv "artifacts/${{ inputs.artifact_file }}" publish/
           cd publish
           
           asset="${IMG}"

.github/workflows/release.yml

@@ -63,26 +63,17 @@ jobs:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
     secrets: inherit
 
-  raspberrypi-4:
+  raspberrypi:
     if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
     uses: ./.github/workflows/build-sd-images.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
       board_id: rpi4b
-      board_name: RaspberryPi4
-    secrets: inherit
-
-  raspberrypi-5:
-    if: ${{ false && inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
-    uses: ./.github/workflows/build-sd-images.yml
-    with:
-      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      board_id: rpi5b
-      board_name: RaspberryPi5
+      board_name: RaspberryPi 4+
     secrets: inherit
 
   armbian-test-image:
-    if: ${{ inputs.test_image_url != '' || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
+    if: ${{ inputs.test_image_url != '' }}
     uses: ./.github/workflows/build-sd-images.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
@@ -216,31 +207,19 @@ jobs:
       artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
 
-  raspberrypi-4-release:
+  raspberrypi-release:
     needs:
-      - raspberrypi-4
+      - raspberrypi
       - github-release
     if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
     uses: ./.github/workflows/publish-image.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.raspberrypi-4.outputs.artifact_name }}"
-      artifact_file: "${{ needs.raspberrypi-4.outputs.artifact_file }}"
+      artifact_id: "${{ needs.raspberrypi.outputs.artifact_name }}"
+      artifact_file: "${{ needs.raspberrypi.outputs.artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
     secrets: inherit
 
-  raspberrypi-5-release:
-    needs:
-      - raspberrypi-5
-      - github-release
-    if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
-    uses: ./.github/workflows/publish-image.yml
-    with:
-      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.raspberrypi-5.outputs.artifact_name }}"
-      artifact_file: "${{ needs.raspberrypi-5.outputs.artifact_file }}"
-      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
-    secrets: inherit
   odroidxu4-release:
     needs:
       - odroidxu4

README.md

@@ -138,21 +138,22 @@ and press "Download".
 Optionally, you can copy the corresponding md5 sum from the releases page and paste it in the "Checksum" field to have
 Proxmox check that the downloaded image was not corrupted
 
-### 2. Installer by tteck
+### 2. Installer by community-scripts (formerly tteck)
 
-Use the [install script][ncp-proxmox-install-script-v5] from [tteck][tteck-profile] to install the LXC container on your Proxmox instance
+Use the [install script][ncp-proxmox-install-script-v5] from [community-scripts][community-scripts-profile] to install the LXC container on your Proxmox instance
 
-He has multiple helper scripts available for Proxmox on his [website][website-helper-scripts], do go have a look if you're using Proxmox. :+1:
+[community-scripts][community-scripts-profile] has multiple helper scripts available for Proxmox on their [website][website-helper-scripts], go have a look if you're using Proxmox. :+1:
 
-Installation: `bash -c "$(wget -qLO - https://github.com/tteck/Proxmox/raw/main/ct/nextcloudpi.sh)"`
+
+Installation: `bash -c "$(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVE/main/ct/nextcloudpi.sh)"`
 
 Default Settings: `2GB RAM - 8GB Storage - 2vCPU`
 
-_(Check his [website][website-helper-scripts] if this has changed and we haven't had the time to update it here yet, it's located under: Media - Photo > NextcloudPi LXC)_
+_(Check their [website][website-helper-scripts] if this has changed and we haven't had the time to update it here yet, it's located under: Operating-Systems > NextcloudPi LXC)_
 
-Thenk you [tteck][tteck-profile] :heart: for making the helper script & letting us use this for Proxmox installations :pray:
+Thank you to the [community-scripts][community-scripts-profile] organisation :heart: for making the helper script & letting us use this for Proxmox installations :pray:
 
-You can find his GitHub repository with his helper scripts [here][gh-helper-scripts-repo].
+You can find the GitHub repository with the helper scripts here: [here][gh-helper-scripts-repo].
 
 ## How to build
 
@@ -293,15 +294,15 @@ You can find us on the [Forum][nc-forum], [Telegram][chat-telegram] or [Matrix][
 
 [nc-badge]: https://img.shields.io/badge/Nextcloud-0082C9?style=for-the-badge&logo=Nextcloud&logoColor=white
 
-<!-- TTECK -->
+<!-- COMMUNITY-SCRIPTS -->
 
-[tteck-profile]: https://github.com/tteck
+[community-scripts-profile]: https://github.com/community-scripts/
 
-[gh-helper-scripts-repo]: https://github.com/tteck/Proxmox
+[gh-helper-scripts-repo]: https://github.com/community-scripts/ProxmoxVE
 
-[website-helper-scripts]: https://tteck.github.io/Proxmox/
+[website-helper-scripts]: https://community-scripts.github.io/ProxmoxVE/
 
-[ncp-proxmox-install-script-v5]: https://github.com/tteck/Proxmox/blob/main/install/nextcloudpi-v5-install.sh
+[ncp-proxmox-install-script-v5]: https://github.com/community-scripts/ProxmoxVE/blob/main/install/nextcloudpi-install.sh
 
 <!-- IMAGES -->
 

bin/ncp-provisioning.sh

@@ -82,5 +82,13 @@ EOF
   cat /usr/local/etc/instance.cfg
 }
 
+systemctl is-enabled -q [email protected] || {
+  max="$(nproc || echo '2')"
+  max="$((max-1))"
+  for i in $(seq 1 "$max")
+  do
+    systemctl enable --now "nextcloud-ai-worker@${i}.service"
+  done
+}
 
 exit 0

bin/ncp-update-nc.d/update-nc.sh

@@ -142,11 +142,12 @@ cp nextcloud-old/config/config.php nextcloud/config/
 ####################
 cp -raT nextcloud-old/themes/ nextcloud/themes/
 
-# copy old NCP apps
+# copy old NC apps
 ####################
-for app in nextcloudpi previewgenerator; do
-  if [[ -d nextcloud-old/apps/"${app}" ]]; then
-    cp -r -L nextcloud-old/apps/"${app}" /var/www/nextcloud/apps/
+for app in nextcloud-old/apps/*; do
+  if ! [[ -d /var/www/nextcloud/apps/"$(basename "$app")" ]]
+  then
+    cp -r -L "${app}" /var/www/nextcloud/apps/
   fi
 done
 
@@ -195,7 +196,12 @@ $ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n
 $ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n
 $ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n
 $ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n
+$ncc | grep -q db:convert-mysql-charset && $ncc db:convert-mysql-charset -n
 $ncc maintenance:repair --help | grep -q -e '--include-expensive' && $ncc maintenance:repair --include-expensive
+if $ncc app_api:daemon:list | grep 'No registered daemon configs.' > /dev/null 2>&1
+then
+  $ncc app:disable app_api
+fi
 
 # use the correct version for custom apps
 NCVER="$(nc_version)"

bin/ncp/CONFIG/nc-init.sh

@@ -171,6 +171,9 @@ EOF
   # we handle this ourselves
   ncc app:disable updatenotification
 
+  # Not supported in Nextcloudpi without manual setup
+  ncc app:disable app_api
+
   # ncp-previewgenerator
   local ncver
   ncver="$(ncc status 2>/dev/null | grep "version:" | awk '{ print $3 }')"

bin/ncp/CONFIG/nc-nextcloud.sh

@@ -83,6 +83,25 @@ EOF
   update-rc.d redis-server enable
   clear_opcache
 
+  # NC service workers
+  cat > /etc/systemd/system/[email protected] <<'EOF'
+[Unit]
+Description=Nextcloud AI worker %i
+After=network.target
+
+[Service]
+ExecStart=php occ background-job:worker -t 60 'OC\\TaskProcessing\\SynchronousBackgroundJob'
+Restart=always
+StartLimitInterval=60
+StartLimitBurst=10
+WorkingDirectory=/var/www/nextcloud
+User=www-data
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+
   # service to randomize passwords on first boot
   mkdir -p /usr/lib/systemd/system
   cat > /usr/lib/systemd/system/nc-provisioning.service <<'EOF'

bin/nextcloud-domain.sh

@@ -11,9 +11,12 @@ done
 # wicd service finishes before completing DHCP
 while :; do
   local_ip="$(get_ip)"
-  pub_ip="$(curl -m4 icanhazip.com 2>/dev/null)"
 
-  [[ "$pub_ip"   != "" ]] && ncc config:system:set trusted_domains 11 --value="$pub_ip"
+  pub_ipv4="$(curl -4 -m4 icanhazip.com 2>/dev/null)"
+  pub_ipv6="$(curl -6 -m4 icanhazip.com 2>/dev/null)"
+  [[ -z "$pub_ipv4" ]] || ncc config:system:set trusted_domains 11 --value="$pub_ipv4"
+  [[ -z "$pub_ipv6" ]] || ncc config:system:set trusted_domains 12 --value="[$pub_ipv6]"
+
   [[ "$local_ip" != "" ]] && break
 
   sleep 3