[Fixes #13577] Spawn Celery workers for harvesters (#13579)

Author: Gpetrak

.env.sample

@@ -222,15 +222,23 @@ LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember
 # CELERY__MAX_MEMORY_PER_CHILD="200000"
 # ## 
 # Note right autoscale value must coincide with worker concurrency value
-# CELERY__AUTOSCALE_VALUES="15,10" 
-# CELERY__WORKER_CONCURRENCY="10"
+# CELERY__AUTOSCALE_VALUES="10,5" 
+# CELERY__WORKER_CONCURRENCY="4"
 # ##
 # CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E"
 # CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule"
 # CELERY__LOG_LEVEL="INFO"
 # CELERY__LOG_FILE="/var/log/celery.log"
 # CELERY__WORKER_NAME="worker1@%h"
 
+# CELERY for harvesters
+# CELERY__HARVESTER_WORKER_NAME="harvesting_worker@%h"
+# CELERY__HARVESTER_CONCURRENCY="3"
+# CELERY__HARVESTER_AUTOSCALE_VALUES="5,3"
+# CELERY__HARVESTER_MAX_MEMORY_PER_CHILD="500000"
+# CELERY__HARVESTER_LOG_FILE="/var/log/celery_harvester.log"
+
+
 # PostgreSQL
 POSTGRESQL_MAX_CONNECTIONS=200
 

celery-cmd

@@ -6,7 +6,7 @@ CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"}
 CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker@%h.state"}
 # expressed in KB
 CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"}
-CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"15,10"}
+CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"10,5"}
 CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"}
 CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"}
 CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"}
@@ -15,8 +15,27 @@ CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"}
 CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"}
 CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"}
 
+# Harvester settings
+CELERY__HARVESTER_WORKER_NAME=${CELERY__HARVESTER_WORKER_NAME:-"harvesting_worker@%h"}
+CELERY__HARVESTER_CONCURRENCY=${CELERY__HARVESTER_CONCURRENCY:-"10"}
+CELERY__HARVESTER_AUTOSCALE_VALUES=${CELERY__HARVESTER_AUTOSCALE_VALUES:-"15,10"}
+CELERY__HARVESTER_MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"500000"}
+CELERY__HARVESTER_LOG_FILE=${CELERY__HARVESTER_LOG_FILE:-"/var/log/celery_harvester.log"}
+
 $CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \
     --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \
     --statedb=$CELERY__STATE_DB --scheduler=$CELERY__BEAT_SCHEDULE \
     --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \
-    --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD
+    --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD \
+    -X harvesting &
+
+$CELERY_BIN -A $CELERY_APP worker -Q harvesting \
+    --autoscale=$CELERY__HARVESTER_AUTOSCALE_VALUES \
+    --max-memory-per-child=$CELERY__HARVESTER_MAX_MEMORY_PER_CHILD \
+    --loglevel=$CELERY__LOG_LEVEL \
+    -n $CELERY__HARVESTER_WORKER_NAME \
+    --concurrency=$CELERY__HARVESTER_CONCURRENCY \
+    -f $CELERY__HARVESTER_LOG_FILE &
+
+# Keep the container alive
+wait