From fe51811a754fe248f988f642fdf7f1005f1886cb Mon Sep 17 00:00:00 2001 From: Fruchix Date: Tue, 29 Oct 2024 22:59:00 +0100 Subject: [PATCH 1/5] feat: add ability to inject a function into each job The function must be passed as 3rd parameter to job_pool_init. This function is called at the end of each job. The function is aware of each local variable from _job_pool_worker and each global variable of job_pool.sh, meaning its definition can contain references to those variables, even though then do not exist in the current program (that uses job_pool.sh). --- job_pool.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/job_pool.sh b/job_pool.sh index 2f801e6..d4f4f37 100644 --- a/job_pool.sh +++ b/job_pool.sh @@ -45,6 +45,9 @@ job_pool_pool_size=-1 # \brief variable to check for number of non-zero exits job_pool_nerrors=0 +# function that will be called at the end of each job. By default, no function is called. +job_pool_injected_function="" + ################################################################################ # private functions ################################################################################ @@ -124,6 +127,11 @@ function _job_pool_worker() flock --unlock 8 exec 8>&- _job_pool_echo "### _job_pool_worker-${id}: exited ${result}: ${cmd} $@" + + # run the injected function if it was provided + if [[ "${job_pool_injected_function}" != "" ]]; then + "${job_pool_injected_function}" + fi fi done exec 7>&- @@ -157,12 +165,14 @@ function _job_pool_start_workers() # \brief initializes the job pool # \param[in] pool_size number of parallel jobs allowed # \param[in] echo_command 1 to turn on echo, 0 to turn off +# \param[in] job_pool_injected_function (optional) a function that will be called at the end of each job function job_pool_init() { local pool_size=$1 local echo_command=$2 # set the global attibutes + job_pool_injected_function=$3 job_pool_pool_size=${pool_size:=1} job_pool_echo_command=${echo_command:=0} From ebf2974c3bce361826a7ece750435391c7d8e943 Mon Sep 17 00:00:00 2001 From: Fruchix Date: Tue, 29 Oct 2024 23:27:13 +0100 Subject: [PATCH 2/5] feat: add example of function injection --- job_pool_sample.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/job_pool_sample.sh b/job_pool_sample.sh index 1a38032..43e83f7 100755 --- a/job_pool_sample.sh +++ b/job_pool_sample.sh @@ -33,3 +33,49 @@ job_pool_shutdown # check the $job_pool_nerrors for the number of jobs that exited non-zero echo "job_pool_nerrors: ${job_pool_nerrors}" + + +##################################################### +# Demonstration of function injection into each job # +##################################################### + +echo -e "\nDemonstration of function injection into each job:" + +# sleep some time ($1) then echo something ($2) +function sleep_n_echo() +{ + sleep "$1" + echo "$2" +} + +# kill all workers if the local variable "result" from _job_pool_worker +# indicates that the job failed +function kill_jobs() +{ + # result is undefined in this script, but will be defined when + # the function is injected in _job_pool_worker + if [[ "${result}" != "0" ]]; then + # get the pids of all workers: + # - each worker's process is named after the current script (here, job_pool_sample.sh), + # so we use this name to get the pids + # - we do not include the current script's pid ($$) as it is not a worker, + # (we do not want to kill the script itself, only the workers) + local workers_pids=("$(pgrep -f "$0" | grep -v $$)") + kill ${workers_pids[@]} &> /dev/null & + fi +} + +# allow 3 parallel jobs, and kill all jobs at the first fail using "kill_jobs" function +job_pool_init 3 1 kill_jobs + +# simulate 3 jobs, where one fails before the others are finished, and interrupts the others +job_pool_run sleep_n_echo 3 a # job 1 +job_pool_run /bin/false # job 2 +job_pool_run sleep_n_echo 3 b # job 3 + +# the job 2 will kill all other running workers, using the function "kill_jobs", +# that is ran after processing each job + +job_pool_shutdown + +echo -e "\nOnly the failed job exited, the others did not because they were canceled." From 7755573c9e11bc5b4849edbe4c6bd85be5a132c4 Mon Sep 17 00:00:00 2001 From: Fruchix Date: Wed, 30 Oct 2024 22:11:56 +0100 Subject: [PATCH 3/5] feat: add pre and post job injected function Two different functions can now be injected, one that will be run before each job, and one after. If only a post job function is wanted, the pre job function should be "" (in order for the post job function to be the fourth parameter). Updated the demonstration for pre and post functions injection. --- job_pool.sh | 26 +++++++++++++++++++------- job_pool_sample.sh | 24 ++++++++++++++++++------ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/job_pool.sh b/job_pool.sh index d4f4f37..20fc631 100644 --- a/job_pool.sh +++ b/job_pool.sh @@ -45,8 +45,11 @@ job_pool_pool_size=-1 # \brief variable to check for number of non-zero exits job_pool_nerrors=0 -# function that will be called at the end of each job. By default, no function is called. -job_pool_injected_function="" +# function that will be called before each job. By default, no function is called. +job_pool_function_pre_job="" + +# function that will be called after each job. By default, no function is called. +job_pool_function_post_job="" ################################################################################ # private functions @@ -109,6 +112,11 @@ function _job_pool_worker() # will know we are exiting. echo "${cmd}" >&7 else + # run the pre job injected function if it was provided + if [[ "${job_pool_function_pre_job}" != "" ]]; then + "${job_pool_function_pre_job}" + fi + _job_pool_echo "### _job_pool_worker-${id}: ${cmd}" # run the job { ${cmd} "$@" ; } @@ -128,9 +136,9 @@ function _job_pool_worker() exec 8>&- _job_pool_echo "### _job_pool_worker-${id}: exited ${result}: ${cmd} $@" - # run the injected function if it was provided - if [[ "${job_pool_injected_function}" != "" ]]; then - "${job_pool_injected_function}" + # run the post job injected function if it was provided + if [[ "${job_pool_function_post_job}" != "" ]]; then + "${job_pool_function_post_job}" fi fi done @@ -165,14 +173,18 @@ function _job_pool_start_workers() # \brief initializes the job pool # \param[in] pool_size number of parallel jobs allowed # \param[in] echo_command 1 to turn on echo, 0 to turn off -# \param[in] job_pool_injected_function (optional) a function that will be called at the end of each job +# \param[in] job_pool_function_pre_job (optional) a function that will be called before each job +# \param[in] job_pool_function_post_job (optional) a function that will be called after each job. +# +# To only have a post job function, the given parameter for the pre job function should be: "". function job_pool_init() { local pool_size=$1 local echo_command=$2 # set the global attibutes - job_pool_injected_function=$3 + job_pool_function_pre_job=$3 + job_pool_function_post_job=$4 job_pool_pool_size=${pool_size:=1} job_pool_echo_command=${echo_command:=0} diff --git a/job_pool_sample.sh b/job_pool_sample.sh index 43e83f7..bf6d9d4 100755 --- a/job_pool_sample.sh +++ b/job_pool_sample.sh @@ -48,10 +48,22 @@ function sleep_n_echo() echo "$2" } -# kill all workers if the local variable "result" from _job_pool_worker +# Injected function that will be called before each job +# +# Print which worker is starting which job +function print_starting_job() +{ + echo " # _job_pool_worker-${id}: Starting job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" +} + +# Injected function that will be called afetr each job +# +# Kill all workers if the local variable "result" from _job_pool_worker # indicates that the job failed -function kill_jobs() +function kill_workers() { + echo " # _job_pool_worker-${id}: Finished job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" + # result is undefined in this script, but will be defined when # the function is injected in _job_pool_worker if [[ "${result}" != "0" ]]; then @@ -65,16 +77,16 @@ function kill_jobs() fi } -# allow 3 parallel jobs, and kill all jobs at the first fail using "kill_jobs" function -job_pool_init 3 1 kill_jobs +# allow 3 parallel jobs, and kill all jobs at the first fail using "kill_workers" function +job_pool_init 3 0 print_starting_job kill_workers # simulate 3 jobs, where one fails before the others are finished, and interrupts the others job_pool_run sleep_n_echo 3 a # job 1 job_pool_run /bin/false # job 2 job_pool_run sleep_n_echo 3 b # job 3 -# the job 2 will kill all other running workers, using the function "kill_jobs", -# that is ran after processing each job +# the job 2 will kill all other running workers, using the function "kill_workers" +# (that is ran after processing each job) job_pool_shutdown From 5fe460d4e032c99a8e74af7803c6ffc733310227 Mon Sep 17 00:00:00 2001 From: Fruchix Date: Wed, 30 Oct 2024 22:19:21 +0100 Subject: [PATCH 4/5] refactor: put sample for injected functions in its own file --- job_pool_sample.sh | 58 -------------------------- job_pool_sample_injected_functions.sh | 60 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 58 deletions(-) create mode 100755 job_pool_sample_injected_functions.sh diff --git a/job_pool_sample.sh b/job_pool_sample.sh index bf6d9d4..1a38032 100755 --- a/job_pool_sample.sh +++ b/job_pool_sample.sh @@ -33,61 +33,3 @@ job_pool_shutdown # check the $job_pool_nerrors for the number of jobs that exited non-zero echo "job_pool_nerrors: ${job_pool_nerrors}" - - -##################################################### -# Demonstration of function injection into each job # -##################################################### - -echo -e "\nDemonstration of function injection into each job:" - -# sleep some time ($1) then echo something ($2) -function sleep_n_echo() -{ - sleep "$1" - echo "$2" -} - -# Injected function that will be called before each job -# -# Print which worker is starting which job -function print_starting_job() -{ - echo " # _job_pool_worker-${id}: Starting job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" -} - -# Injected function that will be called afetr each job -# -# Kill all workers if the local variable "result" from _job_pool_worker -# indicates that the job failed -function kill_workers() -{ - echo " # _job_pool_worker-${id}: Finished job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" - - # result is undefined in this script, but will be defined when - # the function is injected in _job_pool_worker - if [[ "${result}" != "0" ]]; then - # get the pids of all workers: - # - each worker's process is named after the current script (here, job_pool_sample.sh), - # so we use this name to get the pids - # - we do not include the current script's pid ($$) as it is not a worker, - # (we do not want to kill the script itself, only the workers) - local workers_pids=("$(pgrep -f "$0" | grep -v $$)") - kill ${workers_pids[@]} &> /dev/null & - fi -} - -# allow 3 parallel jobs, and kill all jobs at the first fail using "kill_workers" function -job_pool_init 3 0 print_starting_job kill_workers - -# simulate 3 jobs, where one fails before the others are finished, and interrupts the others -job_pool_run sleep_n_echo 3 a # job 1 -job_pool_run /bin/false # job 2 -job_pool_run sleep_n_echo 3 b # job 3 - -# the job 2 will kill all other running workers, using the function "kill_workers" -# (that is ran after processing each job) - -job_pool_shutdown - -echo -e "\nOnly the failed job exited, the others did not because they were canceled." diff --git a/job_pool_sample_injected_functions.sh b/job_pool_sample_injected_functions.sh new file mode 100755 index 0000000..ca50721 --- /dev/null +++ b/job_pool_sample_injected_functions.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +. job_pool.sh + +##################################################### +# Demonstration of function injection into each job # +##################################################### + +echo "Demonstration of function injection into each job:" + +# sleep some time ($1) then echo something ($2) +function sleep_n_echo() +{ + sleep "$1" + echo "$2" +} + +# Injected function that will be called before each job +# +# Print which worker is starting which job +function print_starting_job() +{ + echo " # _job_pool_worker-${id}: Starting job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" +} + +# Injected function that will be called afetr each job +# +# Kill all workers if the local variable "result" from _job_pool_worker +# indicates that the job failed +function kill_workers() +{ + echo " # _job_pool_worker-${id}: Finished job: ${cmd} $(echo "${args[@]}" | xargs | tr '\v' ' ')" + + # result is undefined in this script, but will be defined when + # the function is injected in _job_pool_worker + if [[ "${result}" != "0" ]]; then + # get the pids of all workers: + # - each worker's process is named after the current script (here, job_pool_sample.sh), + # so we use this name to get the pids + # - we do not include the current script's pid ($$) as it is not a worker, + # (we do not want to kill the script itself, only the workers) + local workers_pids=("$(pgrep -f "$0" | grep -v $$)") + kill ${workers_pids[@]} &> /dev/null & + fi +} + +# allow 3 parallel jobs, and kill all jobs at the first fail using "kill_workers" function +job_pool_init 3 0 print_starting_job kill_workers + +# simulate 3 jobs, where one fails before the others are finished, and interrupts the others +job_pool_run sleep_n_echo 3 a # job 1 +job_pool_run /bin/false # job 2 +job_pool_run sleep_n_echo 3 b # job 3 + +# the job 2 will kill all other running workers, using the function "kill_workers" +# (that is ran after processing each job) + +job_pool_shutdown + +echo -e "\nOnly the failed job exited, the others did not because they were canceled." From c08992fb56dd58462a31d0b4aae536f2d0655915 Mon Sep 17 00:00:00 2001 From: Fruchix Date: Wed, 30 Oct 2024 22:27:56 +0100 Subject: [PATCH 5/5] docs: update README according to injected functions addition --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 2ce51e3..6ea38e4 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ The interface consists of four public functions and one variable that reports ho # \brief initializes the job pool # \param[in] pool_size number of parallel jobs allowed # \param[in] echo_command 1 to turn on echo, 0 to turn off + # \param[in] job_pool_function_pre_job (optional) a function that will be called before each job + # \param[in] job_pool_function_post_job (optional) a function that will be called after each job. + # + # To only have a post job function, the given parameter for the pre job function should be: "". function job_pool_init() # \brief waits for all queued up jobs to complete and shuts down the job pool @@ -65,6 +69,8 @@ The interface consists of four public functions and one variable that reports ho This was inspired by [a discussion on StackOverflow](http://stackoverflow.com/questions/6441509/how-to-write-a-process-pool-bash-shell). +For more information on the parameters "job_pool_function_pre_job" and "job_pool_function_post_job" of the "job_pool_init" function, check the sample program "job_pool_sample_injected_functions.sh". + ## `genpass.sh` `genpass.sh` is a random password generator, it will produce a random password of the desired length which is suitable for manual entering via the keyboard.