Skip to content

Commit 5196f11

Browse files
yacchin1205yacchin1205
committed
Add workflow matrix
1 parent 5d7bf87 commit 5196f11

17 files changed

Lines changed: 8023 additions & 6 deletions

‎.github/scripts/generate_ci_config.sh‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ set -xeuo pipefail
33

44
if [[ $# -lt 2 ]]; then
55
cat >&2 <<'USAGE'
6-
Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub]
6+
Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable]
77
USAGE
88
exit 1
99
fi
@@ -14,6 +14,7 @@ shift 2
1414
MINIO=false
1515
JUPYTERHUB=false
1616
WEKO=false
17+
FLOWABLE=false
1718

1819
for arg in "$@"; do
1920
case "$arg" in
@@ -26,6 +27,9 @@ for arg in "$@"; do
2627
--weko)
2728
WEKO=true
2829
;;
30+
--flowable)
31+
FLOWABLE=true
32+
;;
2933
*)
3034
echo "Unknown argument: ${arg}" >&2
3135
exit 1
@@ -117,4 +121,22 @@ ignore_https_errors: ${IGNORE_HTTPS_ERRORS_VALUE}
117121
EOF
118122
fi
119123

124+
if [[ "${FLOWABLE}" == "true" ]]; then
125+
GATEWAY_BASE_URL_VALUE=${GATEWAY_BASE_URL:-http://192.168.168.167:8088/}
126+
WORKFLOW_BATCH_PROJECT_COUNT_VALUE=${WORKFLOW_BATCH_PROJECT_COUNT:-50}
127+
128+
cat >> "${OUTPUT}" <<EOF
129+
130+
# Flowable Workflow settings
131+
workflow_enabled: true
132+
gateway_base_url: '${GATEWAY_BASE_URL_VALUE}'
133+
workflow_batch_project_count: ${WORKFLOW_BATCH_PROJECT_COUNT_VALUE}
134+
EOF
135+
else
136+
cat >> "${OUTPUT}" <<'EOF'
137+
138+
workflow_enabled: false
139+
EOF
140+
fi
141+
120142
cat "${OUTPUT}"

‎.github/scripts/setup_flowable.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/bin/bash
2+
set -xeuo pipefail
3+
4+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
6+
COMMAND=${1:-}
7+
FLOWABLE_ROOT=${2:-}
8+
RDM_ROOT=${3:-}
9+
10+
if [[ -z "${COMMAND}" ]]; then
11+
echo "Usage: $0 <prepare|install|down> <flowable_root_dir> [rdm_root_dir]" >&2
12+
exit 1
13+
fi
14+
15+
wait_for_url() {
16+
local url="$1"
17+
local attempts=30
18+
local delay=10
19+
for ((i=1; i<=attempts; i++)); do
20+
if curl -f "$url" >/dev/null 2>&1; then
21+
echo "${url} is reachable"
22+
return 0
23+
fi
24+
echo "Attempt ${i}/${attempts} failed for ${url}" >&2
25+
sleep "$delay"
26+
done
27+
echo "Timed out waiting for ${url}" >&2
28+
return 1
29+
}
30+
31+
case "$COMMAND" in
32+
prepare)
33+
if [[ -z "${FLOWABLE_ROOT}" || -z "${RDM_ROOT}" ]]; then
34+
echo "Usage: $0 prepare <flowable_root_dir> <rdm_root_dir>" >&2
35+
exit 1
36+
fi
37+
38+
# Ensure RDM workflow keys directory exists
39+
mkdir -p "${RDM_ROOT}/addons/workflow/tests/keys"
40+
41+
# Generate RSA key pair for RDM service (used by RDM to sign tokens to gateway)
42+
openssl genrsa -out "${RDM_ROOT}/addons/workflow/tests/keys/rdm-service-v1.key" 2048
43+
openssl rsa -in "${RDM_ROOT}/addons/workflow/tests/keys/rdm-service-v1.key" -pubout \
44+
-out "${RDM_ROOT}/addons/workflow/tests/keys/rdm-service-v1.pub"
45+
46+
# Generate RSA key pair for gateway (used by gateway to sign tokens back to RDM)
47+
openssl genrsa -out "${RDM_ROOT}/addons/workflow/tests/keys/gateway-dev-v1.key" 2048
48+
openssl rsa -in "${RDM_ROOT}/addons/workflow/tests/keys/gateway-dev-v1.key" -pubout \
49+
-out "${RDM_ROOT}/addons/workflow/tests/keys/gateway-dev-v1.pub"
50+
51+
# Create workflow addon local.py for RDM
52+
mkdir -p "${RDM_ROOT}/addons/workflow/settings"
53+
cat > "${RDM_ROOT}/addons/workflow/settings/local.py" << 'EOF'
54+
RDM_TO_WORKFLOW_GATEWAY_KEYS = [
55+
{
56+
'kid': 'rdm-service-v1',
57+
'alg': 'RS256',
58+
'public_key_path': '/code/addons/workflow/tests/keys/rdm-service-v1.pub',
59+
'private_key_path': '/code/addons/workflow/tests/keys/rdm-service-v1.key',
60+
},
61+
]
62+
EOF
63+
64+
# Copy keys to gateway config
65+
cp "${RDM_ROOT}/addons/workflow/tests/keys/rdm-service-v1.key" "${FLOWABLE_ROOT}/config/rdm-service.key"
66+
cp "${RDM_ROOT}/addons/workflow/tests/keys/gateway-dev-v1.key" "${FLOWABLE_ROOT}/config/gateway-dev-v1.key"
67+
68+
# Create keyset.json with the RDM public key for gateway
69+
RDM_PUBLIC_KEY=$(cat "${RDM_ROOT}/addons/workflow/tests/keys/rdm-service-v1.pub" | python3 -c "import sys, json; print(json.dumps(sys.stdin.read()))")
70+
cat > "${FLOWABLE_ROOT}/config/keyset.json" << EOF
71+
{
72+
"keys": [
73+
{
74+
"kid": "rdm-service-v1",
75+
"alg": "RS256",
76+
"public_key": ${RDM_PUBLIC_KEY}
77+
}
78+
]
79+
}
80+
EOF
81+
82+
# Generate Fernet encryption key
83+
ENCRYPTION_KEY=$(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())")
84+
85+
# Create .env file for gateway CI environment
86+
cat > "${FLOWABLE_ROOT}/.env" << EOF
87+
# Flowable REST admin credentials
88+
FLOWABLE_REST_APP_ADMIN_USER_ID=rest-admin
89+
FLOWABLE_REST_APP_ADMIN_PASSWORD=rest-admin
90+
91+
# Gateway -> Flowable wiring
92+
FLOWABLE_REST_BASE_URL=http://flowable:8080/flowable-rest
93+
94+
# RDM keyset configuration
95+
RDM_KEYSET_PATH=/app/config/keyset.json
96+
97+
# Gateway internal wiring
98+
GATEWAY_INTERNAL_URL=http://gateway:8088
99+
RDM_ALLOWED_DOMAINS=http://192.168.168.167:5000
100+
RDM_ALLOWED_API_DOMAINS=http://192.168.168.167:8000
101+
RDM_ALLOWED_WATERBUTLER_URLS=http://192.168.168.167:7777
102+
103+
# Database connection
104+
DATABASE_URL=postgresql://gateway:gateway@postgres:5432/gateway
105+
106+
# Gateway signing key
107+
GATEWAY_SIGNING_KEY_ID=gateway-dev-v1
108+
GATEWAY_SIGNING_PRIVATE_KEY_PATH=/app/config/gateway-dev-v1.key
109+
110+
# Encryption key for stored delegation tokens
111+
ENCRYPTION_KEY=${ENCRYPTION_KEY}
112+
EOF
113+
echo "Flowable gateway configuration prepared"
114+
;;
115+
install)
116+
if [[ -z "${FLOWABLE_ROOT}" || ! -d "${FLOWABLE_ROOT}" ]]; then
117+
echo "Flowable root directory not found: ${FLOWABLE_ROOT}" >&2
118+
exit 1
119+
fi
120+
121+
pushd "${FLOWABLE_ROOT}"
122+
123+
# Build and start the stack
124+
docker compose up -d --build
125+
126+
popd
127+
128+
# Wait for gateway to be ready
129+
wait_for_url "http://192.168.168.167:8088/healthz"
130+
131+
# Initialize database
132+
pushd "${FLOWABLE_ROOT}"
133+
docker compose run --rm gateway python -m gateway.init_db
134+
popd
135+
136+
echo "Flowable gateway installed"
137+
;;
138+
down)
139+
if [[ -z "${FLOWABLE_ROOT}" || ! -d "${FLOWABLE_ROOT}" ]]; then
140+
echo "Flowable root directory not found, skipping cleanup"
141+
exit 0
142+
fi
143+
144+
pushd "${FLOWABLE_ROOT}"
145+
docker compose down -v || true
146+
popd
147+
;;
148+
*)
149+
echo "Unknown command: ${COMMAND}" >&2
150+
exit 1
151+
;;
152+
esac

‎.github/scripts/setup_test_data.py‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,26 @@
2323
'family_name_ja': 'ユーザー2',
2424
'password': 'testpass456',
2525
},
26+
{
27+
'username': 'teststaff@example.com',
28+
'fullname': 'Test Staff',
29+
'given_name': 'Test',
30+
'family_name': 'Staff',
31+
'given_name_ja': 'テスト',
32+
'family_name_ja': 'スタッフ',
33+
'password': 'testpass789',
34+
'is_staff': True,
35+
},
36+
{
37+
'username': 'testuser3@example.com',
38+
'fullname': 'Test User 3',
39+
'given_name': 'Test',
40+
'family_name': 'User 3',
41+
'given_name_ja': 'テスト',
42+
'family_name_ja': 'ユーザー3',
43+
'password': 'testpass321',
44+
'institution': 'Massachusetts Institute of Technology [Test]',
45+
},
2646
]
2747

2848
for user_data in test_users:
@@ -47,10 +67,11 @@
4767
user.have_email = True
4868
# Set jobs for profile completion (required for is_full_account_required_info)
4969
# Structure must match unserialize_job in website/profile/views.py
70+
user_institution = user_data.get('institution', INSTITUTION_NAME)
5071
user.jobs = [{
51-
'institution': INSTITUTION_NAME,
72+
'institution': user_institution,
5273
'department': None,
53-
'institution_ja': INSTITUTION_NAME,
74+
'institution_ja': user_institution,
5475
'department_ja': None,
5576
'title': None,
5677
'startMonth': None,
@@ -63,6 +84,9 @@
6384
if user_data.get('is_superuser', False):
6485
user.is_superuser = True
6586
user.is_staff = True
87+
# Set staff if specified (without superuser)
88+
elif user_data.get('is_staff', False):
89+
user.is_staff = True
6690
user.save()
6791

6892
# Create email for the user
@@ -101,9 +125,10 @@
101125
print(f"PROJECT_NAME_{username}: {project.title}")
102126

103127
# Affiliate users with institution
104-
inst = Institution.objects.get(name=INSTITUTION_NAME)
105128
for user_data in test_users:
106129
user = OSFUser.objects.get(username=user_data['username'])
130+
user_institution = user_data.get('institution', INSTITUTION_NAME)
131+
inst = Institution.objects.get(name=user_institution)
107132
if inst not in user.affiliated_institutions.all():
108133
user.affiliated_institutions.add(inst)
109134
user.save()

0 commit comments

Comments
 (0)