Skip to content

Commit cab5459

Browse files
dentinydentiny
committed
merge community extension template
Signed-off-by: dentiny <[email protected]>
1 parent 0d9d925 commit cab5459

16 files changed

+895
-0
lines changed

‎.github/workflows/_extension_archive.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#
2+
# Reusable workflow that deploys the extensions source code
3+
4+
name: Extension Archieve Sources
5+
on:
6+
workflow_call:
7+
inputs:
8+
# The name of the extension
9+
extension_name:
10+
required: true
11+
type: string
12+
repository:
13+
required: false
14+
type: string
15+
default: ""
16+
ref:
17+
required: false
18+
type: string
19+
default: ""
20+
jobs:
21+
fetch_repo:
22+
name: Fetch repo and save
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v3
26+
with:
27+
repository: ${{ inputs.repository }}
28+
ref: ${{ inputs.ref }}
29+
fetch-depth: 0
30+
submodules: 'true'
31+
path: ./repo
32+
33+
- name: Compress the repository sources
34+
run: |
35+
rm -rf repo/duckdb
36+
zip -9 -r compressed.zip repo
37+
38+
- name: Deploy the submodule
39+
shell: bash
40+
env:
41+
AWS_ACCESS_KEY_ID: ${{ secrets.DUCKDB_COMMUNITY_EXTENSION_S3_ID }}
42+
AWS_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_COMMUNITY_EXTENSION_S3_SECRET }}
43+
AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }}
44+
BUCKET_NAME: duckdb-community-extensions
45+
PIP_BREAK_SYSTEM_PACKAGES: 1
46+
run: |
47+
python3 -m pip install awscli
48+
if [ "${AWS_SECRET_ACCESS_KEY}" ]; then
49+
aws s3 cp compressed.zip s3://${BUCKET_NAME}/archived_extension_sources/${{ inputs.extension_name }}/repo_sources_${{ inputs.ref }}.zip
50+
fi

‎.github/workflows/_extension_deploy.yml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#
2+
# Reusable workflow that deploys the artifacts produced by .github/workflows/_extension_distribution.yml
3+
#
4+
# Note: this workflow can be used to deploy an extension from a GH artifact. It is intended for Out-of-tree
5+
# extensions that live the duckdb GitHub organisation (only repo's within the same organisation can share secrets
6+
# through reusable workflows)
7+
8+
9+
name: Extension Deployment
10+
on:
11+
workflow_call:
12+
inputs:
13+
# The name of the extension
14+
extension_name:
15+
required: true
16+
type: string
17+
# DuckDB version to build against
18+
duckdb_version:
19+
required: true
20+
type: string
21+
# ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64'
22+
exclude_archs:
23+
required: false
24+
type: string
25+
default: ""
26+
# Whether to upload this deployment as the latest. This may overwrite a previous deployment.
27+
deploy_latest:
28+
required: false
29+
type: boolean
30+
default: false
31+
# Whether to upload this deployment under a versioned path. These will not be deleted automatically
32+
deploy_versioned:
33+
required: false
34+
type: boolean
35+
default: false
36+
# Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times
37+
artifact_postfix:
38+
required: false
39+
type: string
40+
default: ""
41+
# Override the default deploy script with a custom script
42+
deploy_script:
43+
required: false
44+
type: string
45+
default: "./duckdb/scripts/extension-upload-single.sh"
46+
# Override the default matrix parse script with a custom script
47+
matrix_parse_script:
48+
required: false
49+
type: string
50+
default: "./duckdb/scripts/modify_distribution_matrix.py"
51+
repository:
52+
required: false
53+
type: string
54+
default: ""
55+
ref:
56+
required: false
57+
type: string
58+
default: ""
59+
jobs:
60+
generate_matrix:
61+
name: Generate matrix
62+
runs-on: ubuntu-latest
63+
outputs:
64+
deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }}
65+
steps:
66+
- uses: actions/checkout@v3
67+
with:
68+
repository: ${{ inputs.repository }}
69+
ref: ${{ inputs.ref }}
70+
fetch-depth: 0
71+
submodules: 'true'
72+
73+
- uses: actions/checkout@v3
74+
with:
75+
repository: 'duckdb/duckdb'
76+
ref: ${{ inputs.duckdb_version }}
77+
path: duckdb
78+
fetch-depth: 0
79+
80+
- id: parse-matrices
81+
run: |
82+
python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
83+
deploy_matrix="`cat deploy_matrix.json`"
84+
echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT
85+
echo `cat $GITHUB_OUTPUT`
86+
87+
deploy:
88+
name: Deploy
89+
runs-on: ubuntu-latest
90+
needs: generate_matrix
91+
if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }}
92+
strategy:
93+
matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}}
94+
95+
steps:
96+
- uses: actions/checkout@v3
97+
with:
98+
repository: ${{ inputs.repository }}
99+
ref: ${{ inputs.ref }}
100+
fetch-depth: 0
101+
submodules: 'false'
102+
103+
- uses: actions/checkout@v3
104+
with:
105+
repository: 'duckdb/duckdb'
106+
ref: ${{ inputs.duckdb_version }}
107+
path: duckdb
108+
fetch-depth: 0
109+
110+
- uses: actions/download-artifact@v4
111+
with:
112+
name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}}
113+
path: |
114+
/tmp/extension
115+
116+
- name: Deploy
117+
shell: bash
118+
env:
119+
AWS_ACCESS_KEY_ID: ${{ secrets.DUCKDB_COMMUNITY_EXTENSION_S3_ID }}
120+
AWS_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_COMMUNITY_EXTENSION_S3_SECRET }}
121+
AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }}
122+
BUCKET_NAME: duckdb-community-extensions
123+
DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_COMMUNITY_SIGNING_PK }}
124+
DUCKDB_DEPLOY_SCRIPT_MODE: for_real
125+
PIP_BREAK_SYSTEM_PACKAGES: 1
126+
run: |
127+
pwd
128+
python3 -m pip install awscli
129+
git config --global --add safe.directory '*'
130+
cd duckdb
131+
git fetch --tags
132+
export DUCKDB_VERSION=`git tag --points-at HEAD`
133+
export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
134+
cd ..
135+
git fetch --tags
136+
export EXT_VERSION=`git tag --points-at HEAD`
137+
export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`}
138+
${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}}
139+
140+
clean_cache:
141+
needs:
142+
- deploy
143+
- generate_matrix
144+
if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }}
145+
strategy:
146+
matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}}
147+
uses: ./.github/workflows/clean_caches.yml
148+
secrets: inherit
149+
with:
150+
pattern: '*/${{ matrix.duckdb_arch }}/${{ inputs.extension_name }}.duckdb_extension.*'

‎.github/workflows/build.yml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
name: Community Extension Build
2+
on:
3+
pull_request:
4+
paths-ignore:
5+
- '**'
6+
- '!scripts/build.py'
7+
- '!.github/workflows/build.yml'
8+
- '!extensions/*/description.yml'
9+
push:
10+
paths-ignore:
11+
- '**'
12+
- '!scripts/build.py'
13+
- '!.github/workflows/build.yml'
14+
- '!extensions/*/description.yml'
15+
workflow_dispatch:
16+
inputs:
17+
extension_name:
18+
type: string
19+
duckdb_version:
20+
type: string
21+
duckdb_tag:
22+
type: string
23+
deploy:
24+
type: string
25+
more_excluded:
26+
type: string
27+
skip_tests:
28+
required: false
29+
type: boolean
30+
default: false
31+
workflow_call:
32+
inputs:
33+
extension_name:
34+
type: string
35+
duckdb_version:
36+
type: string
37+
duckdb_tag:
38+
type: string
39+
deploy:
40+
type: string
41+
more_excluded:
42+
type: string
43+
skip_tests:
44+
required: false
45+
type: boolean
46+
default: false
47+
48+
jobs:
49+
prepare:
50+
outputs:
51+
COMMUNITY_EXTENSION_NAME: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_NAME }}
52+
COMMUNITY_EXTENSION_GITHUB: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_GITHUB }}
53+
COMMUNITY_EXTENSION_REF: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_REF }}
54+
COMMUNITY_EXTENSION_DEPLOY: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_DEPLOY }}
55+
COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS: ${{ inputs.more_excluded }}${{ steps.parse.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }}
56+
COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS }}
57+
COMMUNITY_EXTENSION_VCPKG_URL: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL == '' && 'https://github.com/microsoft/vcpkg.git' || steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL }}
58+
COMMUNITY_EXTENSION_VCPKG_COMMIT: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT == '' && '5e5d0e1cd7785623065e77eff011afdeec1a3574' || steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT }}
59+
env:
60+
DUCKDB_LATEST_STABLE: 'v1.2.0'
61+
DUCKDB_VERSION: ${{ inputs.duckdb_version || 'v1.2.0' }}
62+
runs-on: ubuntu-latest
63+
steps:
64+
- uses: actions/checkout@v4
65+
with:
66+
fetch-depth: 0
67+
68+
- name: Get all changed description files
69+
id: changed-files
70+
uses: tj-actions/changed-files@v44
71+
with:
72+
files: |
73+
extensions/*/description.yml
74+
75+
- name: Parse description files
76+
id: parse
77+
env:
78+
ALL_CHANGED_FILES: ${{ github.event_name == 'workflow_dispatch' && format('extensions/{0}/description.yml', inputs.extension_name) || steps.changed-files.outputs.all_changed_files }}
79+
PIP_BREAK_SYSTEM_PACKAGES: 1
80+
run: |
81+
pip install pyyaml
82+
# scripts/build.py takes DUCKDB_VERSION from environment
83+
python scripts/build.py
84+
cat env.sh >> $GITHUB_OUTPUT
85+
echo `cat $GITHUB_OUTPUT`
86+
87+
build:
88+
needs: prepare
89+
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
90+
if: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME != '' }}
91+
with:
92+
duckdb_version: ${{ inputs.duckdb_version || 'v1.2.0' }}
93+
duckdb_tag: ${{ inputs.duckdb_tag || '' }}
94+
ci_tools_version: 'main'
95+
exclude_archs: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }}
96+
extra_toolchains: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS }}
97+
extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }}
98+
override_repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }}
99+
override_ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }}
100+
vcpkg_url: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_VCPKG_URL}}
101+
vcpkg_commit: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT}}
102+
skip_tests: ${{ inputs.skip_tests || false }}
103+
104+
doc_test:
105+
if: ${{ inputs.deploy != 'false' }}
106+
needs:
107+
- prepare
108+
- build
109+
uses: ./.github/workflows/generate_docs.yml
110+
with:
111+
extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }}
112+
113+
archive:
114+
needs:
115+
- doc_test
116+
- prepare
117+
- build
118+
uses: ./.github/workflows/_extension_archive.yml
119+
if: ${{ (github.head_ref || github.ref_name) == 'main' }}
120+
secrets: inherit
121+
with:
122+
extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }}
123+
repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }}
124+
ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }}
125+
126+
deploy:
127+
needs:
128+
- prepare
129+
- build
130+
uses: ./.github/workflows/_extension_deploy.yml
131+
if: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_DEPLOY != '' }}
132+
secrets: inherit
133+
with:
134+
deploy_latest: true
135+
duckdb_version: ${{ inputs.duckdb_version || 'v1.2.0' }}
136+
exclude_archs: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }}
137+
extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }}
138+
repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }}
139+
ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }}

‎.github/workflows/build_all.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build all extensions
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
duckdb_version:
6+
type: string
7+
duckdb_tag:
8+
type: string
9+
deploy:
10+
type: string
11+
12+
jobs:
13+
collect_extensions:
14+
outputs:
15+
COMMUNITY_EXTENSION_LIST: ${{ steps.generate_list.outputs.EXTENSION_LIST }}
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v4
19+
with:
20+
fetch-depth: 0
21+
22+
- name: Generate extension list
23+
id: generate_list
24+
run: |
25+
./scripts/get_extension_list.sh
26+
cat extension_list
27+
cat extension_list >> $GITHUB_OUTPUT
28+
29+
build_all:
30+
needs:
31+
- collect_extensions
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
extension_name: ${{ fromJson(needs.collect_extensions.outputs.COMMUNITY_EXTENSION_LIST) }}
36+
uses: ./.github/workflows/build.yml
37+
secrets: inherit
38+
with:
39+
extension_name: ${{ matrix.extension_name }}
40+
duckdb_version: ${{ inputs.duckdb_version }}
41+
duckdb_tag: ${{ inputs.duckdb_tag }}
42+
deploy: ${{ inputs.deploy }}

0 commit comments

Comments
 (0)