Skip to content

Commit fcb3b5e

Browse files
raptorsunraptorsun
committed
konflux: do not fetch binary python wheels
Signed-off-by: Haoyu Sun <hasun@redhat.com>
1 parent 4f99014 commit fcb3b5e

15 files changed

Lines changed: 1057 additions & 291 deletions

‎.tekton/lightspeed-stack-pull-request.yaml‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,41 @@ spec:
3434
- name: build-source-image
3535
value: 'true'
3636
- name: prefetch-input
37-
value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt", "requirements.torch.txt"]}]'
37+
value: |
38+
[
39+
{
40+
"type": "rpm",
41+
"path": "."
42+
},
43+
{
44+
"type": "pip",
45+
"path": ".",
46+
"requirements_files": [
47+
"requirements.x86_64.txt",
48+
"requirements.aarch64.txt",
49+
"requirements.hermetic.txt",
50+
"requirements.torch.txt",
51+
"requirements.binary.txt"
52+
],
53+
"requirements_build_files": [
54+
"requirements-build.txt"
55+
],
56+
"binary": {
57+
"os": "linux",
58+
"arch": "x86_64,aarch64",
59+
"py_version": "312",
60+
"packages": "torch,faiss-cpu,uv"
61+
}
62+
},
63+
{
64+
"type": "generic",
65+
"path": "."
66+
}
67+
]
3868
- name: hermetic
3969
value: 'true'
70+
- name: hermeto-mode
71+
value: 'permissive'
4072
- name: dockerfile
4173
value: Containerfile
4274
pipelineSpec:
@@ -195,6 +227,8 @@ spec:
195227
value: $(params.output-image).prefetch
196228
- name: ociArtifactExpiresAfter
197229
value: $(params.image-expires-after)
230+
- name: mode
231+
value: $(params.hermeto-mode)
198232
runAfter:
199233
- clone-repository
200234
taskRef:

‎.tekton/lightspeed-stack-push.yaml‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,41 @@ spec:
3131
- name: build-source-image
3232
value: 'true'
3333
- name: prefetch-input
34-
value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt", "requirements.torch.txt"]}]'
34+
value: |
35+
[
36+
{
37+
"type": "rpm",
38+
"path": "."
39+
},
40+
{
41+
"type": "pip",
42+
"path": ".",
43+
"requirements_files": [
44+
"requirements.x86_64.txt",
45+
"requirements.aarch64.txt",
46+
"requirements.hermetic.txt",
47+
"requirements.torch.txt",
48+
"requirements.binary.txt"
49+
],
50+
"requirements_build_files": [
51+
"requirements-build.txt"
52+
],
53+
"binary": {
54+
"os": "linux",
55+
"arch": "x86_64,aarch64",
56+
"py_version": "312",
57+
"packages": "torch,faiss-cpu,uv"
58+
}
59+
},
60+
{
61+
"type": "generic",
62+
"path": "."
63+
}
64+
]
3565
- name: hermetic
3666
value: 'true'
67+
- name: hermeto-mode
68+
value: 'permissive'
3769
- name: dockerfile
3870
value: Containerfile
3971
pipelineSpec:
@@ -186,6 +218,8 @@ spec:
186218
value: $(params.output-image).prefetch
187219
- name: ociArtifactExpiresAfter
188220
value: $(params.image-expires-after)
221+
- name: mode
222+
value: $(params.hermeto-mode)
189223
runAfter:
190224
- clone-repository
191225
taskRef:

‎Containerfile‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ USER root
1515

1616
# Install gcc - required by polyleven python package on aarch64
1717
# (dependency of autoevals, no pre-built binary wheels for linux on aarch64)
18-
RUN dnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs gcc
18+
# cargo for maturin requirements
19+
RUN dnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs gcc cmake cargo
1920

2021
# Install uv package manager
2122
RUN pip3.12 install "uv>=0.8.15"
@@ -25,13 +26,25 @@ RUN pip3.12 install "uv>=0.8.15"
2526
COPY ${LSC_SOURCE_DIR}/src ./src
2627
COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.*.txt ./
2728

29+
# Extract generic dependencies from artifacts.lock.yaml and install them
30+
RUN if [ -f /cachi2/cachi2.env ]; then \
31+
tar -xzf /cachi2/output/deps/generic/googletest-release-1.12.1.tar.gz -C /tmp && \
32+
cd /tmp/googletest-release-1.12.1 && \
33+
cmake . && \
34+
make && \
35+
make install; \
36+
fi
37+
2838
# Bundle additional dependencies for library mode.
2939
# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
3040
# cachi2.env has these env vars:
3141
# PIP_FIND_LINKS=/cachi2/output/deps/pip
3242
# PIP_NO_INDEX=true
3343
RUN if [ -f /cachi2/cachi2.env ]; then \
34-
. /cachi2/cachi2.env && uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && . .venv/bin/activate && pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements.$(uname -m).txt -r requirements.torch.txt; \
44+
. /cachi2/cachi2.env && \
45+
uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && \
46+
. .venv/bin/activate && \
47+
pip install --no-index --find-links ${PIP_FIND_LINKS} --no-binary :all: -r requirements.$(uname -m).txt && pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements-binary.txt -r requirements.torch.txt; \
3548
else \
3649
uv sync --locked --no-dev --group llslibdev; \
3750
fi

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ PYTHON_REGISTRY = pypi
66

77
# PyTorch version
88
TORCH_VERSION := 2.7.1
9-
9+
# Faiss version
10+
FAISS_VERSION := 1.13.0
1011

1112
run: ## Run the service locally
1213
uv run src/lightspeed_stack.py
@@ -113,7 +114,9 @@ konflux-requirements: ## generate hermetic requirements.*.txt file for konflux b
113114
uv pip compile pyproject.toml -o requirements.aarch64.txt --generate-hashes --group llslibdev --python-platform aarch64-unknown-linux-gnu --torch-backend cpu --python-version 3.12 --refresh
114115
./scripts/remove_torch_deps.sh requirements.x86_64.txt
115116
./scripts/remove_torch_deps.sh requirements.aarch64.txt
116-
echo "torch==${TORCH_VERSION}" | uv pip compile - -o requirements.torch.txt --generate-hashes --python-version 3.12 --torch-backend cpu --emit-index-url --no-deps --index-url https://download.pytorch.org/whl/cpu --refresh
117+
echo "torch==${TORCH_VERSION}" | uv pip compile - -o requirements.torch.txt --generate-hashes --python-version 3.12 --torch-backend cpu --emit-index-url --no-deps --index-url https://download.pytorch.org/whl/cpu --refresh
118+
echo "faiss-cpu==${FAISS_VERSION}" | uv pip compile - -o requirements.binary.txt --generate-hashes --python-version 3.12 --no-deps --universal
119+
uv run pybuild-deps compile --output-file=requirements-build.txt requirements.aarch64.txt requirements.x86_64.txt requirements.hermetic.txt
117120

118121
help: ## Show this help screen
119122
@echo 'Usage: make <OPTIONS> ... <TARGETS>'

‎artifacts.lock.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
metadata:
3+
version: "1.0"
4+
artifacts:
5+
- download_url: "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz"
6+
checksum: "sha256:81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2"
7+
filename: "googletest-release-1.12.1.tar.gz"

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ dev = [
122122
"openapi-to-md>=0.1.0b2",
123123
"pytest-subtests>=0.14.2",
124124
"bandit>=1.8.6",
125+
"pybuild-deps==0.5.0",
126+
"pip==24.3.1",
125127
]
126128
llslibdev = [
127129
# To check llama-stack API provider dependecies:

0 commit comments

Comments
 (0)