Refactor pg connection

copier.yml

@@ -38,12 +38,6 @@ odoo_version:
   default: "{{branch_name}}"
   when: false
 
-# Remove ?
-use_secret:
-  type: bool
-  help: generate docker-compose dedicate file for secret
-  # only for bootstraping clea-ci|prod.secrets.docker-compose.yml
-
 org_name:
   type: str
   default: Akretion

src/.gitlab-ci.yml.jinja

@@ -64,9 +64,9 @@ before_script:
   - echo $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED
   - echo $CI_COMMIT_REF_SLUG
   - export MAIN_BRANCH=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}}
-  - export BUILD_NAME=${CI_PROJECT_NAME}--${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}}
-  - export CI_DOMAIN=${CI_DOMAIN}
-  - export DOMAIN=$BUILD_NAME.${CI_DOMAIN}
+  - export BUILD_NAME=${CI_PROJECT_NAME}_${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}}
+  - export PGDATABASE=$BUILD_NAME
+  - export DOMAIN=${CI_PROJECT_NAME}--${CI_MERGE_REQUEST_IID:-preprod-${CI_COMMIT_REF_SLUG}}.${CI_DOMAIN}
 
 prebuild:
   # warmup caches once per MR
@@ -98,16 +98,12 @@ pregetdb:
   script:
     # no need to wait for a build
     # can be run from anywhere
-    # manage secrets
-    # run only if ci.secrets.docker-compose.yml exists
-    - test -f ci.secrets.docker-compose.yml && sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml
-    # if no secrets exists, create an empty one to make .env happy
-    - test -f secrets.docker-compose.yml || touch secrets.docker-compose.yml
 
     # TODO: factorise this:
     - cp .env-ci .env
     # TODO use copier or something to inialize this env variables
     - echo BUILD_NAME=${BUILD_NAME} >> .env
+    - echo PGDATABASE=${BUILD_NAME} >> .env
     - echo CI_PROJECT_NAME=${CI_PROJECT_NAME} >> .env
     - echo "DOMAIN=${DOMAIN}" >> .env
     # uid , gid used by docker in postgres to fix perms on .db
@@ -139,6 +135,8 @@ fetch:
     - git remote add --mirror=fetch $REMOTE ~/$REMOTE || true
     - git fetch --all
     - git checkout ${NEW_HEAD}
+    - sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml
+    - ./bin/generate_env
   needs: ["prebuild"]
 
 build:
@@ -147,19 +145,6 @@ build:
     # ak build then docker build
     - cd ~gitlab-runner/builds/${AK_WORKING_DIR}
 
-    - cp .env-ci .env
-    # TODO use copier or something to inialize this env variables
-    - echo BUILD_NAME=${BUILD_NAME} >> .env
-    - echo CI_PROJECT_NAME=${CI_PROJECT_NAME} >> .env
-    - echo "DOMAIN=${DOMAIN}" >> .env
-    # uid , gid used by docker in postgres to fix perms on .db
-    - echo "UID=`id -u`" >> .env
-    - echo "GID=`id -g`" >> .env
-
-    # manage secrets
-    # run only if ci.secrets.docker-compose.yml exists
-    - test -f ci.secrets.docker-compose.yml && sops -d ci.secrets.docker-compose.yml > secrets.docker-compose.yml
-
     # ak build
     - cd odoo && time ak clone && time ak sparse && time ak build && cd ..
 
@@ -310,55 +295,31 @@ lint:
   # can be run sooner
   needs: ["fetch"]
 
-
 test:
   stage: test
   script:
     - CURRENT_PATH=`pwd`
     - cd ~gitlab-runner/builds/${AK_WORKING_DIR}
-    # get list of modules to test
-    # add back --select-exclude ${EXCLUDE_MODULES:-none} from a path
-    # qu'est ce que ça fait ça ?
-    #- docker compose run odoo initdb ${DB_NAME} --cache-prefix ${CI_PROJECT_NAME:0:7}
-    # TODO: réflechir si on laisse ça ici ou dans le conteneur
-
-    # list of all local-src modules
-    - ADDONS_TEST=$(manifestoo --select-addons-dir odoo/local-src list --separator=,)
-    - ADDONS_TEST_DEP=$(manifestoo --select-addons-dir odoo/local-src list-depends --separator=,)
-    - echo $ADDONS_TEST
-    - echo $ADDONS_TEST_DEP
-    # start from a clean state
-    - docker compose --profile db run --rm bedrock dropdb --force --if-exists ${BUILD_NAME}_test
-    # --labels traefik.enable=false in order to not mess with the review
-    # init db with dependencies of our modules
-    # we force PG port in order to bypass pgbouncer
-    - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" -e "PGPORT=5432" -e "DB_PORT=5432" --rm odoo click-odoo-initdb -n ${BUILD_NAME}_test -m ${ADDONS_TEST_DEP} --cache-prefix ${CI_PROJECT_NAME:0:7}
-    # run the tests here
-    - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage run --include "/odoo/local-src/*" --branch /odoo/bin/odoo --stop-after-init -i ${ADDONS_TEST} --test-enable --workers=0
-    # display report in MR
-    - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage report
-    # export for the artifacts
-    - docker compose run --labels traefik.enable=false -e DB_NAME="${BUILD_NAME}_test" -e PGDATABASE="${BUILD_NAME}_test" --rm --workdir /data/odoo/filestore odoo coverage xml --skip-empty
-    # extract coverage report
-    - docker compose cp odoo:/data/odoo/filestore/coverage.xml coverage.xml
-    # move coverage report to accessible location from gitlab-ci
-    - cp coverage.xml ${CURRENT_PATH}/coverage.xml
+    - docker compose --env-file .env-test kill
+    - dropdb --force --if-exists ${BUILD_NAME}_test
+    - docker compose --env-file .env-test run --rm odoo initdb ${BUILD_NAME}_test --cache-prefix ${CI_PROJECT_NAME:0:7}
+    - docker compose --env-file .env-test run --rm odoo runtests
+    - mv shared $CURRENT_PATH/
   rules:
     # Run tests in MR if there is no Skiptest tag
     - if: $AK_IS_MR == "true" && $CI_MERGE_REQUEST_LABELS !~ /Skiptest/
+    # Always run test in major branch
+    - if: $AK_IS_MAJOR_BRANCH == "true" && $AK_DO_OPENUPGRADE_MIGRATION != "true"
   needs:
-      # update_db do a docker compose stop, we don't want to be halted
-      # if this job start sooner
-      # optional true, let us the ability to run if update_db is cancelled
-    - job: "update_db"
-      optional: true
-  allow_failure: true
-  coverage: '/TOTAL(\W+\d+){5}\%/'
+    - job: "build"
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
+    when: always
     reports:
+      junit: shared/test_results/*.xml
       coverage_report:
         coverage_format: cobertura
-        path: coverage.xml
+        path: shared/coverage.xml
 
 # Run the container used for the review
 review:
@@ -401,7 +362,7 @@ stop_review:
 review_preprod:
   stage: review_preprod
   script:
-    - echo "DB_NAME=$BUILD_NAME" >> .env
+    - echo "PGDATABASE=$BUILD_NAME" >> .env
     - echo "DOMAIN=${CI_PROJECT_NAME}--preprod-${CI_COMMIT_REF_SLUG}.${CI_DOMAIN}" >> .env
     - cd ~gitlab-runner/builds/${AK_WORKING_DIR}
     - docker compose up -d

src/bin/generate_env

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euxo pipefail
+
+# generate env file (env, .env-test)
+# Usage:
+#   ./generate_env
+
+echo "generate env file"
+cp .env-ci .env
+# uid , gid used by docker in postgres to fix perms on .db
+cat << EOF >> .env
+BUILD_NAME=${BUILD_NAME}
+CI_PROJECT_NAME=${CI_PROJECT_NAME}
+DOMAIN=${DOMAIN}
+UID=`id -u`
+GID=`id -g`
+EOF
+
+cp .env .env-test
+echo "PGDATABASE=${BUILD_NAME}" >> .env
+
+cat << EOF >> .env
+ENABLE_TRAEFIK=false
+COMPOSE_PROJECT_NAME=${BUILD_NAME}_test
+PGDATABASE=${BUILD_NAME}_test
+EOF

src/ci.docker-compose.yml.jinja

@@ -31,19 +31,16 @@ services:
       # but take care to use a different DSN than the production
 
       PGSSLMODE: require
-      DB_NAME: ${BUILD_NAME}
-      PGDATABASE: ${BUILD_NAME}
-      DB_USER: ${CI_PROJECT_NAME}
-      PGUSER: ${CI_PROJECT_NAME}
-      DB_HOST:
-      PGHOST:
+      PGDATABASE: $PGDATABASE
+      PGHOST: $PGHOST
+      PGPASSWORD: $PGPASSWORD
+      PGUSER: $PGUSER
 {% if branch_name != "18.0" %}
       # use pgbouncer
-      DB_PORT: 6432
       PGPORT: 6432
 {% endif %}
     labels:
-      - "traefik.enable=true"
+      - "traefik.enable=${ENABLE_TRAEFIK:-true}"
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.rule=Host(`${DOMAIN}`)"
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.tls=false"
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME}-long.rule=Host(`${DOMAIN}`) && PathPrefix(`/longpolling/`)"
@@ -65,24 +62,19 @@ services:
         target: /data/odoo/filestore
       - type: bind
         source: ./shared
-        target: /data/odoo/shared
+        target: /odoo/shared
         bind:
           # create shared dir if needed
           create_host_path: true
   bedrock:
     image: ghcr.io/akretion/odoo-docker:{{ odoo_version }}-light-latest
     profiles: [db]
     environment:
-      DB_NAME: ${BUILD_NAME}
-      PGDATABASE: ${BUILD_NAME}
-      DB_USER: ${CI_PROJECT_NAME}
-      PGUSER: ${CI_PROJECT_NAME}
-      DB_HOST:
-      PGHOST:
-      {% if not use_secret %}
-      PGPASSWORD: ${PGPASSWORD}
-      DB_PASSWORD: ${PGPASSWORD}
-      {% endif %}
+      PGSSLMODE: require
+      PGDATABASE: $PGDATABASE
+      PGHOST: $PGHOST
+      PGPASSWORD: $PGPASSWORD
+      PGUSER: $PGUSER
       # we give AK_TEMPLATE_DB here for get_spare
       AK_TEMPLATE_DB: ${AK_TEMPLATE_DB}
     volumes:

src/{% if use_secret %}clear-ci.secrets.docker-compose.yml{% endif %}.jinja → src/clear-ci.secrets.docker-compose.yml.jinja

@@ -5,12 +5,6 @@
 services:
   odoo:
     environment:
-      DB_PASSWORD:
-      PGPASSWORD:
       ENCRYPTION_KEY_CI:
       AUTH_OAUTH_PROVIDER_CLIENT_ID: {{ project_name }}
       AUTH_OAUTH_PROVIDER_CLIENT_SECRET:
-  bedrock:
-    environment:
-      DB_PASSWORD:
-      PGPASSWORD:

src/{% if use_secret %}clear-prod.secrets.docker-compose.yml{% endif %}.jinja → src/clear-prod.secrets.docker-compose.yml.jinja

@@ -4,8 +4,6 @@
 services:
   odoo:
     environment:
-      DB_PASSWORD:
-      PGPASSWORD:
       ENCRYPTION_KEY_PROD:
       ENCRYPTION_KEY_CI:
       AUTH_OAUTH_PROVIDER_CLIENT_ID: {{ project_name }}

src/dev.docker-compose.yml.jinja

@@ -37,11 +37,6 @@ services:
       # you may want to add queue_job here
       SERVER_WIDE_MODULES: web
 
-      # DB_ and PG specified because entrypoint is not always ran if docker exec
-      DB_USER: odoo
-      DB_HOST: db
-      DB_PASSWORD: odoo
-      DB_NAME: db
       PGDATABASE: db
       PGUSER: odoo
       PGHOST: db

src/odoo/Dockerfile.jinja

@@ -22,6 +22,8 @@ COPY ./src/odoo/tools /odoo/src/odoo/tools
 COPY ./src/odoo/upgrade /odoo/src/odoo/upgrade
 RUN /odoo/bin/pip install -e /odoo/src
 
+COPY bin/* /usr/local/bin/
+
 FROM base as thisproject
 COPY ./requirements.txt /odoo/requirements.txt
 RUN /odoo/bin/pip install --no-cache-dir \

src/odoo/bin/initdb

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Usage: initdb dbtest
+
+echo "Create database $1"
+
+EXTRA_ARGS="${@:2}"
+ADDONS_INIT=$(manifestoo --select-exclude ${EXCLUDE_MODULES:-none} --select-addons-dir /odoo/local-src list-depends --separator=,)
+
+# Some module like product_code_unique can break the installation if their are installed too early, so you can use the variable "EXCLUDE_DEPENDENCIES" to exclude them
+
+export ADDONS_INIT
+ADDONS_INIT=l10n_generic_coa,$(python3 -c "import os; print(','.join([x for x in os.environ.get('ADDONS_INIT').split(',') if x not in os.environ.get('EXCLUDE_DEPENDENCIES', '').split(',')]))")
+
+click-odoo-initdb -n $1 -m ${ADDONS_INIT} ${EXTRA_ARGS}

src/odoo/bin/runtests

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Usage: runtest
+
+# Run tests. Use unbuffer to get a colored output.
+
+ADDONS_TEST=$(manifestoo --select-exclude ${EXCLUDE_MODULES:-none} --select-addons-dir /odoo/local-src list --separator=,)
+# Move to shared folder so coverage file will be shared with the host
+# and so accessible by the gitlab runner
+cd /odoo/shared
+unbuffer coverage run --include "/odoo/local-src/*" --branch $(which odoo) --stop-after-init -i ${ADDONS_TEST} --test-enable --workers=0
+ls -alh
+coverage report -i --precision=2 --skip-empty
+coverage xml -i -o coverage.xml
+
+sed -i "s=<source></source>=<source>/home/gitlab-runner/builds</source>=g" /odoo/shared/coverage.xml
+sed -i "s=/odoo/local-src=${CI_PROJECT_DIR}/odoo/local-src=g" /odoo/shared/coverage.xml
+sed -i "s=/home/gitlab-runner/builds/=builds/=g" /odoo/shared/coverage.xml

src/odoo/local-src/{{ project_name }}/__manifest__.py.jinja → src/odoo/local-src/custom_all/__manifest__.py.jinja

@@ -16,6 +16,7 @@
         "auth_oidc_akretion_data",
         "auth_oidc_environment",
         "database_age_cron",
+        "odoo_test_xmlrunner",
         "server_environment_ir_config_parameter",
         "web_environment_ribbon",
     ],

src/odoo/requirements.txt

@@ -3,6 +3,7 @@ pyjwt # auth_oidc
 python-jose[cryptography] # auth_oidc
 #sentry 1.9.0 fixé dans le manifest du module sentry pour le moment
 sentry-sdk<=1.9.0
+unittest-xml-reporting # odoo_test_xmlrunner
 
 #apispec #shopinvader
 #boto3==1.20.32 # shopinvader

src/odoo/spec.yaml.jinja

@@ -25,6 +25,7 @@ server-tools:
     modules:
        - sentry
        - bus_alt_connection
+       - odoo_test_xmlrunner
     src: https://github.com/oca/server-tools {{ odoo_version }}
 
 web:

src/prod.docker-compose.yml.jinja

@@ -39,17 +39,13 @@ services:
       SERVER_WIDE_MODULES: web,sentry,bus_alt_connection,queue_job
 {% endif %}
 
-      # ask infra team for the following
       PGSSLMODE: require
-      DB_NAME:
-      PGDATABASE:
-      DB_HOST:
-      PGHOST:
-      PGUSER:
-      DB_USER:
+      PGDATABASE: $PGDATABASE
+      PGHOST: $PGHOST
+      PGPASSWORD: $PGPASSWORD
+      PGUSER: $PGUSER
 {% if branch_name != "18.0" %}
       # use pgbouncer
-      DB_PORT: 6432
       PGPORT: 6432
 {% endif %}