diff --git a/.env.from-dump b/.env.from-dump index 4e21a4685..8381294e4 100644 --- a/.env.from-dump +++ b/.env.from-dump @@ -1,2 +1,4 @@ IGNORE_SEEDING=true DONT_EXECUTE_TRIGGERS=true +FROM_DUMP=true +SKIP_CACHE_REBUILD=true diff --git a/.gitignore b/.gitignore index 2e84540ba..2f11a0ed9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ prisma-local/seeds/productionTxs.csv paybutton-config.json dump.sql* +.forever diff --git a/Makefile b/Makefile index f6542b34f..3ea9680dc 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,18 @@ dev-from-dump: $(touch_local_env) docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d +stop-dev-from-dump: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down + +reset-dev-from-dump: + make stop-dev-from-dump && make dev-from-dump + +reset-dev-from-dump-keep-db: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d --force-recreate --no-deps paybutton + +nuke-dev-from-dump: + docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down -v + logs-dev: docker logs -f paybutton-dev diff --git a/docker-compose-from-dump.yml b/docker-compose-from-dump.yml index da676c508..ee778cf27 100644 --- a/docker-compose-from-dump.yml +++ b/docker-compose-from-dump.yml @@ -42,6 +42,7 @@ services: - ./scripts/db/init-supertokens-db.sql:/home/mysql/raw_entrypoint/3-init-supertokens-db.sql - ./scripts/db/prisma-shadow-db-fix.sql:/home/mysql/raw_entrypoint/4-prisma-shadow-db-fix.sql - ./dump.sql:/home/mysql/raw_entrypoint/5-dump.sql + - paybutton-dump-db-data:/var/lib/mysql users-service: container_name: paybutton-users-service @@ -69,3 +70,6 @@ services: - ./redis:/data/redis:z - ./scripts:/data/scripts command: sh -c "sed -i 's/\\r//g' ./scripts/redis-start.sh && sh ./scripts/redis-start.sh" + +volumes: + paybutton-dump-db-data: diff --git a/redis/paymentCache.ts b/redis/paymentCache.ts index ad18d9f6a..fb6939445 100755 --- a/redis/paymentCache.ts +++ b/redis/paymentCache.ts @@ -31,9 +31,11 @@ export async function * getUserUncachedAddresses (userId: string): AsyncGenerato } export const getPaymentList = async (userId: string): Promise => { - const uncachedAddressStream = getUserUncachedAddresses(userId) - for await (const address of uncachedAddressStream) { - void await CacheSet.addressCreation(address) + if (process.env.SKIP_CACHE_REBUILD === undefined) { + const uncachedAddressStream = getUserUncachedAddresses(userId) + for await (const address of uncachedAddressStream) { + void await CacheSet.addressCreation(address) + } } return await getCachedPaymentsForUser(userId) } @@ -304,6 +306,7 @@ export const clearPaymentCacheForAddress = async (addressString: string): Promis } export const initPaymentCache = async (address: Address): Promise => { + if (process.env.SKIP_CACHE_REBUILD !== undefined) return false const cachedKeys = await getCachedWeekKeysForAddress(address.address) if (cachedKeys.length === 0) { await CacheSet.addressCreation(address) @@ -313,10 +316,12 @@ export const initPaymentCache = async (address: Address): Promise => { } export async function * getPaymentStream (userId: string): AsyncGenerator { - const uncachedAddressStream = getUserUncachedAddresses(userId) - for await (const address of uncachedAddressStream) { - console.log('[CACHE]: Creating cache for address', address.address) - await CacheSet.addressCreation(address) + if (process.env.SKIP_CACHE_REBUILD === undefined) { + const uncachedAddressStream = getUserUncachedAddresses(userId) + for await (const address of uncachedAddressStream) { + console.log('[CACHE]: Creating cache for address', address.address) + await CacheSet.addressCreation(address) + } } const userButtonIds: string[] = (await fetchPaybuttonArrayByUserId(userId)) .map(p => p.id) diff --git a/scripts/db-entrypoint.sh b/scripts/db-entrypoint.sh index 3355b8891..9d2138e88 100755 --- a/scripts/db-entrypoint.sh +++ b/scripts/db-entrypoint.sh @@ -10,11 +10,25 @@ sed_vars () { cd /home/mysql/raw_entrypoint || exit for file in *.sql; do - sed_vars "$file" + if [[ "$file" == *dump* ]]; then + echo "Linking dump file: $file" + ln -s /home/mysql/raw_entrypoint/"$file" /home/mysql/entrypoint/"$file" + else + echo "Copying file $file after seding vars" + sed_vars "$file" + fi done cd ../entrypoint/ || exit for file in *.sql; do - mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file" + if [[ "$file" == *dump* ]]; then + filesize=$(stat -Lc%s "$file" 2>/dev/null || stat -Lf%z "$file" 2>/dev/null) + echo "Importing $file ($(numfmt --to=iec $filesize)) ..." + pv "$file" | mariadb -u root -p"$MYSQL_ROOT_PASSWORD" + else + echo "Importing $file ..." + mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file" + fi + echo "Done importing $file" done diff --git a/scripts/db/Dockerfile b/scripts/db/Dockerfile index 2675f8ae6..b31899ec2 100644 --- a/scripts/db/Dockerfile +++ b/scripts/db/Dockerfile @@ -3,7 +3,7 @@ FROM mariadb:latest RUN mkhomedir_helper mysql RUN apt-get update || echo && \ - apt-get install -y gettext || echo + apt-get install -y gettext pv || echo RUN mkdir -p /home/mysql/raw_entrypoint && \ mkdir -p /home/mysql/entrypoint diff --git a/scripts/paybutton-server-start.sh b/scripts/paybutton-server-start.sh index 1028c14fe..64a909685 100755 --- a/scripts/paybutton-server-start.sh +++ b/scripts/paybutton-server-start.sh @@ -17,17 +17,19 @@ logtime=$(date +%Y-%m-%d@%H:%M) [ -e logs/jobs.log ] && mv logs/jobs.log logs/history/jobs_"$logtime".log [ -e logs/ws-server.log ] && mv logs/ws-server.log logs/history/ws-server_"$logtime".log -if [ "$ENVIRONMENT" = "production" ]; then +if [ "$FROM_DUMP" = "true" ] || [ "$ENVIRONMENT" = "production" ]; then yarn prisma migrate deploy || exit 1 yarn prisma generate || exit 1 - pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 - pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 - pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 else yarn prisma migrate dev || exit 1 yarn prisma db seed || exit 1 - pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 - pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 +fi + +pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1 +pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1 +if [ "$ENVIRONMENT" = "production" ]; then + pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1 +else pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- dev || exit 1 fi pm2 logs next