diff --git a/.gitignore b/.gitignore index ff92685b4..18b7acdfe 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,8 @@ tests_playwright/test-data/temp/ /python/mdvtools/llm/structured_schemas.py /scripts/python/mdvtools/llm/structured_schemas.py +# Deployment-generated files +.env.* +.docker-deploy.*.override.yml + .cursor/ diff --git a/README.md b/README.md index aac97dc3b..be658d690 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,59 @@ For front-end development, a vite dev-server can be started with `pnpm run dev` ### Alternative steps for running locally If you prefer to manage tools differently - for a lighter-weight environment, non VSCode-based editors, or integration into other workflows etc, this is also possible. Further documentation can be found on the [MDV documentation website](https://mdv.ndm.ox.ac.uk/) + +## Deployment Scripts (Local Docker) + +The repository includes three deployment entrypoints: + +- `deploy.sh` +- `deploy.bat` +- `deploy_gui.sh` + +All three support: + +- database backend selection (`sqlite` default, `postgres` optional) +- deployment mode (`new` or `replace`) +- deployment name (Compose project name) +- custom app host port +- automatic suggested host port (next free from `5055`) + +### Backend behavior + +- **sqlite (default)**: + - runs `mdv_app` without `mdv_db` + - uses `SQLITE_DB_PATH` (default `/app/mdv/mdv.sqlite3`) +- **postgres**: + - runs `mdv_app` + `mdv_db` + - postgres container is pinned to `mdv-db` for easier reuse detection across deployments + - uses `DB_USER`, `DB_PASSWORD`, `DB_HOST` + - uses a shared database name `mdv` by default + - sets `DB_SCHEMA` to the deployment name by default + - can also run in **reuse** mode (app only), where `mdv_db` is not started and `DB_HOST` points to an existing Postgres instance + - deploy scripts auto-detect running Postgres containers and default to `reuse` mode when found + +At startup, application logs will include the selected backend (for example `Database backend selected: sqlite (...)`). + +### Running multiple deployments side-by-side + +To keep multiple instances without replacing an existing one, use: + +- mode: `new` +- a unique deployment name (for example `mdv_a`, `mdv_b`) +- different host ports (for example `5055`, `5056`) + +Each deployment writes its own env file (`.env.`) and runs under its own Compose project name. + +### Local image vs Docker Hub image + +Deploy scripts now prefer a local `mdvadmin/mdv:stable` image if it already exists, and skip pulling `mdv_app` in that case. + +If no local image is available, scripts pull from Docker Hub as before. + +To test local code changes: + +```sh +docker compose -f docker-compose.yml build mdv_app +``` + +Then run your deploy script and choose backend/settings as needed. diff --git a/deploy.bat b/deploy.bat index dcf67eeaa..d0ccab826 100644 --- a/deploy.bat +++ b/deploy.bat @@ -1,4 +1,5 @@ @echo off +setlocal EnableDelayedExpansion echo Welcome to the MDV application deployment script! @@ -20,10 +21,375 @@ if errorlevel 1 ( exit /b 1 ) +echo. +set /p DEPLOYMENT_NAME=Deployment name [mdv]: +if "%DEPLOYMENT_NAME%"=="" set DEPLOYMENT_NAME=mdv +set HAS_EXISTING_DEPLOYMENT=0 +for /f %%i in ('docker ps -aq --filter "label=com.docker.compose.project=%DEPLOYMENT_NAME%"') do set HAS_EXISTING_DEPLOYMENT=1 +if "%HAS_EXISTING_DEPLOYMENT%"=="1" ( + set /p DEPLOY_ACTION=Deployment "%DEPLOYMENT_NAME%" exists. Action ^(redeploy/rename/exit^) [redeploy]: + if /I "!DEPLOY_ACTION!"=="" set DEPLOY_ACTION=redeploy + if /I "!DEPLOY_ACTION!"=="exit" exit /b 0 + if /I "!DEPLOY_ACTION!"=="rename" ( + set /p DEPLOYMENT_NAME=New deployment name: + if "!DEPLOYMENT_NAME!"=="" ( + echo Error: deployment name cannot be empty. + exit /b 1 + ) + set HAS_EXISTING_DEPLOYMENT=0 + for /f %%i in ('docker ps -aq --filter "label=com.docker.compose.project=!DEPLOYMENT_NAME!"') do set HAS_EXISTING_DEPLOYMENT=1 + if "!HAS_EXISTING_DEPLOYMENT!"=="1" ( + echo Error: deployment "!DEPLOYMENT_NAME!" already exists. Please choose a different name. + exit /b 1 + ) + set DEPLOY_MODE=new + ) else ( + set DEPLOY_MODE=replace + ) +) else ( + set DEPLOY_MODE=new + echo No existing deployment found for "%DEPLOYMENT_NAME%". Using new mode. +) +set ENV_FILE=.env.%DEPLOYMENT_NAME% + +set PREVIOUS_STORAGE_MODE= +set PREVIOUS_HOST_DATA_PATH= +if /I "%DEPLOY_MODE%"=="replace" ( + for /f "tokens=1,* delims=|" %%A in ('powershell -NoProfile -Command "$m = docker inspect '%DEPLOYMENT_NAME%-mdv_app-1' 2>$null | ConvertFrom-Json; if($m){ $mount = $m[0].Mounts | Where-Object {$_.Destination -eq '/app/mdv'} | Select-Object -First 1; if($mount){ if($mount.Type -eq 'bind'){ Write-Output ('host|' + $mount.Source) } elseif($mount.Type -eq 'volume'){ Write-Output 'reuse|' } } }"') do ( + set PREVIOUS_STORAGE_MODE=%%A + set PREVIOUS_HOST_DATA_PATH=%%B + ) +) +set DEFAULT_STORAGE_MODE=create +if /I "%DEPLOY_MODE%"=="replace" ( + if /I "!PREVIOUS_STORAGE_MODE!"=="host" ( + set DEFAULT_STORAGE_MODE=host + ) else if /I "!PREVIOUS_STORAGE_MODE!"=="reuse" ( + set DEFAULT_STORAGE_MODE=reuse + ) else ( + docker volume inspect %DEPLOYMENT_NAME%_mdv-data >nul 2>&1 + if not errorlevel 1 set DEFAULT_STORAGE_MODE=reuse + ) +) +set STORAGE_MODE= +set HOST_DATA_PATH= +set RESET_DATA_VOLUME=0 +set REMOVE_OLD_HOST_PATH=0 +set REMOVE_OLD_VOLUME=0 + +set APP_PORT_DEFAULT=5055 +for /f %%p in ('powershell -NoProfile -Command "$ports = docker ps --format '{{.Ports}}'; $p=5055; while($true){ if(-not($ports -match ('0.0.0.0:'+$p+'->') -or $ports -match ('\[\:\:\]:'+$p+'->'))){ break }; $p++ }; Write-Output $p"') do set APP_PORT_DEFAULT=%%p +if /I "%DEPLOY_MODE%"=="replace" ( + set EXISTING_APP_PORT= + for /f %%p in ('powershell -NoProfile -Command "$line = docker port '%DEPLOYMENT_NAME%-mdv_app-1' 5055/tcp 2>$null | Select-Object -First 1; if($line -match ':(\d+)$'){ Write-Output $matches[1] }"') do ( + set APP_PORT_DEFAULT=%%p + set EXISTING_APP_PORT=%%p + ) + if not "!EXISTING_APP_PORT!"=="" echo Detected existing app port !APP_PORT_DEFAULT! for "%DEPLOYMENT_NAME%". +) +set /p APP_PORT=Application host port [%APP_PORT_DEFAULT%]: +if "%APP_PORT%"=="" set APP_PORT=%APP_PORT_DEFAULT% + +set PREV_DB_BACKEND= +set PREV_DB_HOST= +set PREV_DB_SCHEMA= +set PREV_DB_USER= +set PREV_DB_PASSWORD= +set PREV_DB_NAME= +if exist "%ENV_FILE%" ( + for /f "usebackq tokens=1,* delims==" %%A in ("%ENV_FILE%") do ( + if /I "%%A"=="DB_BACKEND" set PREV_DB_BACKEND=%%B + if /I "%%A"=="DB_HOST" set PREV_DB_HOST=%%B + if /I "%%A"=="DB_SCHEMA" set PREV_DB_SCHEMA=%%B + if /I "%%A"=="DB_USER" set PREV_DB_USER=%%B + if /I "%%A"=="DB_PASSWORD" set PREV_DB_PASSWORD=%%B + if /I "%%A"=="DB_NAME" set PREV_DB_NAME=%%B + ) +) + +REM Configure .env with database backend selection (default sqlite) +echo. +set /p DB_BACKEND=Choose database backend ^(sqlite/postgres^) [sqlite]: +if "%DB_BACKEND%"=="" set DB_BACKEND=sqlite +if /I not "%DB_BACKEND%"=="sqlite" if /I not "%DB_BACKEND%"=="postgres" ( + echo Invalid backend "%DB_BACKEND%". Falling back to sqlite. + set DB_BACKEND=sqlite +) + +set DB_HOST=mdv-db +if "%SQLITE_DB_PATH%"=="" set SQLITE_DB_PATH=/app/mdv/mdv.sqlite3 +set POSTGRES_MODE=dedicated +set REUSE_DB_CONTAINER=mdv-db +set REUSE_DB_NETWORK= +set DB_SCHEMA=%DEPLOYMENT_NAME% +set SHARED_PG_ENV=.env.postgres-shared +set DB_NAME=mdv +set SHARED_DB_NETWORK=mdv-shared-db +set SHARED_DB_HOST_PATH=%CD%\mdv-data\postgres-shared + +if /I "%DB_BACKEND%"=="postgres" ( + set EXISTING_POSTGRES=0 + docker container inspect mdv-db >nul 2>&1 + if not errorlevel 1 set EXISTING_POSTGRES=1 + if "!EXISTING_POSTGRES!"=="1" ( + for /f %%i in ('docker inspect mdv-db --format "{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}"') do set REUSE_DB_NETWORK=%%i + ) + set POSTGRES_MODE=reuse + echo Using shared postgres deployment mode ^(reuse^). + if "!EXISTING_POSTGRES!"=="1" ( + echo Detected running shared Postgres container ^(mdv-db^). + ) + if /I "!POSTGRES_MODE!"=="reuse" ( + set DB_USER= + set DB_PASSWORD= + if exist "!SHARED_PG_ENV!" ( + for /f "usebackq tokens=1,* delims==" %%A in ("!SHARED_PG_ENV!") do ( + if /I "%%A"=="DB_USER" set DB_USER=%%B + if /I "%%A"=="DB_PASSWORD" set DB_PASSWORD=%%B + ) + ) + if "!DB_USER!"=="" if "!EXISTING_POSTGRES!"=="1" for /f %%i in ('powershell -NoProfile -Command "$envs=(docker inspect mdv-db --format '{{json .Config.Env}}' ^| ConvertFrom-Json); ($envs ^| ? {$_ -like 'POSTGRES_USER=*'} ^| %% {$_.Split('=')[1]})"') do set DB_USER=%%i + if "!DB_PASSWORD!"=="" if "!EXISTING_POSTGRES!"=="1" for /f %%i in ('powershell -NoProfile -Command "$envs=(docker inspect mdv-db --format '{{json .Config.Env}}' ^| ConvertFrom-Json); ($envs ^| ? {$_ -like 'POSTGRES_PASSWORD=*'} ^| %% {$_.Split('=')[1]})"') do set DB_PASSWORD=%%i + if "!DB_USER!"=="" set DB_USER=testuser + if "!DB_PASSWORD!"=="" set DB_PASSWORD=testpass + set /p DB_NAME=Enter DB_NAME [!DB_NAME!]: + if "!DB_NAME!"=="" set DB_NAME=mdv + + if "!EXISTING_POSTGRES!"=="0" ( + set /p CREATE_SHARED_DB_INPUT=Shared postgres container "mdv-db" not found. Create it now? ^(y/n^) [y]: + if /I "!CREATE_SHARED_DB_INPUT!"=="n" ( + echo Cannot continue in reuse mode without shared postgres. + exit /b 1 + ) + set /p SHARED_DB_HOST_PATH=Shared Postgres host folder [!SHARED_DB_HOST_PATH!]: + if "!SHARED_DB_HOST_PATH!"=="" set SHARED_DB_HOST_PATH=%CD%\mdv-data\postgres-shared + docker network inspect !SHARED_DB_NETWORK! >nul 2>&1 + if errorlevel 1 docker network create !SHARED_DB_NETWORK! >nul 2>&1 + if not exist "!SHARED_DB_HOST_PATH!" mkdir "!SHARED_DB_HOST_PATH!" + docker run -d --name mdv-db --network !SHARED_DB_NETWORK! -e POSTGRES_USER=!DB_USER! -e POSTGRES_PASSWORD=!DB_PASSWORD! -e POSTGRES_DB=!DB_NAME! -v "!SHARED_DB_HOST_PATH!:/var/lib/postgresql/data" postgres:16 >nul 2>&1 + if errorlevel 1 ( + echo Failed to create shared postgres container mdv-db. + exit /b 1 + ) + set REUSE_DB_NETWORK=!SHARED_DB_NETWORK! + set EXISTING_POSTGRES=1 + ) else ( + for /f %%i in ('docker inspect mdv-db --format "{{.State.Running}}"') do set SHARED_DB_RUNNING=%%i + if /I not "!SHARED_DB_RUNNING!"=="true" ( + echo Shared postgres container mdv-db exists but is stopped. Starting it... + docker start mdv-db >nul 2>&1 + ) + ) + if "!REUSE_DB_NETWORK!"=="" for /f %%i in ('docker inspect mdv-db --format "{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}"') do set REUSE_DB_NETWORK=%%i + set DEFAULT_DB_HOST=mdv-db + set /p DB_HOST=Existing Postgres host [!DEFAULT_DB_HOST!]: + if "!DB_HOST!"=="" set DB_HOST=!DEFAULT_DB_HOST! + echo Reusing saved/existing Postgres credentials for convenience. + ) +) else ( + if "!DB_USER!"=="" set DB_USER=testuser + if "!DB_PASSWORD!"=="" set DB_PASSWORD=testpass + if "!DB_NAME!"=="" set DB_NAME=mdv + set DB_SCHEMA= +) + +if /I "%DB_BACKEND%"=="postgres" ( + set STORAGE_MODE=reuse + echo App data storage is not prompted for postgres backend ^(shared postgres deployment^). + if /I "!PREV_DB_BACKEND!"=="sqlite" ( + if /I "!PREVIOUS_STORAGE_MODE!"=="host" if /I not "!PREVIOUS_HOST_DATA_PATH!"=="" ( + set /p REMOVE_OLD_HOST_INPUT=Switching sqlite->postgres. Remove old host folder "!PREVIOUS_HOST_DATA_PATH!" and its data? ^(y/n^) [n]: + if /I "!REMOVE_OLD_HOST_INPUT!"=="y" set REMOVE_OLD_HOST_PATH=1 + ) + docker volume inspect %DEPLOYMENT_NAME%_mdv-data >nul 2>&1 + if not errorlevel 1 ( + set /p REMOVE_OLD_VOLUME_INPUT=Switching sqlite->postgres. Remove old app volume "%DEPLOYMENT_NAME%_mdv-data"? ^(y/n^) [n]: + if /I "!REMOVE_OLD_VOLUME_INPUT!"=="y" set REMOVE_OLD_VOLUME=1 + ) + ) +) else ( + if /I "%DEPLOY_MODE%"=="replace" ( + if /I "!DEFAULT_STORAGE_MODE!"=="host" ( + echo Detected existing host-mapped data folder. Defaulting storage mode to host. + ) else if /I "!DEFAULT_STORAGE_MODE!"=="reuse" ( + echo Detected existing volume-based data mount. Defaulting storage mode to reuse. + ) + ) + set /p STORAGE_MODE=Data storage mode ^(create/reuse/host^) [%DEFAULT_STORAGE_MODE%]: + if "!STORAGE_MODE!"=="" set STORAGE_MODE=!DEFAULT_STORAGE_MODE! + if /I not "!STORAGE_MODE!"=="create" if /I not "!STORAGE_MODE!"=="reuse" if /I not "!STORAGE_MODE!"=="host" ( + echo Invalid storage mode "!STORAGE_MODE!". Falling back to !DEFAULT_STORAGE_MODE!. + set STORAGE_MODE=!DEFAULT_STORAGE_MODE! + ) + if /I "!STORAGE_MODE!"=="host" ( + set HOST_PATH_DEFAULT=.\mdv-data\%DEPLOYMENT_NAME% + if /I "!PREVIOUS_STORAGE_MODE!"=="host" if not "!PREVIOUS_HOST_DATA_PATH!"=="" set HOST_PATH_DEFAULT=!PREVIOUS_HOST_DATA_PATH! + set /p HOST_DATA_PATH=Host folder for /app/mdv [!HOST_PATH_DEFAULT!]: + if "!HOST_DATA_PATH!"=="" set HOST_DATA_PATH=!HOST_PATH_DEFAULT! + if /I "!PREVIOUS_STORAGE_MODE!"=="reuse" ( + set /p REMOVE_OLD_VOLUME_INPUT=Switching from named volume to host folder. Remove old volume "%DEPLOYMENT_NAME%_mdv-data"? ^(y/n^) [n]: + if /I "!REMOVE_OLD_VOLUME_INPUT!"=="y" set REMOVE_OLD_VOLUME=1 + ) + ) else if /I "!STORAGE_MODE!"=="create" ( + if /I "!PREVIOUS_STORAGE_MODE!"=="host" if /I not "!PREVIOUS_HOST_DATA_PATH!"=="" ( + set /p REMOVE_OLD_HOST_INPUT=Switching from host folder to named volume. Remove old host folder "!PREVIOUS_HOST_DATA_PATH!" and its data? ^(y/n^) [n]: + if /I "!REMOVE_OLD_HOST_INPUT!"=="y" set REMOVE_OLD_HOST_PATH=1 + ) + ) + if /I "%DEPLOY_MODE%"=="replace" if /I "!STORAGE_MODE!"=="create" ( + docker volume inspect %DEPLOYMENT_NAME%_mdv-data >nul 2>&1 + if not errorlevel 1 ( + set /p REMOVE_DATA_INPUT=Existing data volume "%DEPLOYMENT_NAME%_mdv-data" found. Remove existing data before redeploy? ^(y/n^) [n]: + if /I "!REMOVE_DATA_INPUT!"=="y" ( + set RESET_DATA_VOLUME=1 + ) else ( + set STORAGE_MODE=reuse + echo Keeping existing data volume ^(reusing existing data^). + ) + ) + ) +) + +if /I "%STORAGE_MODE%"=="host" ( + if /I "!PREVIOUS_STORAGE_MODE!"=="host" if /I not "!PREVIOUS_HOST_DATA_PATH!"=="" if /I not "!PREVIOUS_HOST_DATA_PATH!"=="!HOST_DATA_PATH!" ( + set /p REMOVE_OLD_HOST_INPUT=Previous host folder "!PREVIOUS_HOST_DATA_PATH!" differs. Remove old folder and its data? ^(y/n^) [n]: + if /I "!REMOVE_OLD_HOST_INPUT!"=="y" set REMOVE_OLD_HOST_PATH=1 + ) + if exist "!HOST_DATA_PATH!\." ( + set HOST_PATH_NOT_EMPTY=0 + for /f %%i in ('dir /b "!HOST_DATA_PATH!" 2^>nul') do set HOST_PATH_NOT_EMPTY=1 + if "!HOST_PATH_NOT_EMPTY!"=="1" ( + set /p CLEAR_HOST_INPUT=Host folder "!HOST_DATA_PATH!" is not empty. Remove existing data before deployment? ^(y/n^) [n]: + if /I "!CLEAR_HOST_INPUT!"=="y" ( + del /f /q "!HOST_DATA_PATH!\*" >nul 2>&1 + for /d %%D in ("!HOST_DATA_PATH!\*") do rd /s /q "%%D" + ) else ( + echo Keeping existing host folder data. + ) + ) + ) else if exist "!HOST_DATA_PATH!" ( + echo Error: "!HOST_DATA_PATH!" exists but is not a folder. + exit /b 1 + ) else ( + mkdir "!HOST_DATA_PATH!" + ) +) + +set POSTGRES_USER=!DB_USER! +set POSTGRES_PASSWORD=!DB_PASSWORD! +set POSTGRES_DB=!DB_NAME! + +if /I "!DB_BACKEND!"=="postgres" ( + ( + echo DB_USER=!DB_USER! + echo DB_PASSWORD=!DB_PASSWORD! + ) > "!SHARED_PG_ENV!" +) + +echo. +set ENABLE_AUTH=0 +set DEFAULT_AUTH_METHOD=dev +set DEFAULT_AUTH_METHOD_ENV=dummy +set FLASK_SECRET_KEY= +set LOGIN_REDIRECT_URL= +set AUTH0_DOMAIN= +set AUTH0_CLIENT_ID= +set AUTH0_CLIENT_SECRET= +set AUTH0_CALLBACK_URL= +set AUTH0_AUDIENCE= +set AUTH0_DB_CONNECTION= +set AUTH0_PUBLIC_KEY_URI= +set SHIBBOLETH_LOGIN_URL= +set SHIBBOLETH_LOGOUT_URL= + +set /p ENABLE_AUTH_INPUT=Enable Authentication? ^(y/n^) [n]: +if /I "!ENABLE_AUTH_INPUT!"=="y" ( + set ENABLE_AUTH=1 + set /p DEFAULT_AUTH_METHOD=Authentication method ^(dev/auth0/shibboleth^) [dev]: + if "!DEFAULT_AUTH_METHOD!"=="" set DEFAULT_AUTH_METHOD=dev + if /I "!DEFAULT_AUTH_METHOD!"=="dummy" set DEFAULT_AUTH_METHOD=dev + if /I not "!DEFAULT_AUTH_METHOD!"=="dev" if /I not "!DEFAULT_AUTH_METHOD!"=="auth0" if /I not "!DEFAULT_AUTH_METHOD!"=="shibboleth" ( + echo Invalid auth method "!DEFAULT_AUTH_METHOD!". Falling back to dev. + set DEFAULT_AUTH_METHOD=dev + ) + + if /I "!DEFAULT_AUTH_METHOD!"=="dev" ( + set FLASK_SECRET_KEY=change-me-in-production + set /p LOGIN_REDIRECT_URL=Enter LOGIN_REDIRECT_URL [/login_dev]: + if "!LOGIN_REDIRECT_URL!"=="" set LOGIN_REDIRECT_URL=/login_dev + ) else if /I "!DEFAULT_AUTH_METHOD!"=="auth0" ( + set /p FLASK_SECRET_KEY=Enter FLASK_SECRET_KEY [change-me-in-production]: + if "!FLASK_SECRET_KEY!"=="" set FLASK_SECRET_KEY=change-me-in-production + set /p LOGIN_REDIRECT_URL=Enter LOGIN_REDIRECT_URL [/login]: + if "!LOGIN_REDIRECT_URL!"=="" set LOGIN_REDIRECT_URL=/login + + set /p AUTH0_DOMAIN=Enter AUTH0_DOMAIN: + set /p AUTH0_CLIENT_ID=Enter AUTH0_CLIENT_ID: + set /p AUTH0_CLIENT_SECRET=Enter AUTH0_CLIENT_SECRET: + set /p AUTH0_CALLBACK_URL=Enter AUTH0_CALLBACK_URL: + set /p AUTH0_AUDIENCE=Enter AUTH0_AUDIENCE: + set /p AUTH0_DB_CONNECTION=Enter AUTH0_DB_CONNECTION: + set /p AUTH0_PUBLIC_KEY_URI=Enter AUTH0_PUBLIC_KEY_URI: + ) else ( + set /p FLASK_SECRET_KEY=Enter FLASK_SECRET_KEY [change-me-in-production]: + if "!FLASK_SECRET_KEY!"=="" set FLASK_SECRET_KEY=change-me-in-production + set /p LOGIN_REDIRECT_URL=Enter LOGIN_REDIRECT_URL [/login_sso]: + if "!LOGIN_REDIRECT_URL!"=="" set LOGIN_REDIRECT_URL=/login_sso + + set /p SHIBBOLETH_LOGIN_URL=Enter SHIBBOLETH_LOGIN_URL: + set /p SHIBBOLETH_LOGOUT_URL=Enter SHIBBOLETH_LOGOUT_URL: + ) +) + +set DEFAULT_AUTH_METHOD_ENV=!DEFAULT_AUTH_METHOD! +if /I "!DEFAULT_AUTH_METHOD_ENV!"=="dev" set DEFAULT_AUTH_METHOD_ENV=dummy + +echo Writing .env file... +( + echo # Flask Configuration + echo FLASK_ENV=production + echo PYTHONUNBUFFERED=1 + echo. + echo # Database Configuration + echo DB_BACKEND=!DB_BACKEND! + echo POSTGRES_MODE=!POSTGRES_MODE! + echo SQLITE_DB_PATH=!SQLITE_DB_PATH! + echo DB_USER=!DB_USER! + echo DB_PASSWORD=!DB_PASSWORD! + echo DB_NAME=!DB_NAME! + echo DB_HOST=!DB_HOST! + echo DB_SCHEMA=!DB_SCHEMA! + echo. + echo POSTGRES_USER=!POSTGRES_USER! + echo POSTGRES_PASSWORD=!POSTGRES_PASSWORD! + echo POSTGRES_DB=!POSTGRES_DB! + echo. + echo ENABLE_AUTH=!ENABLE_AUTH! + echo DEFAULT_AUTH_METHOD=!DEFAULT_AUTH_METHOD_ENV! + if "!ENABLE_AUTH!"=="1" ( + echo FLASK_SECRET_KEY=!FLASK_SECRET_KEY! + echo LOGIN_REDIRECT_URL=!LOGIN_REDIRECT_URL! + if /I "!DEFAULT_AUTH_METHOD!"=="auth0" ( + echo AUTH0_DOMAIN=!AUTH0_DOMAIN! + echo AUTH0_CLIENT_ID=!AUTH0_CLIENT_ID! + echo AUTH0_CLIENT_SECRET=!AUTH0_CLIENT_SECRET! + echo AUTH0_CALLBACK_URL=!AUTH0_CALLBACK_URL! + echo AUTH0_AUDIENCE=!AUTH0_AUDIENCE! + echo AUTH0_DB_CONNECTION=!AUTH0_DB_CONNECTION! + echo AUTH0_PUBLIC_KEY_URI=!AUTH0_PUBLIC_KEY_URI! + ) + if /I "!DEFAULT_AUTH_METHOD!"=="shibboleth" ( + echo SHIBBOLETH_LOGIN_URL=!SHIBBOLETH_LOGIN_URL! + echo SHIBBOLETH_LOGOUT_URL=!SHIBBOLETH_LOGOUT_URL! + ) + ) + echo ENABLE_CHAT=0 +) > %ENV_FILE% + REM Function to download and run docker-compose :run_docker_compose echo Setting DOCKER_COMPOSE_URL environment variable... -set DOCKER_COMPOSE_URL=https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-local.yml +set DOCKER_COMPOSE_URL=https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-compose.yml echo Downloading docker-compose.yml from %DOCKER_COMPOSE_URL%... curl -fsSL -o docker-compose.yml %DOCKER_COMPOSE_URL% if errorlevel 1 ( @@ -31,24 +397,154 @@ if errorlevel 1 ( exit /b 1 ) -echo Pulling stable Docker images... -docker compose -f docker-compose.yml pull +powershell -NoProfile -Command "(Get-Content docker-compose.yml) -replace '\"\d+:5055\"','\"%APP_PORT%:5055\"' | Set-Content docker-compose.yml" if errorlevel 1 ( - echo Error: Failed to pull Docker images. Check the configuration and try again. + echo Error: Failed to set app port in docker-compose.yml. exit /b 1 ) -echo Running docker-compose with docker-compose.yml... -docker compose -f docker-compose.yml up -d +set "MDV_MOUNT=mdv-data:/app/mdv" +if /I "%STORAGE_MODE%"=="host" set "MDV_MOUNT=%HOST_DATA_PATH%:/app/mdv" +powershell -NoProfile -Command "$mount = ($env:MDV_MOUNT -replace '\\','/'); $content = Get-Content docker-compose.yml; $content = $content -replace '^\s*-\s*.+:/app/mdv\s*$', (' - ''{0}''' -f $mount); Set-Content docker-compose.yml $content" if errorlevel 1 ( - echo Error: Failed to run docker-compose. Check the configuration and try again. + echo Error: Failed to set data mount in docker-compose.yml. exit /b 1 ) +echo Creating deployment override for port %APP_PORT%... +> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo services: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo mdv_app: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo env_file: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo - "%ENV_FILE%" +if /I "%POSTGRES_MODE%"=="reuse" if not "!REUSE_DB_NETWORK!"=="" ( +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo networks: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo - reused_db_net +) +if /I "%DB_BACKEND%"=="postgres" if /I not "%POSTGRES_MODE%"=="reuse" ( +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo mdv_db: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo container_name: mdv-db +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo env_file: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo - "%ENV_FILE%" +) +if /I "%POSTGRES_MODE%"=="reuse" if not "!REUSE_DB_NETWORK!"=="" ( +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo networks: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo reused_db_net: +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo external: true +>> .docker-deploy.%DEPLOYMENT_NAME%.override.yml echo name: !REUSE_DB_NETWORK! +) + +set USE_LOCAL_MDV_IMAGE=0 +docker image inspect mdvadmin/mdv:stable >nul 2>&1 +if not errorlevel 1 ( + set USE_LOCAL_MDV_IMAGE=1 + echo Detected local image mdvadmin/mdv:stable. Skipping mdv_app pull. +) + +if /I "%DEPLOY_MODE%"=="replace" ( + echo Stopping existing deployment "%DEPLOYMENT_NAME%"... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml down --remove-orphans + if "!RESET_DATA_VOLUME!"=="1" ( + docker volume rm %DEPLOYMENT_NAME%_mdv-data >nul 2>&1 + ) + if "!REMOVE_OLD_VOLUME!"=="1" ( + docker volume rm %DEPLOYMENT_NAME%_mdv-data >nul 2>&1 + ) + if "!REMOVE_OLD_HOST_PATH!"=="1" if not "!PREVIOUS_HOST_DATA_PATH!"=="" ( + rd /s /q "!PREVIOUS_HOST_DATA_PATH!" >nul 2>&1 + ) +) + +if /I "%DB_BACKEND%"=="sqlite" ( + if not exist docker-sqlite.override.yml ( + echo Error: Missing docker-sqlite.override.yml in project root. + exit /b 1 + ) + + if "!USE_LOCAL_MDV_IMAGE!"=="0" ( + echo Pulling mdv_app image... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml -f docker-sqlite.override.yml pull mdv_app + if errorlevel 1 ( + echo Error: Failed to pull mdv_app image. + exit /b 1 + ) + ) + + echo Running docker-compose ^(sqlite backend^)... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml -f docker-sqlite.override.yml up -d --no-deps mdv_app + if errorlevel 1 ( + echo Error: Failed to run docker-compose for sqlite backend. + exit /b 1 + ) + + REM Ensure postgres service is not left running from previous deploys. + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml stop mdv_db >nul 2>&1 + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml rm -f mdv_db >nul 2>&1 + docker volume rm %DEPLOYMENT_NAME%_postgres-data >nul 2>&1 + if /I "!PREV_DB_BACKEND!"=="postgres" if /I "!PREV_DB_HOST!"=="mdv-db" if not "!PREV_DB_SCHEMA!"=="" ( + if "!PREV_DB_USER!"=="" set PREV_DB_USER=testuser + if "!PREV_DB_NAME!"=="" set PREV_DB_NAME=mdv + echo Cleaning previous postgres schema "!PREV_DB_SCHEMA!" from mdv-db... + docker exec mdv-db psql -U "!PREV_DB_USER!" -d "!PREV_DB_NAME!" -c "DROP SCHEMA IF EXISTS ""!PREV_DB_SCHEMA!"" CASCADE;" >nul 2>&1 + set NON_SYSTEM_SCHEMA_COUNT= + docker exec mdv-db psql -U "!PREV_DB_USER!" -d "!PREV_DB_NAME!" -t -A -c "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name NOT IN ('pg_catalog','information_schema','public') AND schema_name NOT LIKE 'pg_toast%%' AND schema_name NOT LIKE 'pg_temp_%%';" > "%TEMP%\mdv_schema_count.tmp" 2>nul + for /f %%s in (%TEMP%\mdv_schema_count.tmp) do set NON_SYSTEM_SCHEMA_COUNT=%%s + del /f /q "%TEMP%\mdv_schema_count.tmp" >nul 2>&1 + if "!NON_SYSTEM_SCHEMA_COUNT!"=="0" ( + echo No non-system schemas remain in mdv-db. Stopping and removing mdv-db container. + docker stop mdv-db >nul 2>&1 + docker rm mdv-db >nul 2>&1 + ) + ) +) else ( + if /I "%POSTGRES_MODE%"=="reuse" ( + if "!USE_LOCAL_MDV_IMAGE!"=="0" ( + echo Pulling mdv_app image... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml pull mdv_app + if errorlevel 1 ( + echo Error: Failed to pull mdv_app image. + exit /b 1 + ) + ) + + echo Running docker-compose ^(postgres reuse mode, app only^)... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml up -d --no-deps mdv_app + if errorlevel 1 ( + echo Error: Failed to run docker-compose for postgres reuse mode. + exit /b 1 + ) + + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml stop mdv_db >nul 2>&1 + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml rm -f mdv_db >nul 2>&1 + ) else ( + if "!USE_LOCAL_MDV_IMAGE!"=="1" ( + echo Pulling postgres image only ^(mdv_app is local^)... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml pull mdv_db + if errorlevel 1 ( + echo Error: Failed to pull postgres image. Check the configuration and try again. + exit /b 1 + ) + ) else ( + echo Pulling stable Docker images... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml pull + if errorlevel 1 ( + echo Error: Failed to pull Docker images. Check the configuration and try again. + exit /b 1 + ) + ) + + echo Running docker-compose with docker-compose.yml... + docker compose -p %DEPLOYMENT_NAME% --env-file %ENV_FILE% -f docker-compose.yml -f .docker-deploy.%DEPLOYMENT_NAME%.override.yml up -d + if errorlevel 1 ( + echo Error: Failed to run docker-compose. Check the configuration and try again. + exit /b 1 + ) + ) +) + echo MDV application deployment completed successfully! echo. -echo ****** Open your web browser and go to https://localhost:5055 to access the MDV application ****** +echo ****** Open your web browser and go to https://localhost:%APP_PORT% to access the MDV application ****** echo. exit /b 0 diff --git a/deploy.sh b/deploy.sh index 3d3a992bb..3ce05ecc9 100755 --- a/deploy.sh +++ b/deploy.sh @@ -11,6 +11,29 @@ GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color +DEPLOYMENT_NAME="mdv" +DEPLOY_MODE="replace" +APP_PORT="5055" +ENV_FILE=".env" +POSTGRES_MODE="dedicated" +REUSE_DB_CONTAINER="mdv-db" +REUSE_DB_NETWORK="" +DB_SCHEMA="" +DB_NAME="mdv" +SHARED_DB_NETWORK="mdv-shared-db" +SHARED_DB_HOST_PATH="./mdv-data/postgres-shared" +STORAGE_MODE="create" +HOST_DATA_PATH="" +RESET_DATA_VOLUME="0" +PREVIOUS_STORAGE_MODE="" +PREVIOUS_HOST_DATA_PATH="" +REMOVE_OLD_HOST_PATH="0" +REMOVE_OLD_VOLUME="0" +PREV_DB_BACKEND="" +PREV_DB_HOST="" +PREV_DB_SCHEMA="" +PREV_DB_USER="" +PREV_DB_NAME="" # Logo echo -e "${BLUE}" @@ -73,10 +96,103 @@ check_existing_service() { fi } +prompt_deployment_settings() { + get_next_available_port() { + local port=5055 + while docker ps --format "{{.Ports}}" | grep -E "(0.0.0.0|\\[::\\]):${port}->" >/dev/null; do + port=$((port + 1)) + done + echo "$port" + } + get_existing_deployment_port() { + local ports_line="" + ports_line=$(docker ps \ + --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" \ + --filter "label=com.docker.compose.service=mdv_app" \ + --format "{{.Ports}}" | head -n 1) + local port_pattern='(0\.0\.0\.0|\[::\]|::):([0-9]+)->5055' + if [[ "$ports_line" =~ $port_pattern ]]; then + echo "${BASH_REMATCH[2]}" + fi + } + get_existing_storage_mount() { + local app_container="${DEPLOYMENT_NAME}-mdv_app-1" + local mount_type="" + local mount_source="" + if docker container inspect "$app_container" >/dev/null 2>&1; then + mount_type=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/app/mdv"}}{{.Type}}{{end}}{{end}}' "$app_container") + mount_source=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/app/mdv"}}{{.Source}}{{end}}{{end}}' "$app_container") + if [[ -n "$mount_type" ]]; then + echo "${mount_type}|${mount_source}" + fi + fi + } + + read -p "Deployment name [mdv]: " deployment_name_input + DEPLOYMENT_NAME=${deployment_name_input:-mdv} + + if docker ps -aq --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" | grep -q .; then + read -p "Deployment '${DEPLOYMENT_NAME}' exists. Action (redeploy/rename/exit) [redeploy]: " deploy_action_input + DEPLOY_ACTION=$(echo "${deploy_action_input:-redeploy}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$DEPLOY_ACTION" == "exit" ]]; then + echo "Exiting deployment." + exit 0 + elif [[ "$DEPLOY_ACTION" == "rename" ]]; then + read -p "New deployment name: " new_deployment_name + if [[ -z "$new_deployment_name" ]]; then + echo -e "${RED}Error: deployment name cannot be empty.${NC}" + exit 1 + fi + DEPLOYMENT_NAME="$new_deployment_name" + if docker ps -aq --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" | grep -q .; then + echo -e "${RED}Error: deployment '${DEPLOYMENT_NAME}' already exists. Choose a different name.${NC}" + exit 1 + fi + DEPLOY_MODE="new" + else + DEPLOY_MODE="replace" + fi + else + DEPLOY_MODE="new" + echo -e "${YELLOW}No existing deployment found for '${DEPLOYMENT_NAME}'. Using new mode.${NC}" + fi + + DEFAULT_STORAGE_MODE="create" + PREVIOUS_STORAGE_MOUNT=$(get_existing_storage_mount) + if [[ -n "$PREVIOUS_STORAGE_MOUNT" ]]; then + PREVIOUS_STORAGE_TYPE=${PREVIOUS_STORAGE_MOUNT%%|*} + PREVIOUS_STORAGE_SOURCE=${PREVIOUS_STORAGE_MOUNT#*|} + if [[ "$PREVIOUS_STORAGE_TYPE" == "bind" ]]; then + PREVIOUS_STORAGE_MODE="host" + PREVIOUS_HOST_DATA_PATH="$PREVIOUS_STORAGE_SOURCE" + DEFAULT_STORAGE_MODE="host" + elif [[ "$PREVIOUS_STORAGE_TYPE" == "volume" ]]; then + PREVIOUS_STORAGE_MODE="reuse" + DEFAULT_STORAGE_MODE="reuse" + fi + elif [[ "$DEPLOY_MODE" == "replace" ]] && docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + DEFAULT_STORAGE_MODE="reuse" + PREVIOUS_STORAGE_MODE="reuse" + fi + + APP_PORT_DEFAULT=$(get_next_available_port) + if [[ "$DEPLOY_MODE" == "replace" ]]; then + EXISTING_APP_PORT=$(get_existing_deployment_port) + if [[ -n "$EXISTING_APP_PORT" ]]; then + APP_PORT_DEFAULT="$EXISTING_APP_PORT" + echo -e "${YELLOW}Detected existing app port ${APP_PORT_DEFAULT} for '${DEPLOYMENT_NAME}'.${NC}" + fi + fi + read -p "Application host port [${APP_PORT_DEFAULT}]: " app_port_input + APP_PORT=${app_port_input:-$APP_PORT_DEFAULT} + + ENV_FILE=".env.${DEPLOYMENT_NAME}" +} + # Function to create or validate the .env file create_or_validate_env_file() { - env_file=".env" + env_file="$ENV_FILE" if [ ! -f "$env_file" ]; then echo "$env_file does not exist. Creating it now..." @@ -84,7 +200,19 @@ create_or_validate_env_file() { fi # Source existing .env file to retain values - [ -f "$env_file" ] && source "$env_file" + if [ -f "$env_file" ]; then + # Windows CRLF line endings break `source` (": command not found") + if grep -q $'\r' "$env_file" 2>/dev/null; then + sed -i.bak $'s/\r$//' "$env_file" + fi + # shellcheck disable=SC1090 + source "$env_file" + fi + PREV_DB_BACKEND="${DB_BACKEND:-}" + PREV_DB_HOST="${DB_HOST:-}" + PREV_DB_SCHEMA="${DB_SCHEMA:-}" + PREV_DB_USER="${DB_USER:-}" + PREV_DB_NAME="${DB_NAME:-}" # Function to prompt user for input prompt_variable() { @@ -110,28 +238,179 @@ create_or_validate_env_file() { echo "$var_name=$new_value" # Return the value assignment (not the value alone) } - # Check if Postgres volume exists - DB_VOLUME_EXISTS=$(docker volume ls -q | grep -E ".*_postgres-data" | wc -l) + ensure_shared_postgres() { + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + is_running=$(docker inspect -f '{{.State.Running}}' "$REUSE_DB_CONTAINER" 2>/dev/null || echo "false") + if [[ "$is_running" != "true" ]]; then + echo -e "${YELLOW}Shared postgres container '$REUSE_DB_CONTAINER' exists but is stopped. Starting it...${NC}" + docker start "$REUSE_DB_CONTAINER" >/dev/null + fi + REUSE_DB_NETWORK=$(docker inspect "$REUSE_DB_CONTAINER" --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') + return 0 + fi - if [ "$DB_VOLUME_EXISTS" -gt 0 ]; then - echo -e "${YELLOW}⚠ Existing Postgres data volume detected." - echo -e "DB_USER, DB_PASSWORD, and DB_NAME will NOT be changed to keep your database data safe.${NC}" + read -p "Shared postgres container '$REUSE_DB_CONTAINER' not found. Create it now? (Y/n): " create_shared_input + create_shared_choice=$(echo "${create_shared_input:-y}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$create_shared_choice" == "n" || "$create_shared_choice" == "no" ]]; then + echo -e "${RED}Cannot continue in reuse mode without a shared postgres container.${NC}" + exit 1 + fi - # Preserve previously sourced values or use defaults if missing - DB_USER="${DB_USER:-testuser}" - DB_PASSWORD="${DB_PASSWORD:-testpass}" - DB_NAME="${DB_NAME:-testdb}" + read -p "Shared Postgres host folder [${SHARED_DB_HOST_PATH}]: " shared_db_path_input + SHARED_DB_HOST_PATH=${shared_db_path_input:-$SHARED_DB_HOST_PATH} + docker network inspect "$SHARED_DB_NETWORK" >/dev/null 2>&1 || docker network create "$SHARED_DB_NETWORK" >/dev/null + mkdir -p "$SHARED_DB_HOST_PATH" + if ! docker run -d \ + --name "$REUSE_DB_CONTAINER" \ + --network "$SHARED_DB_NETWORK" \ + -e POSTGRES_USER="$DB_USER" \ + -e POSTGRES_PASSWORD="$DB_PASSWORD" \ + -e POSTGRES_DB="$DB_NAME" \ + -v "$SHARED_DB_HOST_PATH:/var/lib/postgresql/data" \ + postgres:16 >/dev/null; then + echo -e "${RED}Failed to create shared postgres container '$REUSE_DB_CONTAINER'.${NC}" + exit 1 + fi + + for _i in $(seq 1 30); do + if docker exec "$REUSE_DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + REUSE_DB_NETWORK="$SHARED_DB_NETWORK" + DB_HOST="$REUSE_DB_CONTAINER" + return 0 + fi + sleep 1 + done + echo -e "${RED}Shared postgres container '$REUSE_DB_CONTAINER' did not become ready in time.${NC}" + exit 1 + } + + # Select DB backend (default sqlite) + echo + read -p "Choose database backend (sqlite/postgres) [sqlite]: " db_backend_choice + DB_BACKEND=${db_backend_choice:-${DB_BACKEND:-sqlite}} + DB_BACKEND=$(echo "$DB_BACKEND" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$DB_BACKEND" != "sqlite" && "$DB_BACKEND" != "postgres" ]]; then + echo -e "${YELLOW}Unknown backend '$DB_BACKEND'. Falling back to sqlite.${NC}" + DB_BACKEND="sqlite" + fi + + DB_HOST="mdv-db" + SQLITE_DB_PATH="${SQLITE_DB_PATH:-/app/mdv/mdv.sqlite3}" + DB_SCHEMA="$DEPLOYMENT_NAME" + + if [[ "$DB_BACKEND" == "postgres" ]]; then + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + REUSE_DB_NETWORK=$(docker inspect "$REUSE_DB_CONTAINER" --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') + fi + EXISTING_POSTGRES_COUNT=0 + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + EXISTING_POSTGRES_COUNT=1 + fi + POSTGRES_MODE="reuse" + echo -e "${YELLOW}Using shared postgres deployment mode (reuse).${NC}" + DB_USER=$(prompt_variable "DB_USER" false "testuser" | cut -d'=' -f2 | tr -d '\n' | xargs) + DB_PASSWORD=$(prompt_variable "DB_PASSWORD" true "testpass" | cut -d'=' -f2 | tr -d '\n' | xargs) + DB_NAME=$(prompt_variable "DB_NAME" false "$DB_NAME" | cut -d'=' -f2 | tr -d '\n' | xargs) + DB_HOST="$REUSE_DB_CONTAINER" + ensure_shared_postgres + DB_HOST=$(prompt_variable "DB_HOST" false "${DB_HOST:-$REUSE_DB_CONTAINER}" | cut -d'=' -f2 | tr -d '\n' | xargs) else - echo -e "${YELLOW}Enter DB credentials for new database setup:${NC}" - echo -e "${YELLOW}If you want to keep DB data in future redeploys, keep these values the same.${NC}" + # Keep defaults present in .env for compatibility if user switches backend later. + DB_USER="${DB_USER:-testuser}" + DB_PASSWORD="${DB_PASSWORD:-testpass}" + DB_NAME="${DB_NAME:-mdv}" + DB_SCHEMA="" + fi - # Capture the output of prompt_variable and extract the values - DB_USER=$(prompt_variable "DB_USER" false "testuser"| cut -d'=' -f2 | tr -d '\n' | xargs) - DB_PASSWORD=$(prompt_variable "DB_PASSWORD" true "testpass"| cut -d'=' -f2 | tr -d '\n' | xargs) - DB_NAME=$(prompt_variable "DB_NAME" false "testdb"| cut -d'=' -f2 | tr -d '\n' | xargs) + if [[ "$DB_BACKEND" == "postgres" ]]; then + STORAGE_MODE="reuse" + echo -e "${YELLOW}App data storage prompt skipped for postgres backend.${NC}" + if [[ "${PREV_DB_BACKEND:-}" == "sqlite" ]]; then + if [[ "$PREVIOUS_STORAGE_MODE" == "host" && -n "$PREVIOUS_HOST_DATA_PATH" ]]; then + read -p "Switching sqlite->postgres. Remove old host folder '${PREVIOUS_HOST_DATA_PATH}' and its data? (y/N): " remove_old_host_input + remove_old_host_choice=$(echo "${remove_old_host_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_old_host_choice" == "y" || "$remove_old_host_choice" == "yes" ]]; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + read -p "Switching sqlite->postgres. Remove old app volume '${DEPLOYMENT_NAME}_mdv-data'? (y/N): " remove_old_volume_input + remove_old_volume_choice=$(echo "${remove_old_volume_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_old_volume_choice" == "y" || "$remove_old_volume_choice" == "yes" ]]; then + REMOVE_OLD_VOLUME="1" + fi + fi + fi + else + if [[ "$DEPLOY_MODE" == "replace" ]]; then + if [[ "$DEFAULT_STORAGE_MODE" == "host" ]]; then + echo -e "${YELLOW}Detected existing host-mapped data folder. Defaulting storage mode to host.${NC}" + elif [[ "$DEFAULT_STORAGE_MODE" == "reuse" ]]; then + echo -e "${YELLOW}Detected existing volume-based data mount. Defaulting storage mode to reuse.${NC}" + fi + fi + read -p "Data storage mode (create/reuse/host) [${DEFAULT_STORAGE_MODE}]: " storage_mode_input + STORAGE_MODE=$(echo "${storage_mode_input:-$DEFAULT_STORAGE_MODE}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$STORAGE_MODE" != "create" && "$STORAGE_MODE" != "reuse" && "$STORAGE_MODE" != "host" ]]; then + echo -e "${YELLOW}Unknown storage mode '$STORAGE_MODE'. Falling back to ${DEFAULT_STORAGE_MODE}.${NC}" + STORAGE_MODE="$DEFAULT_STORAGE_MODE" + fi + if [[ "$STORAGE_MODE" == "host" ]]; then + HOST_PATH_DEFAULT="./mdv-data/${DEPLOYMENT_NAME}" + if [[ "$PREVIOUS_STORAGE_MODE" == "host" && -n "$PREVIOUS_HOST_DATA_PATH" ]]; then + HOST_PATH_DEFAULT="$PREVIOUS_HOST_DATA_PATH" + fi + read -p "Host folder for /app/mdv [${HOST_PATH_DEFAULT}]: " host_data_input + HOST_DATA_PATH=${host_data_input:-$HOST_PATH_DEFAULT} + if [[ "$PREVIOUS_STORAGE_MODE" == "reuse" ]]; then + read -p "Switching from named volume to host folder. Remove old volume '${DEPLOYMENT_NAME}_mdv-data'? (y/N): " remove_old_volume_input + remove_old_volume_choice=$(echo "${remove_old_volume_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_old_volume_choice" == "y" || "$remove_old_volume_choice" == "yes" ]]; then + REMOVE_OLD_VOLUME="1" + fi + fi + elif [[ "$STORAGE_MODE" == "create" && "$PREVIOUS_STORAGE_MODE" == "host" && -n "$PREVIOUS_HOST_DATA_PATH" ]]; then + read -p "Switching from host folder to named volume. Remove old host folder '${PREVIOUS_HOST_DATA_PATH}' and its data? (y/N): " remove_old_host_input + remove_old_host_choice=$(echo "${remove_old_host_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_old_host_choice" == "y" || "$remove_old_host_choice" == "yes" ]]; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if [[ "$DEPLOY_MODE" == "replace" && "$STORAGE_MODE" == "create" ]] && docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + read -p "Existing data volume '${DEPLOYMENT_NAME}_mdv-data' found. Remove existing data before redeploy? (y/N): " remove_data_input + remove_data_choice=$(echo "${remove_data_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_data_choice" == "y" || "$remove_data_choice" == "yes" ]]; then + RESET_DATA_VOLUME="1" + else + echo -e "${YELLOW}Keeping existing data volume (reusing existing data).${NC}" + STORAGE_MODE="reuse" + fi + fi + fi + + if [[ "$STORAGE_MODE" == "host" ]]; then + if [[ "$PREVIOUS_STORAGE_MODE" == "host" && -n "$PREVIOUS_HOST_DATA_PATH" && "$PREVIOUS_HOST_DATA_PATH" != "$HOST_DATA_PATH" ]]; then + read -p "Previous host folder '${PREVIOUS_HOST_DATA_PATH}' differs. Remove old folder and its data? (y/N): " remove_old_host_input + remove_old_host_choice=$(echo "${remove_old_host_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$remove_old_host_choice" == "y" || "$remove_old_host_choice" == "yes" ]]; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if [[ -e "$HOST_DATA_PATH" && ! -d "$HOST_DATA_PATH" ]]; then + echo -e "${RED}Error: '$HOST_DATA_PATH' exists but is not a folder.${NC}" + exit 1 + fi + if [[ -d "$HOST_DATA_PATH" ]] && [[ -n "$(ls -A "$HOST_DATA_PATH" 2>/dev/null)" ]]; then + read -p "Host folder '$HOST_DATA_PATH' is not empty. Remove existing data before deployment? (y/N): " clear_host_input + clear_host_choice=$(echo "${clear_host_input:-n}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$clear_host_choice" == "y" || "$clear_host_choice" == "yes" ]]; then + rm -rf "$HOST_DATA_PATH"/* "$HOST_DATA_PATH"/.[!.]* "$HOST_DATA_PATH"/..?* 2>/dev/null || true + else + echo -e "${YELLOW}Keeping existing host folder data.${NC}" + fi + fi fi - # Set PostgreSQL variables POSTGRES_USER=${DB_USER} POSTGRES_PASSWORD=${DB_PASSWORD} POSTGRES_DB=${DB_NAME} @@ -143,10 +422,14 @@ create_or_validate_env_file() { echo "PYTHONUNBUFFERED=1" echo "# Database Configuration" + echo "DB_BACKEND=$DB_BACKEND" + echo "POSTGRES_MODE=$POSTGRES_MODE" + echo "SQLITE_DB_PATH=$SQLITE_DB_PATH" echo "DB_USER=$DB_USER" echo "DB_PASSWORD=$DB_PASSWORD" echo "DB_NAME=$DB_NAME" - echo "DB_HOST=mdv_db" # Default without prompting + echo "DB_HOST=$DB_HOST" + echo "DB_SCHEMA=$DB_SCHEMA" echo "POSTGRES_USER=$POSTGRES_USER" echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" @@ -156,19 +439,47 @@ create_or_validate_env_file() { read -p "Enable Authentication? (y/n): " enable_auth if [[ "$enable_auth" == "y" ]]; then echo "ENABLE_AUTH=1" - echo "$(prompt_variable "FLASK_SECRET_KEY" true)" - echo "$(prompt_variable "LOGIN_REDIRECT_URL" false)" - echo "$(prompt_variable "AUTH0_DOMAIN" false)" - echo "$(prompt_variable "AUTH0_CLIENT_ID" false)" - echo "$(prompt_variable "AUTH0_CLIENT_SECRET" true)" - echo "$(prompt_variable "AUTH0_CALLBACK_URL" false)" - echo "$(prompt_variable "AUTH0_AUDIENCE" false)" - echo "$(prompt_variable "AUTH0_DB_CONNECTION" false)" - echo "$(prompt_variable "AUTH0_PUBLIC_KEY_URI" false)" - echo "$(prompt_variable "SHIBBOLETH_LOGIN_URL" false)" - echo "$(prompt_variable "SHIBBOLETH_LOGOUT_URL" false)" + AUTH_METHOD_DEFAULT=${DEFAULT_AUTH_METHOD:-dev} + read -p "Authentication method (dev/auth0/shibboleth) [${AUTH_METHOD_DEFAULT}]: " auth_method_input + DEFAULT_AUTH_METHOD=$(echo "${auth_method_input:-$AUTH_METHOD_DEFAULT}" | tr '[:upper:]' '[:lower:]' | xargs) + if [[ "$DEFAULT_AUTH_METHOD" == "dummy" ]]; then + DEFAULT_AUTH_METHOD="dev" + fi + if [[ "$DEFAULT_AUTH_METHOD" != "dev" && "$DEFAULT_AUTH_METHOD" != "auth0" && "$DEFAULT_AUTH_METHOD" != "shibboleth" ]]; then + echo -e "${YELLOW}Unknown auth method '$DEFAULT_AUTH_METHOD'. Falling back to dev.${NC}" + DEFAULT_AUTH_METHOD="dev" + fi + OUTPUT_AUTH_METHOD="$DEFAULT_AUTH_METHOD" + if [[ "$OUTPUT_AUTH_METHOD" == "dev" ]]; then + OUTPUT_AUTH_METHOD="dummy" + fi + echo "DEFAULT_AUTH_METHOD=$OUTPUT_AUTH_METHOD" + + if [[ "$DEFAULT_AUTH_METHOD" == "dev" ]]; then + DEV_SECRET_VALUE="${FLASK_SECRET_KEY:-change-me-in-production}" + echo "FLASK_SECRET_KEY=$DEV_SECRET_VALUE" + echo "$(prompt_variable "LOGIN_REDIRECT_URL" false "/login_dev")" + elif [[ "$DEFAULT_AUTH_METHOD" == "auth0" ]]; then + echo "$(prompt_variable "FLASK_SECRET_KEY" true "change-me-in-production")" + echo "$(prompt_variable "LOGIN_REDIRECT_URL" false "/login")" + echo "$(prompt_variable "AUTH0_DOMAIN" false)" + echo "$(prompt_variable "AUTH0_CLIENT_ID" false)" + echo "$(prompt_variable "AUTH0_CLIENT_SECRET" true)" + echo "$(prompt_variable "AUTH0_CALLBACK_URL" false)" + echo "$(prompt_variable "AUTH0_AUDIENCE" false)" + echo "$(prompt_variable "AUTH0_DB_CONNECTION" false)" + echo "$(prompt_variable "AUTH0_PUBLIC_KEY_URI" false)" + # Sync Auth0 users into the DB on app startup so the cache reflects them (no restart) + echo "RUN_AUTH0_SYNC_ON_START=1" + else + echo "$(prompt_variable "FLASK_SECRET_KEY" true "change-me-in-production")" + echo "$(prompt_variable "LOGIN_REDIRECT_URL" false "/login_sso")" + echo "$(prompt_variable "SHIBBOLETH_LOGIN_URL" false)" + echo "$(prompt_variable "SHIBBOLETH_LOGOUT_URL" false)" + fi else echo "ENABLE_AUTH=0" + echo "DEFAULT_AUTH_METHOD=dummy" fi # Chat read -p "Enable Chat? (y/n): " enable_chat @@ -187,6 +498,11 @@ create_or_validate_env_file() { run_docker_compose() { local compose_url=$1 local compose_file=$(basename "$compose_url") + local sqlite_override_file="docker-sqlite.override.yml" + local deploy_override_file=".docker-deploy.${DEPLOYMENT_NAME}.override.yml" + local use_local_mdv_image="0" + local mdv_mount="mdv-data:/app/mdv" + local escaped_mount="" if [ ! -f "$compose_file" ]; then echo "Downloading $compose_file..." @@ -196,21 +512,143 @@ run_docker_compose() { fi fi - echo "Pulling stable Docker images..." - docker compose -f "$compose_file" pull + if ! sed -E -i.bak "s/\"[0-9]+:5055\"/\"${APP_PORT}:5055\"/" "$compose_file"; then + echo -e "${RED} Error: Failed to set app port in $compose_file.${NC}" + exit 1 + fi - echo "Starting Docker Compose..." - docker compose -f "$compose_file" up -d || { - echo -e "${RED} Error: Failed to start Docker Compose." + if [[ "$STORAGE_MODE" == "host" ]]; then + mkdir -p "$HOST_DATA_PATH" + mdv_mount="${HOST_DATA_PATH}:/app/mdv" + fi + escaped_mount=${mdv_mount//\\/\\\\} + escaped_mount=${escaped_mount//&/\\&} + escaped_mount=${escaped_mount//|/\\|} + escaped_mount=${escaped_mount//\"/\\\"} + if ! sed -E -i.bak "s|^[[:space:]]*-[[:space:]]*.+:/app/mdv[[:space:]]*$| - \"${escaped_mount}\"|" "$compose_file"; then + echo -e "${RED} Error: Failed to set data mount in $compose_file.${NC}" exit 1 - } + fi + + cat > "$deploy_override_file" <> "$deploy_override_file" <> "$deploy_override_file" <> "$deploy_override_file" </dev/null 2>&1; then + use_local_mdv_image="1" + echo "Detected local image mdvadmin/mdv:stable. Skipping mdv_app pull." + fi + + if [[ "$DEPLOY_MODE" == "replace" ]]; then + echo "Stopping existing deployment '${DEPLOYMENT_NAME}'..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" down --remove-orphans || true + if [[ "$RESET_DATA_VOLUME" == "1" ]]; then + docker volume rm "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1 || true + fi + if [[ "$REMOVE_OLD_VOLUME" == "1" ]]; then + docker volume rm "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1 || true + fi + if [[ "$REMOVE_OLD_HOST_PATH" == "1" && -n "$PREVIOUS_HOST_DATA_PATH" ]]; then + rm -rf "$PREVIOUS_HOST_DATA_PATH" >/dev/null 2>&1 || true + fi + fi + + if [[ "${DB_BACKEND:-sqlite}" == "sqlite" ]]; then + if [ ! -f "$sqlite_override_file" ]; then + echo -e "${RED} Error: Missing $sqlite_override_file in project root.${NC}" + exit 1 + fi + if [[ "$use_local_mdv_image" != "1" ]]; then + echo "Pulling mdv_app image..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" -f "$sqlite_override_file" pull mdv_app + fi + + echo "Starting Docker Compose (sqlite backend)..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" -f "$sqlite_override_file" up -d --no-deps mdv_app || { + echo -e "${RED} Error: Failed to start Docker Compose for sqlite backend." + exit 1 + } + + # Ensure legacy postgres service is not left running from previous deploys. + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" stop mdv_db >/dev/null 2>&1 || true + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" rm -f mdv_db >/dev/null 2>&1 || true + docker volume rm "${DEPLOYMENT_NAME}_postgres-data" >/dev/null 2>&1 || true + if [[ "${PREV_DB_BACKEND:-}" == "postgres" && "${PREV_DB_HOST:-}" == "mdv-db" && -n "${PREV_DB_SCHEMA:-}" ]]; then + PREV_DB_USER_SAFE="${PREV_DB_USER:-testuser}" + PREV_DB_NAME_SAFE="${PREV_DB_NAME:-mdv}" + echo "Cleaning previous postgres schema '${PREV_DB_SCHEMA}' from mdv-db..." + docker exec mdv-db psql -U "$PREV_DB_USER_SAFE" -d "$PREV_DB_NAME_SAFE" -c "DROP SCHEMA IF EXISTS \"${PREV_DB_SCHEMA}\" CASCADE;" >/dev/null 2>&1 || true + NON_SYSTEM_SCHEMA_COUNT=$(docker exec mdv-db psql -U "$PREV_DB_USER_SAFE" -d "$PREV_DB_NAME_SAFE" -t -A -c "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name NOT IN ('pg_catalog','information_schema','public') AND schema_name NOT LIKE 'pg_toast%' AND schema_name NOT LIKE 'pg_temp_%';" 2>/dev/null | tr -d '[:space:]') + if [[ "${NON_SYSTEM_SCHEMA_COUNT:-1}" == "0" ]]; then + echo "No non-system schemas remain in mdv-db. Stopping and removing mdv-db container." + docker stop mdv-db >/dev/null 2>&1 || true + docker rm mdv-db >/dev/null 2>&1 || true + fi + fi + elif [[ "${POSTGRES_MODE:-dedicated}" == "reuse" ]]; then + if [[ "$use_local_mdv_image" != "1" ]]; then + echo "Pulling mdv_app image..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull mdv_app + fi + + echo "Starting Docker Compose (postgres reuse mode, app only)..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" up -d --no-deps mdv_app || { + echo -e "${RED} Error: Failed to start MDV app in postgres reuse mode.${NC}" + exit 1 + } + + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" stop mdv_db >/dev/null 2>&1 || true + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" rm -f mdv_db >/dev/null 2>&1 || true + else + if [[ "$use_local_mdv_image" == "1" ]]; then + echo "Pulling postgres image only (mdv_app is local)..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull mdv_db + else + echo "Pulling stable Docker images..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull + fi + + echo "Starting Docker Compose..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" up -d || { + echo -e "${RED} Error: Failed to start Docker Compose." + exit 1 + } + fi echo -e "${GREEN}✓ Docker Compose started successfully!" } # Function to open the application in the browser open_browser() { - URL="http://localhost:5055" + URL="http://localhost:${APP_PORT}" echo -e "${GREEN}✓ Application deployed successfully!" echo -e "${GREEN} Open $URL in your browser." } @@ -220,13 +658,14 @@ echo "Starting MDV Deployment..." check_docker #check_existing_service +prompt_deployment_settings # Create app_logs directory if it doesn't exist #mkdir -p app_logs && echo "Ensured app_logs directory exists." create_or_validate_env_file -DOCKER_COMPOSE_URL="https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-local.yml" +DOCKER_COMPOSE_URL="https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-compose.yml" run_docker_compose "$DOCKER_COMPOSE_URL" open_browser diff --git a/deploy_gui.sh b/deploy_gui.sh index b83146053..5c87a1c7c 100755 --- a/deploy_gui.sh +++ b/deploy_gui.sh @@ -1,4 +1,27 @@ #!/bin/bash +DEPLOYMENT_NAME="mdv" +DEPLOY_MODE="replace" +APP_PORT="5055" +ENV_FILE=".env" +POSTGRES_MODE="dedicated" +REUSE_DB_CONTAINER="mdv-db" +REUSE_DB_NETWORK="" +DB_SCHEMA="" +DB_NAME="mdv" +SHARED_DB_NETWORK="mdv-shared-db" +SHARED_DB_HOST_PATH="./mdv-data/postgres-shared" +STORAGE_MODE="create" +HOST_DATA_PATH="" +RESET_DATA_VOLUME="0" +PREVIOUS_STORAGE_MODE="" +PREVIOUS_HOST_DATA_PATH="" +REMOVE_OLD_HOST_PATH="0" +REMOVE_OLD_VOLUME="0" +PREV_DB_BACKEND="" +PREV_DB_HOST="" +PREV_DB_SCHEMA="" +PREV_DB_USER="" +PREV_DB_NAME="" # Function to check if Zenity is installed check_zenity_installed() { @@ -25,10 +48,107 @@ check_docker_daemon() { } # Function to create or validate the .env file using Zenity +prompt_deployment_settings() { + get_next_available_port() { + local port=5055 + while docker ps --format "{{.Ports}}" | grep -E "(0.0.0.0|\\[::\\]):${port}->" >/dev/null; do + port=$((port + 1)) + done + echo "$port" + } + get_existing_deployment_port() { + local ports_line="" + ports_line=$(docker ps \ + --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" \ + --filter "label=com.docker.compose.service=mdv_app" \ + --format "{{.Ports}}" | head -n 1) + local port_pattern='(0\.0\.0\.0|\[::\]|::):([0-9]+)->5055' + if [[ "$ports_line" =~ $port_pattern ]]; then + echo "${BASH_REMATCH[2]}" + fi + } + get_existing_storage_mount() { + local app_container="${DEPLOYMENT_NAME}-mdv_app-1" + local mount_type="" + local mount_source="" + if docker container inspect "$app_container" >/dev/null 2>&1; then + mount_type=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/app/mdv"}}{{.Type}}{{end}}{{end}}' "$app_container") + mount_source=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/app/mdv"}}{{.Source}}{{end}}{{end}}' "$app_container") + if [ -n "$mount_type" ]; then + echo "${mount_type}|${mount_source}" + fi + fi + } + + DEPLOYMENT_NAME=$(zenity --entry --title="Deployment Name" --text="Deployment name:" --entry-text="mdv") + if [ -z "$DEPLOYMENT_NAME" ]; then + DEPLOYMENT_NAME="mdv" + fi + + if docker ps -aq --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" | grep -q .; then + DEPLOY_ACTION=$(zenity --list --radiolist \ + --title="Existing Deployment Found" \ + --text="Deployment '${DEPLOYMENT_NAME}' already exists. What would you like to do?" \ + --column="Select" --column="Action" \ + TRUE "redeploy" FALSE "rename" FALSE "exit") + if [ -z "$DEPLOY_ACTION" ] || [ "$DEPLOY_ACTION" = "exit" ]; then + exit 0 + elif [ "$DEPLOY_ACTION" = "rename" ]; then + DEPLOYMENT_NAME=$(zenity --entry --title="New Deployment Name" --text="Enter a new deployment name:" --entry-text="${DEPLOYMENT_NAME}_new") + if [ -z "$DEPLOYMENT_NAME" ]; then + zenity --error --text="Deployment name cannot be empty." + exit 1 + fi + if docker ps -aq --filter "label=com.docker.compose.project=${DEPLOYMENT_NAME}" | grep -q .; then + zenity --error --text="Deployment '${DEPLOYMENT_NAME}' already exists. Choose a different name." + exit 1 + fi + DEPLOY_MODE="new" + else + DEPLOY_MODE="replace" + fi + else + DEPLOY_MODE="new" + zenity --info --text="No existing deployment found for '${DEPLOYMENT_NAME}'. Using new mode." + fi + + DEFAULT_STORAGE_MODE="create" + PREVIOUS_STORAGE_MOUNT=$(get_existing_storage_mount) + if [ -n "$PREVIOUS_STORAGE_MOUNT" ]; then + PREVIOUS_STORAGE_TYPE=${PREVIOUS_STORAGE_MOUNT%%|*} + PREVIOUS_STORAGE_SOURCE=${PREVIOUS_STORAGE_MOUNT#*|} + if [ "$PREVIOUS_STORAGE_TYPE" = "bind" ]; then + PREVIOUS_STORAGE_MODE="host" + PREVIOUS_HOST_DATA_PATH="$PREVIOUS_STORAGE_SOURCE" + DEFAULT_STORAGE_MODE="host" + elif [ "$PREVIOUS_STORAGE_TYPE" = "volume" ]; then + PREVIOUS_STORAGE_MODE="reuse" + DEFAULT_STORAGE_MODE="reuse" + fi + elif [ "$DEPLOY_MODE" = "replace" ] && docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + DEFAULT_STORAGE_MODE="reuse" + PREVIOUS_STORAGE_MODE="reuse" + fi + + APP_PORT_DEFAULT=$(get_next_available_port) + if [ "$DEPLOY_MODE" = "replace" ]; then + EXISTING_APP_PORT=$(get_existing_deployment_port) + if [ -n "$EXISTING_APP_PORT" ]; then + APP_PORT_DEFAULT="$EXISTING_APP_PORT" + zenity --info --text="Detected existing app port $APP_PORT_DEFAULT for '$DEPLOYMENT_NAME'." + fi + fi + APP_PORT=$(zenity --entry --title="Application Port" --text="Host port for MDV app:" --entry-text="$APP_PORT_DEFAULT") + if [ -z "$APP_PORT" ]; then + APP_PORT="$APP_PORT_DEFAULT" + fi + + ENV_FILE=".env.${DEPLOYMENT_NAME}" +} # Function to check and update environment variables create_or_validate_env_file() { - env_file=".env" + env_file="$ENV_FILE" if [ ! -f "$env_file" ]; then zenity --info --text="$env_file does not exist. Creating it now..." @@ -37,6 +157,11 @@ create_or_validate_env_file() { # Source the existing .env file to retain values source "$env_file" + PREV_DB_BACKEND="${DB_BACKEND:-}" + PREV_DB_HOST="${DB_HOST:-}" + PREV_DB_SCHEMA="${DB_SCHEMA:-}" + PREV_DB_USER="${DB_USER:-}" + PREV_DB_NAME="${DB_NAME:-}" # Function to ask the user if they want to overwrite an existing variable prompt_variable() { @@ -69,6 +194,167 @@ create_or_validate_env_file() { echo "$new_value" } + ensure_shared_postgres() { + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + is_running=$(docker inspect -f '{{.State.Running}}' "$REUSE_DB_CONTAINER" 2>/dev/null || echo "false") + if [ "$is_running" != "true" ]; then + zenity --info --text="Shared postgres container '$REUSE_DB_CONTAINER' exists but is stopped. Starting it..." + docker start "$REUSE_DB_CONTAINER" >/dev/null + fi + REUSE_DB_NETWORK=$(docker inspect "$REUSE_DB_CONTAINER" --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') + return 0 + fi + + if ! zenity --question --title="Create Shared Postgres" --text="Shared postgres container '$REUSE_DB_CONTAINER' not found.\nCreate it now?"; then + zenity --error --text="Cannot continue in reuse mode without a shared postgres container." + exit 1 + fi + + SHARED_DB_HOST_PATH=$(zenity --entry --title="Shared Postgres Data Folder" --text="Host folder for shared postgres data:" --entry-text="$SHARED_DB_HOST_PATH") + if [ -z "$SHARED_DB_HOST_PATH" ]; then + SHARED_DB_HOST_PATH="./mdv-data/postgres-shared" + fi + docker network inspect "$SHARED_DB_NETWORK" >/dev/null 2>&1 || docker network create "$SHARED_DB_NETWORK" >/dev/null + mkdir -p "$SHARED_DB_HOST_PATH" + if ! docker run -d \ + --name "$REUSE_DB_CONTAINER" \ + --network "$SHARED_DB_NETWORK" \ + -e POSTGRES_USER="$DB_USER" \ + -e POSTGRES_PASSWORD="$DB_PASSWORD" \ + -e POSTGRES_DB="$DB_NAME" \ + -v "$SHARED_DB_HOST_PATH:/var/lib/postgresql/data" \ + postgres:16 >/dev/null; then + zenity --error --text="Failed to create shared postgres container '$REUSE_DB_CONTAINER'." + exit 1 + fi + + for _i in $(seq 1 30); do + if docker exec "$REUSE_DB_CONTAINER" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + REUSE_DB_NETWORK="$SHARED_DB_NETWORK" + DB_HOST="$REUSE_DB_CONTAINER" + return 0 + fi + sleep 1 + done + zenity --error --text="Shared postgres container '$REUSE_DB_CONTAINER' did not become ready in time." + exit 1 + } + + + DB_BACKEND=$(zenity --list --radiolist \ + --title="Select Database Backend" \ + --text="Choose the database backend (default: sqlite)" \ + --column="Select" --column="Backend" \ + TRUE "sqlite" FALSE "postgres") + if [ -z "$DB_BACKEND" ]; then + DB_BACKEND="sqlite" + fi + + SQLITE_DB_PATH=$(prompt_variable "SQLITE_DB_PATH" false "/app/mdv/mdv.sqlite3") + + if [ "$DB_BACKEND" = "postgres" ]; then + DB_SCHEMA="$DEPLOYMENT_NAME" + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + REUSE_DB_NETWORK=$(docker inspect "$REUSE_DB_CONTAINER" --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') + fi + EXISTING_POSTGRES_COUNT=0 + if docker container inspect "$REUSE_DB_CONTAINER" >/dev/null 2>&1; then + EXISTING_POSTGRES_COUNT=1 + fi + POSTGRES_MODE="reuse" + zenity --info --text="Using shared postgres deployment mode (reuse)." + DB_USER=$(prompt_variable "DB_USER" false "testuser") + DB_PASSWORD=$(prompt_variable "DB_PASSWORD" true "testpass") + DB_NAME=$(prompt_variable "DB_NAME" false "$DB_NAME") + DB_HOST="$REUSE_DB_CONTAINER" + ensure_shared_postgres + DB_HOST=$(prompt_variable "DB_HOST" false "$DB_HOST") + else + DB_USER="${DB_USER:-testuser}" + DB_PASSWORD="${DB_PASSWORD:-testpass}" + DB_NAME="${DB_NAME:-mdv}" + DB_HOST="mdv-db" + DB_SCHEMA="" + fi + + if [ "$DB_BACKEND" = "postgres" ]; then + STORAGE_MODE="reuse" + zenity --info --text="App data storage prompt skipped for postgres backend." + if [ "${PREV_DB_BACKEND:-}" = "sqlite" ]; then + if [ "$PREVIOUS_STORAGE_MODE" = "host" ] && [ -n "$PREVIOUS_HOST_DATA_PATH" ]; then + if zenity --question --title="Remove Old Host Folder" --text="Switching sqlite->postgres.\nRemove old host folder '${PREVIOUS_HOST_DATA_PATH}' and its data?"; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + if zenity --question --title="Remove Old App Volume" --text="Switching sqlite->postgres.\nRemove old app volume '${DEPLOYMENT_NAME}_mdv-data'?"; then + REMOVE_OLD_VOLUME="1" + fi + fi + fi + else + if [ "$DEPLOY_MODE" = "replace" ]; then + if [ "$DEFAULT_STORAGE_MODE" = "host" ]; then + zenity --info --text="Detected existing host-mapped data folder. Defaulting storage mode to host." + elif [ "$DEFAULT_STORAGE_MODE" = "reuse" ]; then + zenity --info --text="Detected existing volume-based data mount. Defaulting storage mode to reuse." + fi + fi + if [ "$DEFAULT_STORAGE_MODE" = "reuse" ]; then + STORAGE_MODE=$(zenity --list --radiolist --title="Data Storage Mode" --text="How should /app/mdv data be mounted?" --column="Select" --column="Mode" FALSE "create" TRUE "reuse" FALSE "host") + else + STORAGE_MODE=$(zenity --list --radiolist --title="Data Storage Mode" --text="How should /app/mdv data be mounted?" --column="Select" --column="Mode" TRUE "create" FALSE "reuse" FALSE "host") + fi + if [ -z "$STORAGE_MODE" ]; then + STORAGE_MODE="$DEFAULT_STORAGE_MODE" + fi + if [ "$STORAGE_MODE" = "host" ]; then + HOST_PATH_DEFAULT="./mdv-data/${DEPLOYMENT_NAME}" + if [ "$PREVIOUS_STORAGE_MODE" = "host" ] && [ -n "$PREVIOUS_HOST_DATA_PATH" ]; then + HOST_PATH_DEFAULT="$PREVIOUS_HOST_DATA_PATH" + fi + HOST_DATA_PATH=$(zenity --entry --title="Host Data Folder" --text="Host folder to mount at /app/mdv" --entry-text="$HOST_PATH_DEFAULT") + if [ -z "$HOST_DATA_PATH" ]; then + HOST_DATA_PATH="$HOST_PATH_DEFAULT" + fi + if [ "$PREVIOUS_STORAGE_MODE" = "reuse" ]; then + if zenity --question --title="Remove Old Volume" --text="Switching from named volume to host folder.\nRemove old volume '${DEPLOYMENT_NAME}_mdv-data'?"; then + REMOVE_OLD_VOLUME="1" + fi + fi + elif [ "$STORAGE_MODE" = "create" ] && [ "$PREVIOUS_STORAGE_MODE" = "host" ] && [ -n "$PREVIOUS_HOST_DATA_PATH" ]; then + if zenity --question --title="Remove Old Host Folder" --text="Switching from host folder to named volume.\nRemove old host folder '${PREVIOUS_HOST_DATA_PATH}' and its data?"; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if [ "$DEPLOY_MODE" = "replace" ] && [ "$STORAGE_MODE" = "create" ] && docker volume inspect "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1; then + if zenity --question --title="Existing Data Volume" --text="Existing data volume '${DEPLOYMENT_NAME}_mdv-data' found.\nRemove existing data before redeploy?"; then + RESET_DATA_VOLUME="1" + else + STORAGE_MODE="reuse" + zenity --info --text="Keeping existing data volume (reusing existing data)." + fi + fi + fi + + if [ "$STORAGE_MODE" = "host" ]; then + if [ "$PREVIOUS_STORAGE_MODE" = "host" ] && [ -n "$PREVIOUS_HOST_DATA_PATH" ] && [ "$PREVIOUS_HOST_DATA_PATH" != "$HOST_DATA_PATH" ]; then + if zenity --question --title="Remove Old Host Folder" --text="Previous host folder differs:\n${PREVIOUS_HOST_DATA_PATH}\n\nRemove old folder and its data?"; then + REMOVE_OLD_HOST_PATH="1" + fi + fi + if [ -e "$HOST_DATA_PATH" ] && [ ! -d "$HOST_DATA_PATH" ]; then + zenity --error --text="Error: '$HOST_DATA_PATH' exists but is not a folder." + exit 1 + fi + if [ -d "$HOST_DATA_PATH" ] && [ -n "$(ls -A "$HOST_DATA_PATH" 2>/dev/null)" ]; then + if zenity --question --title="Host Folder Not Empty" --text="Host folder '$HOST_DATA_PATH' is not empty.\nRemove existing data before deployment?"; then + rm -rf "$HOST_DATA_PATH"/* "$HOST_DATA_PATH"/.[!.]* "$HOST_DATA_PATH"/..?* 2>/dev/null || true + else + zenity --info --text="Keeping existing host folder data." + fi + fi + fi # Write the new .env file { @@ -77,16 +363,16 @@ create_or_validate_env_file() { echo "PYTHONUNBUFFERED=1" echo "# Database Configuration" - DB_USER=$(prompt_variable "DB_USER" false) - DB_PASSWORD=$(prompt_variable "DB_PASSWORD" true) - DB_NAME=$(prompt_variable "DB_NAME" false) - DB_HOST=$(prompt_variable "DB_HOST" false "mdv_db") # Default value for DB_HOST + echo "DB_BACKEND=$DB_BACKEND" + echo "POSTGRES_MODE=$POSTGRES_MODE" + echo "SQLITE_DB_PATH=$SQLITE_DB_PATH" # Output Database Variables echo "DB_USER=$DB_USER" echo "DB_PASSWORD=$DB_PASSWORD" echo "DB_NAME=$DB_NAME" echo "DB_HOST=$DB_HOST" + echo "DB_SCHEMA=$DB_SCHEMA" # PostgreSQL should match DB values echo "POSTGRES_USER=$DB_USER" @@ -96,32 +382,62 @@ create_or_validate_env_file() { # Authentication if zenity --question --title="Enable Authentication?" --text="Enable authentication?"; then echo "ENABLE_AUTH=1" - FLASK_SECRET_KEY=$(prompt_variable "FLASK_SECRET_KEY" true) - LOGIN_REDIRECT_URL=$(prompt_variable "LOGIN_REDIRECT_URL" false) - AUTH0_DOMAIN=$(prompt_variable "AUTH0_DOMAIN" false) - AUTH0_CLIENT_ID=$(prompt_variable "AUTH0_CLIENT_ID" false) - AUTH0_CLIENT_SECRET=$(prompt_variable "AUTH0_CLIENT_SECRET" true) - AUTH0_CALLBACK_URL=$(prompt_variable "AUTH0_CALLBACK_URL" false) - AUTH0_AUDIENCE=$(prompt_variable "AUTH0_AUDIENCE" false) - AUTH0_PUBLIC_KEY_URI=$(prompt_variable "AUTH0_PUBLIC_KEY_URI" false) - - # Shibboleth (only if authentication is enabled) - SHIBBOLETH_LOGIN_URL=$(prompt_variable "SHIBBOLETH_LOGIN_URL" false) - SHIBBOLETH_LOGOUT_URL=$(prompt_variable "SHIBBOLETH_LOGOUT_URL" false) - - # Output Authentication Variables + DEFAULT_AUTH_METHOD=$(zenity --list --radiolist \ + --title="Authentication Method" \ + --text="Select authentication provider" \ + --column="Select" --column="Method" \ + TRUE "dev" FALSE "auth0" FALSE "shibboleth") + if [ -z "$DEFAULT_AUTH_METHOD" ]; then + DEFAULT_AUTH_METHOD="dev" + fi + + if [ "$DEFAULT_AUTH_METHOD" = "dummy" ]; then + DEFAULT_AUTH_METHOD="dev" + fi + + if [ "$DEFAULT_AUTH_METHOD" = "dev" ]; then + FLASK_SECRET_KEY="${FLASK_SECRET_KEY:-change-me-in-production}" + LOGIN_REDIRECT_URL=$(prompt_variable "LOGIN_REDIRECT_URL" false "/login_dev") + elif [ "$DEFAULT_AUTH_METHOD" = "auth0" ]; then + FLASK_SECRET_KEY=$(prompt_variable "FLASK_SECRET_KEY" true "change-me-in-production") + LOGIN_REDIRECT_URL=$(prompt_variable "LOGIN_REDIRECT_URL" false "/login") + AUTH0_DOMAIN=$(prompt_variable "AUTH0_DOMAIN" false) + AUTH0_CLIENT_ID=$(prompt_variable "AUTH0_CLIENT_ID" false) + AUTH0_CLIENT_SECRET=$(prompt_variable "AUTH0_CLIENT_SECRET" true) + AUTH0_CALLBACK_URL=$(prompt_variable "AUTH0_CALLBACK_URL" false) + AUTH0_AUDIENCE=$(prompt_variable "AUTH0_AUDIENCE" false) + AUTH0_DB_CONNECTION=$(prompt_variable "AUTH0_DB_CONNECTION" false) + AUTH0_PUBLIC_KEY_URI=$(prompt_variable "AUTH0_PUBLIC_KEY_URI" false) + else + FLASK_SECRET_KEY=$(prompt_variable "FLASK_SECRET_KEY" true "change-me-in-production") + LOGIN_REDIRECT_URL=$(prompt_variable "LOGIN_REDIRECT_URL" false "/login_sso") + SHIBBOLETH_LOGIN_URL=$(prompt_variable "SHIBBOLETH_LOGIN_URL" false) + SHIBBOLETH_LOGOUT_URL=$(prompt_variable "SHIBBOLETH_LOGOUT_URL" false) + fi + + # Output Authentication Variables (provider-specific keys are emitted only when configured) + OUTPUT_AUTH_METHOD="$DEFAULT_AUTH_METHOD" + if [ "$OUTPUT_AUTH_METHOD" = "dev" ]; then + OUTPUT_AUTH_METHOD="dummy" + fi + echo "DEFAULT_AUTH_METHOD=$OUTPUT_AUTH_METHOD" echo "FLASK_SECRET_KEY=$FLASK_SECRET_KEY" echo "LOGIN_REDIRECT_URL=$LOGIN_REDIRECT_URL" - echo "AUTH0_DOMAIN=$AUTH0_DOMAIN" - echo "AUTH0_CLIENT_ID=$AUTH0_CLIENT_ID" - echo "AUTH0_CLIENT_SECRET=$AUTH0_CLIENT_SECRET" - echo "AUTH0_CALLBACK_URL=$AUTH0_CALLBACK_URL" - echo "AUTH0_AUDIENCE=$AUTH0_AUDIENCE" - echo "AUTH0_PUBLIC_KEY_URI=$AUTH0_PUBLIC_KEY_URI" - echo "SHIBBOLETH_LOGIN_URL=$SHIBBOLETH_LOGIN_URL" - echo "SHIBBOLETH_LOGOUT_URL=$SHIBBOLETH_LOGOUT_URL" + if [ "$DEFAULT_AUTH_METHOD" = "auth0" ]; then + echo "AUTH0_DOMAIN=$AUTH0_DOMAIN" + echo "AUTH0_CLIENT_ID=$AUTH0_CLIENT_ID" + echo "AUTH0_CLIENT_SECRET=$AUTH0_CLIENT_SECRET" + echo "AUTH0_CALLBACK_URL=$AUTH0_CALLBACK_URL" + echo "AUTH0_AUDIENCE=$AUTH0_AUDIENCE" + echo "AUTH0_DB_CONNECTION=$AUTH0_DB_CONNECTION" + echo "AUTH0_PUBLIC_KEY_URI=$AUTH0_PUBLIC_KEY_URI" + elif [ "$DEFAULT_AUTH_METHOD" = "shibboleth" ]; then + echo "SHIBBOLETH_LOGIN_URL=$SHIBBOLETH_LOGIN_URL" + echo "SHIBBOLETH_LOGOUT_URL=$SHIBBOLETH_LOGOUT_URL" + fi else echo "ENABLE_AUTH=0" + echo "DEFAULT_AUTH_METHOD=dummy" fi } > "$env_file" @@ -134,6 +450,11 @@ create_or_validate_env_file() { run_docker_compose() { local compose_url=$1 local compose_file=$(basename "$compose_url") + local sqlite_override_file="docker-sqlite.override.yml" + local deploy_override_file=".docker-deploy.${DEPLOYMENT_NAME}.override.yml" + local use_local_mdv_image="0" + local mdv_mount="mdv-data:/app/mdv" + local escaped_mount="" if [ ! -f "$compose_file" ]; then zenity --info --text="Downloading $compose_file..." @@ -143,24 +464,154 @@ run_docker_compose() { fi fi - zenity --info --text="Pulling stable Docker images..." - if ! docker compose -f "$compose_file" pull; then - zenity --error --text="Error: Failed to pull Docker images." + if ! sed -E -i.bak "s/\"[0-9]+:5055\"/\"${APP_PORT}:5055\"/" "$compose_file"; then + zenity --error --text="Error: Failed to set app port in $compose_file." exit 1 fi - zenity --info --text="Starting Docker Compose..." - if ! docker compose -f "$compose_file" up -d; then - zenity --error --text="Error: Failed to start Docker Compose." + if [ "$STORAGE_MODE" = "host" ]; then + mkdir -p "$HOST_DATA_PATH" + mdv_mount="${HOST_DATA_PATH}:/app/mdv" + fi + escaped_mount=${mdv_mount//\\/\\\\} + escaped_mount=${escaped_mount//&/\\&} + escaped_mount=${escaped_mount//|/\\|} + escaped_mount=${escaped_mount//\"/\\\"} + if ! sed -E -i.bak "s|^[[:space:]]*-[[:space:]]*.+:/app/mdv[[:space:]]*$| - \"${escaped_mount}\"|" "$compose_file"; then + zenity --error --text="Error: Failed to set data mount in $compose_file." exit 1 fi + cat > "$deploy_override_file" <> "$deploy_override_file" <> "$deploy_override_file" <> "$deploy_override_file" </dev/null 2>&1; then + use_local_mdv_image="1" + zenity --info --text="Detected local image mdvadmin/mdv:stable. Skipping mdv_app pull." + fi + + if [ "$DEPLOY_MODE" = "replace" ]; then + zenity --info --text="Stopping existing deployment '${DEPLOYMENT_NAME}'..." + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" down --remove-orphans || true + if [ "$RESET_DATA_VOLUME" = "1" ]; then + docker volume rm "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1 || true + fi + if [ "$REMOVE_OLD_VOLUME" = "1" ]; then + docker volume rm "${DEPLOYMENT_NAME}_mdv-data" >/dev/null 2>&1 || true + fi + if [ "$REMOVE_OLD_HOST_PATH" = "1" ] && [ -n "$PREVIOUS_HOST_DATA_PATH" ]; then + rm -rf "$PREVIOUS_HOST_DATA_PATH" >/dev/null 2>&1 || true + fi + fi + + if [ "${DB_BACKEND:-sqlite}" = "sqlite" ]; then + if [ ! -f "$sqlite_override_file" ]; then + zenity --error --text="Missing $sqlite_override_file. Please ensure it exists in the project root." + exit 1 + fi + + if [ "$use_local_mdv_image" != "1" ]; then + zenity --info --text="Pulling mdv_app image..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" -f "$sqlite_override_file" pull mdv_app; then + zenity --error --text="Error: Failed to pull mdv_app image." + exit 1 + fi + fi + + zenity --info --text="Starting Docker Compose (sqlite backend)..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" -f "$sqlite_override_file" up -d --no-deps mdv_app; then + zenity --error --text="Error: Failed to start Docker Compose for sqlite backend." + exit 1 + fi + + # Ensure postgres service is not left running from previous deploys. + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" stop mdv_db >/dev/null 2>&1 || true + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" rm -f mdv_db >/dev/null 2>&1 || true + docker volume rm "${DEPLOYMENT_NAME}_postgres-data" >/dev/null 2>&1 || true + if [ "${PREV_DB_BACKEND:-}" = "postgres" ] && [ "${PREV_DB_HOST:-}" = "mdv-db" ] && [ -n "${PREV_DB_SCHEMA:-}" ]; then + PREV_DB_USER_SAFE="${PREV_DB_USER:-testuser}" + PREV_DB_NAME_SAFE="${PREV_DB_NAME:-mdv}" + docker exec mdv-db psql -U "$PREV_DB_USER_SAFE" -d "$PREV_DB_NAME_SAFE" -c "DROP SCHEMA IF EXISTS \"${PREV_DB_SCHEMA}\" CASCADE;" >/dev/null 2>&1 || true + NON_SYSTEM_SCHEMA_COUNT=$(docker exec mdv-db psql -U "$PREV_DB_USER_SAFE" -d "$PREV_DB_NAME_SAFE" -t -A -c "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name NOT IN ('pg_catalog','information_schema','public') AND schema_name NOT LIKE 'pg_toast%' AND schema_name NOT LIKE 'pg_temp_%';" 2>/dev/null | tr -d '[:space:]') + if [ "${NON_SYSTEM_SCHEMA_COUNT:-1}" = "0" ]; then + docker stop mdv-db >/dev/null 2>&1 || true + docker rm mdv-db >/dev/null 2>&1 || true + fi + fi + elif [ "${POSTGRES_MODE:-dedicated}" = "reuse" ]; then + if [ "$use_local_mdv_image" != "1" ]; then + zenity --info --text="Pulling mdv_app image..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull mdv_app; then + zenity --error --text="Error: Failed to pull mdv_app image." + exit 1 + fi + fi + + zenity --info --text="Starting Docker Compose (postgres reuse mode, app only)..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" up -d --no-deps mdv_app; then + zenity --error --text="Error: Failed to start Docker Compose for postgres reuse mode." + exit 1 + fi + + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" stop mdv_db >/dev/null 2>&1 || true + docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" rm -f mdv_db >/dev/null 2>&1 || true + else + if [ "$use_local_mdv_image" = "1" ]; then + zenity --info --text="Pulling postgres image only (mdv_app is local)..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull mdv_db; then + zenity --error --text="Error: Failed to pull postgres image." + exit 1 + fi + else + zenity --info --text="Pulling stable Docker images..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" pull; then + zenity --error --text="Error: Failed to pull Docker images." + exit 1 + fi + fi + + zenity --info --text="Starting Docker Compose..." + if ! docker compose -p "$DEPLOYMENT_NAME" --env-file "$ENV_FILE" -f "$compose_file" -f "$deploy_override_file" up -d; then + zenity --error --text="Error: Failed to start Docker Compose." + exit 1 + fi + fi + zenity --info --text="Docker Compose started successfully!" } # Function to open the browser open_browser() { - URL="http://localhost:5055" + URL="http://localhost:${APP_PORT}" zenity --question --title="Open Application" --text="Do you want to open MDV in your browser now?" if [ $? -eq 0 ]; then @@ -180,6 +631,7 @@ zenity --info --title="MDV Deployment" --text="Welcome to the MDV application de check_zenity_installed check_docker_installed check_docker_daemon +prompt_deployment_settings # Create app_logs directory if it doesn't exist if [ ! -d "app_logs" ]; then @@ -191,7 +643,7 @@ fi create_or_validate_env_file -DOCKER_COMPOSE_URL="https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-local.yml" +DOCKER_COMPOSE_URL="https://raw.githubusercontent.com/Taylor-CCB-Group/MDV/main/docker-compose.yml" run_docker_compose "$DOCKER_COMPOSE_URL" zenity --info --title="Deployment Complete" --text="MDV application deployed successfully!\nClick OK to proceed." diff --git a/docker-compose.sqlite-multi.yml b/docker-compose.sqlite-multi.yml new file mode 100644 index 000000000..d1bd3b337 --- /dev/null +++ b/docker-compose.sqlite-multi.yml @@ -0,0 +1,61 @@ +# Multi-instance MDV sqlite deployment (pre-loaded image, no postgres). +# +# Usage on the target VM: +# docker load -i mdv-sqlite.tar +# docker tag mdvadmin/mdv:sqlite mdvadmin/mdv_sqlite:stable # if tag differs after load +# docker compose -f docker-compose.sqlite-multi.yml up -d +# +# Requires: +# - config/user_config.json on the host +# - .env.sqlite_1, .env.sqlite_2, .env.sqlite_3 (see repo; gitignored) + +x-mdv-sqlite-common: &mdv-sqlite-common + image: mdvadmin/mdv:sqlite + pull_policy: never + restart: always + volumes: + - ./config/user_config.json:/config/user_config.json + logging: + driver: local + options: + max-size: 50m + max-file: "5" + +services: + mdv_sqlite_1: + <<: *mdv-sqlite-common + container_name: mdv_sqlite_1 + ports: + - "5059:5055" + volumes: + - mdv_sqlite-1-data:/app/mdv + - ./config/user_config.json:/config/user_config.json + env_file: + - .env.sqlite_1 + + mdv_sqlite_2: + <<: *mdv-sqlite-common + container_name: mdv_sqlite_2 + ports: + - "5060:5055" + volumes: + - mdv_sqlite-2-data:/app/mdv + - ./config/user_config.json:/config/user_config.json + env_file: + - .env.sqlite_2 + + mdv_sqlite_3: + <<: *mdv-sqlite-common + container_name: mdv_sqlite_3 + ports: + - "5061:5055" + volumes: + - mdv_sqlite-3-data:/app/mdv + - ./config/user_config.json:/config/user_config.json + env_file: + - .env.sqlite_3 + +volumes: + mdv_sqlite-1-data: + mdv_sqlite-2-data: + mdv_sqlite-3-data: diff --git a/docker-local.yml b/docker-compose.yml similarity index 80% rename from docker-local.yml rename to docker-compose.yml index 87d8e06a7..08e0f955e 100644 --- a/docker-local.yml +++ b/docker-compose.yml @@ -7,14 +7,15 @@ services: - "5055:5055" volumes: #- ./python:/app/python - - mdv-data:/app/mdv + - 'mdv-data:/app/mdv' #- ./app_logs:/app/logs # REQUIRED FOR DEPLOYMENT: Mount external config file for user configuration (outside the image) # for local development, the default will be to use default config.json from the repo with everything enabled. # Uncomment the line below if you want to use custom extensions: # - ./config/user_config.json:/config/user_config.json env_file: - - .env # Load environment variables from the .env file + - path: .env # Load environment variables from the .env file (optional; deploy.sh supplies .env. via override) + required: false # Example .env file content: # DB_NAME=mdv_db # DB_USER=admin @@ -42,9 +43,10 @@ services: volumes: - postgres-data:/var/lib/postgresql/data env_file: - - .env # Load database credentials from the .env file + - path: .env # Load database credentials from the .env file (optional; deploy.sh supplies .env. via override) + required: false restart: always volumes: postgres-data: - mdv-data: \ No newline at end of file + mdv-data: diff --git a/docker-compose.yml.bak b/docker-compose.yml.bak new file mode 100644 index 000000000..08e0f955e --- /dev/null +++ b/docker-compose.yml.bak @@ -0,0 +1,52 @@ +services: + mdv_app: + build: + context: . + image: mdvadmin/mdv:stable + ports: + - "5055:5055" + volumes: + #- ./python:/app/python + - 'mdv-data:/app/mdv' + #- ./app_logs:/app/logs + # REQUIRED FOR DEPLOYMENT: Mount external config file for user configuration (outside the image) + # for local development, the default will be to use default config.json from the repo with everything enabled. + # Uncomment the line below if you want to use custom extensions: + # - ./config/user_config.json:/config/user_config.json + env_file: + - path: .env # Load environment variables from the .env file (optional; deploy.sh supplies .env. via override) + required: false + # Example .env file content: + # DB_NAME=mdv_db + # DB_USER=admin + # DB_PASSWORD=password + # DB_HOST=mdv_db + # MDV_USER_CONFIG_PATH=/config/user_config.json # Only set this if you mount the config file above + # + # To use custom extensions: + # 1. Create a ./config/user_config.json file on the host with content: + # { + # "extensions": ["chat", "project_manager"] + # } + # 2. Uncomment the volume mount line above + # 3. Set MDV_USER_CONFIG_PATH=/config/user_config.json in your .env file + depends_on: + - mdv_db + restart: always + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + mdv_db: + image: postgres:16 + volumes: + - postgres-data:/var/lib/postgresql/data + env_file: + - path: .env # Load database credentials from the .env file (optional; deploy.sh supplies .env. via override) + required: false + restart: always + +volumes: + postgres-data: + mdv-data: diff --git a/docker-sqlite.override.yml b/docker-sqlite.override.yml new file mode 100644 index 000000000..c094f47cd --- /dev/null +++ b/docker-sqlite.override.yml @@ -0,0 +1,3 @@ +services: + mdv_app: + depends_on: [] diff --git a/python/mdvtools/auth/auth0_provider.py b/python/mdvtools/auth/auth0_provider.py index 1d89bfb3a..bc933aabb 100644 --- a/python/mdvtools/auth/auth0_provider.py +++ b/python/mdvtools/auth/auth0_provider.py @@ -19,7 +19,6 @@ from auth0.management import Auth0 from auth0.authentication import GetToken from auth0.exceptions import RateLimitError -import random # Add JWKS cache with thread-safe access _jwks_cache = {} @@ -27,10 +26,6 @@ _jwks_cache_lock = threading.Lock() # Lock to protect cache access JWKS_CACHE_DURATION = 3600 # Cache for 1 hour -# Rate limiting and retry parameters -BASE_DELAY = 1 # Base delay in seconds -MAX_DELAY = 8 # Maximum delay in seconds - class Auth0Provider(AuthProvider): def __init__(self, app, oauth: OAuth, client_id: str, client_secret: str, domain: str): """ @@ -419,8 +414,7 @@ def __init__( max_pagination_rate_limit_retries: int = 5, pagination_retry_delay: float = 2.0, per_page: int = 50, - max_role_fetch_retries: int = 3, - user_processing_delay: float = 1.0 + admin_auth_ids: Optional[set] = None ): self.auth0 = auth0 self.all_projects = all_projects @@ -433,9 +427,9 @@ def __init__( self.max_pagination_rate_limit_retries = max_pagination_rate_limit_retries self.pagination_retry_delay = pagination_retry_delay self.per_page = per_page - self.max_role_fetch_retries = max_role_fetch_retries - self.user_processing_delay = user_processing_delay - + # auth0 user_ids holding the 'admin' role, fetched once per sync + self.admin_auth_ids = admin_auth_ids if admin_auth_ids is not None else set() + def to_dict(self) -> Dict[str, int]: """Convert context stats to dictionary.""" return { @@ -450,66 +444,79 @@ def reset_pagination_rate_limit(self) -> None: self.pagination_rate_limit_count = 0 self.pagination_retry_delay = 2.0 - def _fetch_user_roles_with_retry( - self, - auth0_id: str, + def _fetch_admin_auth_ids( + self, + auth0: Auth0, context: 'SyncContext' - ) -> Optional[Dict[str, Any]]: + ) -> set: """ - Fetch user roles from Auth0 with retry logic for rate limiting. - + Fetch the Auth0 user_ids of every member of the 'admin' role, once. + + Replaces the previous per-user ``users.list_roles()`` call (an N+1 that + made one Management-API request per user) with ~2 requests total: one to + resolve the 'admin' role id, then paged requests for its members. + Args: - auth0_id: Auth0 user ID - context: SyncContext object containing auth0 client, page, and stats - + auth0: Auth0 Management API client + context: SyncContext (used for rate-limit retry configuration) + Returns: - dict: User roles dictionary, or None if fetch failed after retries + set: auth0 user_ids holding the 'admin' role (empty if none / on error). """ - roles = None - retry_count = 0 - success = False - max_retries = context.max_role_fetch_retries - - while retry_count < max_retries and not success: + # 1. Resolve the 'admin' role id. + try: + roles_resp = auth0.roles.list(name_filter='admin') + roles_list = roles_resp.get('roles', []) if isinstance(roles_resp, dict) else [] + admin_role = next((r for r in roles_list if r.get('name') == 'admin'), None) + except Exception as e: + logging.error(f"Failed to resolve Auth0 'admin' role: {e}") + return set() + + if not admin_role: + logging.warning("No 'admin' role found in Auth0; no users will be marked admin.") + return set() + + # 2. Page through the role's members, collecting user_ids. + admin_ids: set = set() + page = 0 + retry_delay = 2.0 + rate_limit_retries = 0 + while True: try: - roles = context.auth0.users.list_roles(auth0_id) - success = True - except RateLimitError as e: - retry_count += 1 - if retry_count >= max_retries: - # Max retries exceeded - error_info = { - 'user_id': auth0_id, - 'page': context.page, - 'processed': context.processed_users, - 'status': getattr(e, 'status_code', None), - 'code': getattr(e, 'error_code', None), - 'message': getattr(e, 'message', str(e)), - 'reset_at': getattr(e, 'reset_at', None), - 'remaining': getattr(e, 'remaining', None), - 'limit': getattr(e, 'limit', None), - } - error_info = {k: v for k, v in error_info.items() if v is not None} - + members = auth0.roles.list_users(admin_role['id'], page=page, per_page=100) + rate_limit_retries = 0 + except RateLimitError: + rate_limit_retries += 1 + if rate_limit_retries > context.max_pagination_rate_limit_retries: logging.error( - f"Rate limit during role fetch for user {auth0_id} after {max_retries} retries: " - f"{error_info}. Skipping user." - ) - return None - else: - # Calculate exponential backoff delay with jitter - delay = min(BASE_DELAY * (2 ** retry_count) + random.uniform(0, 1), MAX_DELAY) - logging.warning( - f"Rate limit during role fetch for user {auth0_id}. " - f"Retrying in {delay:.2f} seconds... (Attempt {retry_count}/{max_retries})" + f"Too many rate-limit retries listing admin members (page {page}); " + f"proceeding with {len(admin_ids)} collected so far." ) - time.sleep(delay) + break + logging.warning( + f"Rate limit listing admin members (page {page}). " + f"Retrying in {retry_delay:.1f}s... (attempt {rate_limit_retries})" + ) + time.sleep(retry_delay) + retry_delay = min(retry_delay * 2, 60.0) + continue except Exception as e: - # Non-rate-limit error - logging.error(f"Error fetching roles for user {auth0_id}: {str(e)}") - return None - - return roles if success else None + logging.error(f"Failed to list admin role members (page {page}): {e}") + break + + users = members.get('users', []) if isinstance(members, dict) else [] + for u in users: + uid = u.get('user_id') + if uid: + admin_ids.add(uid) + + if len(users) < 100: + break + page += 1 + time.sleep(1) # gentle pacing between member pages + + logging.info(f"Fetched {len(admin_ids)} admin member(s) from Auth0.") + return admin_ids def _process_single_user( self, @@ -545,18 +552,9 @@ def _process_single_user( auth_id=auth0_id ) - # Add delay between role requests to avoid rate limiting - time.sleep(context.user_processing_delay) - - # Fetch user's roles with retry mechanism - roles = self._fetch_user_roles_with_retry(auth0_id, context) - if roles is None: - # Failed to fetch roles, skip this user - return False - - roles_list = roles.get('roles', []) - is_admin = any(role['name'] == 'admin' for role in roles_list) - + # Admin status comes from the pre-fetched admin role members (no per-user API call) + is_admin = auth0_id in context.admin_auth_ids + # Update admin status was_admin = db_user.is_admin db_user.is_admin = is_admin @@ -639,7 +637,7 @@ def _handle_pagination_rate_limit( logging.warning( f"Rate limit hit ({new_count}/{context.max_pagination_rate_limit_retries}). " f"Possible issues: frequent calls, concurrent execution, batch size ({context.per_page}), " - f"or insufficient delays (1s pages, {context.user_processing_delay}s roles). Retrying in {context.pagination_retry_delay}s..." + f"or insufficient delays (1s between pages). Retrying in {context.pagination_retry_delay}s..." ) def _log_sync_statistics(self, initial_user_count: int, initial_admin_count: int) -> None: @@ -661,22 +659,19 @@ def _log_sync_statistics(self, initial_user_count: int, initial_admin_count: int f"{final_admin_count} admin users ({final_admin_count - initial_admin_count:+d})" ) - def sync_users_to_db(self, user_processing_delay: float = 1.0) -> None: + def sync_users_to_db(self) -> None: """ Syncs users from Auth0 to the application's database using UserService and UserProjectService. Implements rate limiting and retry logic for Auth0 API calls. - + WARNING: This function makes many Auth0 Management API calls and should be called manually only. This function should only be called from manage_project_permissions.py script. - - Args: - user_processing_delay: Delay in seconds between processing each user (default: 1.0) - + Design concerns: - Auth0 Management API has strict rate limits (typically 2 req/sec for free tier) - - This function makes 1 + N*2 API calls where N = number of users (list users + list_roles per user) - - If called frequently, will hit rate limits quickly - - Consider: caching, background jobs, or incremental sync instead of full sync on-demand + - This function makes ~ceil(N/per_page) + ~ceil(A/per_page) API calls, where N = total + users and A = admin-role members (admins are fetched once, not once per user) + - If called frequently it still adds load; prefer event-driven sync (see issue #515) """ from mdvtools.dbutils.dbmodels import Project @@ -707,10 +702,13 @@ def sync_users_to_db(self, user_processing_delay: float = 1.0) -> None: sync_context = self.SyncContext( auth0=auth0, all_projects=all_projects, - page=0, - user_processing_delay=user_processing_delay + page=0 ) - + + # Fetch all 'admin' role members once, up front. Replaces the old + # per-user list_roles() call (N+1) with ~2 calls total. + sync_context.admin_auth_ids = self._fetch_admin_auth_ids(auth0, sync_context) + # Fetch users from Auth0 connection with pagination while True: try: diff --git a/python/mdvtools/auth/tests/test_auth.py b/python/mdvtools/auth/tests/test_auth.py index 14e5eb040..351198f7a 100644 --- a/python/mdvtools/auth/tests/test_auth.py +++ b/python/mdvtools/auth/tests/test_auth.py @@ -180,14 +180,9 @@ def test_sync_users_to_db_success( {'users': []} # Empty list to signal end ] - # Mock role fetches - first user is admin, rest are not - def mock_list_roles(user_id): - if user_id == 'auth0|user0': - return {'roles': [{'name': 'admin'}]} - else: - return {'roles': []} - - mock_auth0_client.users.list_roles.side_effect = mock_list_roles + # Mock admin-role fetch - user0 is the only admin (fetched once, up front) + mock_auth0_client.roles.list.return_value = {'roles': [{'id': 'rol_admin', 'name': 'admin'}]} + mock_auth0_client.roles.list_users.return_value = {'users': [{'user_id': 'auth0|user0'}]} # Mock OAuth initialization to avoid real HTTP requests with patch('mdvtools.auth.auth0_provider.requests.get') as mock_get: @@ -213,25 +208,26 @@ def mock_list_roles(user_id): # Verify: 75 users processed assert mock_auth0_client.users.list.call_count >= 2 assert mock_user_service.add_or_update_user.call_count == 75 - # Admin user should get project assignments - # `>= 0` is vacuous and doesn't test anything - # assert mock_user_project_service.add_or_update_user_project.call_count >= 0 + # Admin members were fetched once, up front — not once per user (no N+1) + assert mock_auth0_client.roles.list.called + mock_auth0_client.users.list_roles.assert_not_called() - def test_sync_users_to_db_rate_limit_role_fetch( + def test_sync_users_to_db_rate_limit_admin_fetch( self, auth0_app, mock_auth0_client, mock_user_service, mock_user_project_service, mock_db, mock_user_model, mock_project_model, mock_get_token, mock_auth0_class ): - """Test rate limit on role fetch with successful retry.""" + """Test rate limit on the admin-members fetch with successful retry.""" # Setup: single user mock_auth0_client.users.list.side_effect = [ {'users': [{'user_id': 'auth0|user1', 'email': 'user1@test.com'}]}, {'users': []} ] - - # Mock role fetch: rate limit on first call, succeed on retry + + # Admin role resolves fine; listing its members rate-limits once then succeeds + mock_auth0_client.roles.list.return_value = {'roles': [{'id': 'rol_admin', 'name': 'admin'}]} call_count = [0] - def mock_list_roles(user_id): + def mock_list_users(role_id, page, per_page): call_count[0] += 1 if call_count[0] == 1: raise RateLimitError( @@ -239,9 +235,9 @@ def mock_list_roles(user_id): message="Rate limit exceeded", reset_at=1234567890 ) - return {'roles': []} - - mock_auth0_client.users.list_roles.side_effect = mock_list_roles + return {'users': []} + + mock_auth0_client.roles.list_users.side_effect = mock_list_users # Mock OAuth initialization with patch('mdvtools.auth.auth0_provider.requests.get') as mock_get: @@ -263,8 +259,8 @@ def mock_list_roles(user_id): with patch('mdvtools.auth.auth0_provider.time.sleep'): # Speed up test provider.sync_users_to_db() - # Verify: user was processed after retry - assert mock_auth0_client.users.list_roles.call_count == 2 # Initial + retry + # Verify: admin-member fetch was retried after the rate limit + assert mock_auth0_client.roles.list_users.call_count == 2 # Initial + retry assert mock_user_service.add_or_update_user.called def test_sync_users_to_db_rate_limit_pagination( @@ -354,9 +350,6 @@ def test_sync_users_to_db_database_error( {'users': []} ] - # Mock roles - mock_auth0_client.users.list_roles.return_value = {'roles': []} - # Mock database commit to raise error mock_db.session.commit.side_effect = Exception("Database error") diff --git a/python/mdvtools/dbutils/mdv_server_app.py b/python/mdvtools/dbutils/mdv_server_app.py index dd63fa224..197fc2156 100644 --- a/python/mdvtools/dbutils/mdv_server_app.py +++ b/python/mdvtools/dbutils/mdv_server_app.py @@ -30,6 +30,10 @@ # Read environment flag for authentication ENABLE_AUTH = os.getenv("ENABLE_AUTH", "0").lower() in ["1", "true", "yes"] logger.info(f"Authentication enabled: {ENABLE_AUTH}") + +# Opt-in flag: sync_users_to_db() runs on startup as performance improved +RUN_AUTH0_SYNC_ON_START = os.getenv("RUN_AUTH0_SYNC_ON_START", "0").lower() in ["1", "true", "yes"] +logger.info(f"Auth0 sync on startup: {RUN_AUTH0_SYNC_ON_START}") oauth = None if ENABLE_AUTH: @@ -82,9 +86,12 @@ def create_flask_app(config_name=None): try: logger.info("Creating tables") with app.app_context(): - logger.info("Waiting for DB to set up") - if config_name != 'test': + db_backend = app.config.get('DB_BACKEND', 'sqlite') + if config_name != 'test' and db_backend == 'postgres': + logger.info("Waiting for PostgreSQL to set up") wait_for_database(app) + elif config_name != 'test': + logger.info("SQLite backend selected; skipping PostgreSQL readiness checks") if not tables_exist(): logger.info("Creating database tables") db.create_all() @@ -94,8 +101,19 @@ def create_flask_app(config_name=None): if ENABLE_AUTH: try: - # Note: sync_users_to_db() is no longer called automatically on startup. - # It should only be called manually from manage_project_permissions.py script. + # sync_users_to_db() runs on startup as performance improved + if RUN_AUTH0_SYNC_ON_START: + try: + auth_provider = get_auth_provider() + if hasattr(auth_provider, "sync_users_to_db"): + logger.info("RUN_AUTH0_SYNC_ON_START enabled; syncing users from Auth0...") + auth_provider.sync_users_to_db() + logger.info("Auth0 user sync complete.") + else: + logger.info("Auth provider has no sync_users_to_db(); skipping startup sync.") + except Exception as sync_err: + logger.exception(f"Auth0 startup sync failed; continuing with existing DB state: {sync_err}") + logger.info("Caching user-projects data...") cache_user_projects() # Cache the user-project mappings into Redis only when Auth is enabled @@ -181,6 +199,7 @@ def wait_for_database(app): os.getenv('DB_HOST'), db_name ) + db_schema = (os.getenv('DB_SCHEMA') or '').strip() # Update the database URI in the app config (this is the same as the initial URI from load_config) app.config['SQLALCHEMY_DATABASE_URI'] = target_uri @@ -219,6 +238,17 @@ def wait_for_database(app): # The engine should already be valid since the URI hasn't changed test_connection = None try: + if db_schema: + schema_engine = create_engine(target_uri, isolation_level="AUTOCOMMIT") + schema_connection = None + try: + schema_connection = schema_engine.connect() + schema_connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{db_schema}"')) + logger.info(f"Ensured database schema exists: {db_schema}") + finally: + if schema_connection: + schema_connection.close() + schema_engine.dispose() test_connection = db.engine.connect() test_connection.execute(text('SELECT 1')) logger.info("Successfully connected to target database") @@ -300,18 +330,37 @@ def load_config(app, config_name=None, enable_auth=False): if config_name == 'test': app.config['PREFERRED_URL_SCHEME'] = 'http' app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' + app.config['DB_BACKEND'] = 'sqlite' else: app.config['PREFERRED_URL_SCHEME'] = 'https' - - db_user = os.getenv('DB_USER') or read_secret('db_user') - db_password = os.getenv('DB_PASSWORD') or read_secret('db_password') - db_name = os.getenv('DB_NAME') or read_secret('db_name') - db_host = os.getenv('DB_HOST') or app.config.get('db_host') + db_backend = (os.getenv('DB_BACKEND') or 'sqlite').strip().lower() + if db_backend not in ['sqlite', 'postgres']: + logger.warning(f"Unknown DB_BACKEND '{db_backend}', defaulting to sqlite") + db_backend = 'sqlite' - if not all([db_user, db_password, db_name, db_host]): - raise ValueError("Error: One or more required secrets or configurations are missing.") - - app.config['SQLALCHEMY_DATABASE_URI'] = f'postgresql://{db_user}:{db_password}@{db_host}/{db_name}' + app.config['DB_BACKEND'] = db_backend + + if db_backend == 'sqlite': + sqlite_db_path = (os.getenv('SQLITE_DB_PATH') or '/app/mdv/mdv.sqlite3').strip() + app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{sqlite_db_path}' + logger.info(f"Database backend selected: sqlite (path: {sqlite_db_path})") + else: + db_user = os.getenv('DB_USER') or read_secret('db_user') + db_password = os.getenv('DB_PASSWORD') or read_secret('db_password') + db_name = os.getenv('DB_NAME') or read_secret('db_name') + db_host = os.getenv('DB_HOST') or app.config.get('db_host') + db_schema = (os.getenv('DB_SCHEMA') or '').strip() + + if not all([db_user, db_password, db_name, db_host]): + raise ValueError("Error: One or more required secrets or configurations are missing.") + + app.config['SQLALCHEMY_DATABASE_URI'] = f'postgresql://{db_user}:{db_password}@{db_host}/{db_name}' + if db_schema: + app.config['SQLALCHEMY_ENGINE_OPTIONS'] = { + 'connect_args': {'options': f'-csearch_path={db_schema}'} + } + logger.info(f"Using database schema: {db_schema}") + logger.info(f"Database backend selected: postgres (host: {db_host}, db: {db_name})") # Only configure Auth0 if ENABLE_AUTH is True diff --git a/python/mdvtools/dbutils/routes.py b/python/mdvtools/dbutils/routes.py index 8393d0a5f..3c0a63f24 100644 --- a/python/mdvtools/dbutils/routes.py +++ b/python/mdvtools/dbutils/routes.py @@ -138,7 +138,31 @@ def rescan_projects(): else: logger.info("MDV_API_ROOT not set, redirecting to index") return redirect(url_for("index")) - + + @app.route('/refresh_cache', methods=['GET', 'POST']) + def refresh_cache(): + """ + Reloads the in-memory user/project caches from the database. + + Needed because out-of-band DB changes (e.g. the Auth0 sync run via + `docker exec`) update the DB in a separate process and never touch the + running server's cache, so they stay invisible until reload. Hitting this + (as a logged-in admin) refreshes the live cache without a container restart. + """ + logger.info(" /REFRESH CACHE...") + + if ENABLE_AUTH: + user = session.get('user') + if not user: + raise ValueError("User not found in session.") + if not user.get("is_admin", False): + abort(403) # Forbidden + + ok = cache_user_projects() + return jsonify({"refreshed": bool(ok)}), (200 if ok else 500) + + logger.info("Route registered: /refresh_cache") + def get_project_owners(project_id): owners = [] for user_id, projects in user_project_cache.items(): diff --git a/python/mdvtools/dbutils/test/unit/test_mdv_server_app.py b/python/mdvtools/dbutils/test/unit/test_mdv_server_app.py index d980f88be..af6e3244d 100644 --- a/python/mdvtools/dbutils/test/unit/test_mdv_server_app.py +++ b/python/mdvtools/dbutils/test/unit/test_mdv_server_app.py @@ -275,6 +275,7 @@ def test_load_config_invalid_json(self, mock_abspath, mock_dirname, mock_file): @patch('os.path.dirname') @patch('os.path.abspath') @patch.dict(os.environ, { + 'DB_BACKEND': 'postgres', 'DB_USER': 'test_user', 'DB_PASSWORD': 'test_password', 'DB_NAME': 'test_db', @@ -300,8 +301,8 @@ def test_load_config_with_secrets(self, mock_read_secret, mock_abspath, mock_dir mock_dirname.return_value = '/test' mock_read_secret.side_effect = ['secret_user', 'secret_password', 'secret_db'] - # Clear all environment variables to ensure secrets are used - with patch.dict(os.environ, {}, clear=True): + # Keep postgres backend while clearing credentials to ensure secrets are used + with patch.dict(os.environ, {'DB_BACKEND': 'postgres'}, clear=True): load_config(self.app) expected_uri = 'postgresql://secret_user:secret_password@test_host/secret_db' @@ -314,12 +315,53 @@ def test_load_config_test_environment(self, mock_abspath, mock_dirname, mock_fil """Test loading config for test environment.""" mock_abspath.return_value = '/test/path' mock_dirname.return_value = '/test' - + load_config(self.app, config_name='test') - + self.assertEqual(self.app.config['PREFERRED_URL_SCHEME'], 'http') self.assertEqual(self.app.config['SQLALCHEMY_DATABASE_URI'], 'sqlite:///:memory:') + @patch('builtins.open', new_callable=mock_open, read_data='{"track_modifications": false}') + @patch('os.path.dirname') + @patch('os.path.abspath') + def test_load_config_sqlite_backend_default_path(self, mock_abspath, mock_dirname, mock_file): + """Test that DB_BACKEND=sqlite sets a sqlite URI with the default path.""" + mock_abspath.return_value = '/test/path' + mock_dirname.return_value = '/test' + + with patch.dict(os.environ, {'DB_BACKEND': 'sqlite'}, clear=True): + load_config(self.app) + + self.assertEqual(self.app.config['DB_BACKEND'], 'sqlite') + self.assertEqual(self.app.config['SQLALCHEMY_DATABASE_URI'], 'sqlite:////app/mdv/mdv.sqlite3') + + @patch('builtins.open', new_callable=mock_open, read_data='{"track_modifications": false}') + @patch('os.path.dirname') + @patch('os.path.abspath') + def test_load_config_sqlite_backend_custom_path(self, mock_abspath, mock_dirname, mock_file): + """Test that SQLITE_DB_PATH overrides the default sqlite path.""" + mock_abspath.return_value = '/test/path' + mock_dirname.return_value = '/test' + + with patch.dict(os.environ, {'DB_BACKEND': 'sqlite', 'SQLITE_DB_PATH': '/data/custom.sqlite3'}, clear=True): + load_config(self.app) + + self.assertEqual(self.app.config['SQLALCHEMY_DATABASE_URI'], 'sqlite:////data/custom.sqlite3') + + @patch('builtins.open', new_callable=mock_open, read_data='{"track_modifications": false}') + @patch('os.path.dirname') + @patch('os.path.abspath') + def test_load_config_unknown_backend_falls_back_to_sqlite(self, mock_abspath, mock_dirname, mock_file): + """Test that an unrecognised DB_BACKEND value falls back to sqlite.""" + mock_abspath.return_value = '/test/path' + mock_dirname.return_value = '/test' + + with patch.dict(os.environ, {'DB_BACKEND': 'mongodb'}, clear=True): + load_config(self.app) + + self.assertEqual(self.app.config['DB_BACKEND'], 'sqlite') + self.assertIn('sqlite:///', self.app.config['SQLALCHEMY_DATABASE_URI']) + class TestCreateBaseDirectory(unittest.TestCase): """Test cases for the create_base_directory function.""" @@ -701,17 +743,43 @@ def test_create_flask_app_test_config(self, mock_db, mock_create_base_dir, mock_ @patch('mdvtools.dbutils.mdv_server_app.db') @patch('mdvtools.dbutils.mdv_server_app.tables_exist') def test_create_flask_app_production_config(self, mock_tables_exist, mock_db, mock_create_base_dir, mock_load_config, mock_wait_for_db, mock_cache_users, mock_register_app, mock_register_routes, mock_serve_db, mock_serve_fs): - """Test app creation with production configuration.""" + """Test app creation with postgres production configuration calls wait_for_database.""" mock_tables_exist.return_value = True - - # We need to ensure the app has the required config for the auth provider. - mock_load_config.side_effect = lambda app, *args: app.config.update({"DEFAULT_AUTH_METHOD": "dummy"}) - + + mock_load_config.side_effect = lambda app, *args: app.config.update({ + "DEFAULT_AUTH_METHOD": "dummy", + "DB_BACKEND": "postgres", + }) + app = create_flask_app(config_name='production') self.assertIsInstance(app, Flask) mock_wait_for_db.assert_called_once() + @patch('mdvtools.dbutils.mdv_server_app.serve_projects_from_filesystem') + @patch('mdvtools.dbutils.mdv_server_app.serve_projects_from_db') + @patch('mdvtools.dbutils.mdv_server_app.register_routes') + @patch('mdvtools.dbutils.mdv_server_app.ProjectBlueprint.register_app') + @patch('mdvtools.dbutils.mdv_server_app.cache_user_projects') + @patch('mdvtools.dbutils.mdv_server_app.wait_for_database') + @patch('mdvtools.dbutils.mdv_server_app.load_config') + @patch('mdvtools.dbutils.mdv_server_app.create_base_directory') + @patch('mdvtools.dbutils.mdv_server_app.db') + @patch('mdvtools.dbutils.mdv_server_app.tables_exist') + def test_create_flask_app_sqlite_skips_wait_for_database(self, mock_tables_exist, mock_db, mock_create_base_dir, mock_load_config, mock_wait_for_db, mock_cache_users, mock_register_app, mock_register_routes, mock_serve_db, mock_serve_fs): + """Test that wait_for_database is not called when DB_BACKEND is sqlite.""" + mock_tables_exist.return_value = True + + mock_load_config.side_effect = lambda app, *args: app.config.update({ + "DEFAULT_AUTH_METHOD": "dummy", + "DB_BACKEND": "sqlite", + }) + + app = create_flask_app(config_name='production') + + self.assertIsInstance(app, Flask) + mock_wait_for_db.assert_not_called() + @patch('mdvtools.dbutils.mdv_server_app.load_config', side_effect=Exception("Config load failed")) def test_create_flask_app_config_error(self, mock_load_config): """Test create_flask_app with configuration error.""" diff --git a/python/mdvtools/scripts/manage_project_permissions.py b/python/mdvtools/scripts/manage_project_permissions.py index 9ccb8b9a8..bd7e12d3e 100644 --- a/python/mdvtools/scripts/manage_project_permissions.py +++ b/python/mdvtools/scripts/manage_project_permissions.py @@ -186,12 +186,18 @@ def main(): # Otherwise, use normal command parsing for single assignment parser = argparse.ArgumentParser( description='Manage project permissions for users', - epilog='Usage: python manage_project_permissions.py OR python manage_project_permissions.py assign --email ... --project ... --permission ...' + epilog='Usage: python manage_project_permissions.py OR python manage_project_permissions.py sync OR python manage_project_permissions.py assign --email ... --project ... --permission ...' ) # Create subparsers for different commands subparsers = parser.add_subparsers(dest='command', help='Commands', required=True) - + + # Sync-only command (populate users from Auth0; admins auto-get ownership of all projects) + subparsers.add_parser( + 'sync', + help='Sync users from Auth0 to the DB only (no permission assignment)' + ) + # Single assignment command assign_parser = subparsers.add_parser('assign', help='Assign permission for a single user and project') assign_parser.add_argument('--email', required=True, help='User email') @@ -203,9 +209,12 @@ def main(): # Create application context with app.app_context(): - # Sync users from Auth0 first to ensure DB is up to date - sync_users_from_auth0() - if args.command == 'assign': + if args.command == 'sync': + # Sync only — populates users and auto-assigns admins ownership of all projects + success = sync_users_from_auth0() + elif args.command == 'assign': + # Sync users from Auth0 first to ensure DB is up to date, then assign + sync_users_from_auth0() success = assign_permissions(args.email, args.project, args.permission) else: parser.print_help() diff --git a/python/mdvtools/scripts/users_projects_dataframe.py b/python/mdvtools/scripts/users_projects_dataframe.py new file mode 100644 index 000000000..de54c6413 --- /dev/null +++ b/python/mdvtools/scripts/users_projects_dataframe.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +Collect users and projects from the database and merge them into a DataFrame. + +Each row represents one user–project association (from user_projects), with +user and project attributes joined. Output can be printed, saved to CSV, or +returned for use in other code. + +Requires database credentials (env or Docker secrets) as for the main MDV app. +""" +import argparse +import sys +from pathlib import Path +from typing import Dict, Any + +# Add the parent directory to sys.path to import mdvtools modules +_scripts_dir = Path(__file__).resolve().parent +_root = _scripts_dir.parent.parent +if str(_root) not in sys.path: + sys.path.insert(0, str(_root)) + +import pandas as pd +from auth0.authentication import GetToken # pyright: ignore[reportMissingModuleSource] +from auth0.management import Auth0 # pyright: ignore[reportMissingModuleSource] + +from mdvtools.dbutils.dbmodels import db, User, Project, UserProject +from mdvtools.dbutils.safe_mdv_app import app + + +def _fetch_auth0_login_stats() -> Dict[str, Dict[str, Any]]: + """ + Fetch Auth0 login stats keyed by Auth0 user id. + + Returns: + Dict[auth0_user_id, {"last_login": , "logins_count": }] + """ + auth0_domain = app.config.get("AUTH0_DOMAIN") + client_id = app.config.get("AUTH0_CLIENT_ID") + client_secret = app.config.get("AUTH0_CLIENT_SECRET") + auth0_db_connection = app.config.get("AUTH0_DB_CONNECTION") + + if ( + not isinstance(auth0_domain, str) or not auth0_domain.strip() + or not isinstance(client_id, str) or not client_id.strip() + or not isinstance(client_secret, str) or not client_secret.strip() + or not isinstance(auth0_db_connection, str) or not auth0_db_connection.strip() + ): + raise RuntimeError( + "Missing Auth0 configuration. Expected AUTH0_DOMAIN, AUTH0_CLIENT_ID, " + "AUTH0_CLIENT_SECRET, and AUTH0_DB_CONNECTION in app config." + ) + + audience = f"https://{auth0_domain}/api/v2/" + token_client = GetToken( + domain=auth0_domain, + client_id=client_id, + client_secret=client_secret, + ) + mgmt_api_token = token_client.client_credentials(audience=audience)["access_token"] + auth0 = Auth0(auth0_domain, mgmt_api_token) + + result: Dict[str, Dict[str, Any]] = {} + page = 0 + per_page = 100 + while True: + users_page = auth0.users.list( + q=f'identities.connection:"{auth0_db_connection}"', + page=page, + per_page=per_page, + ) + user_list = users_page.get("users", []) + if not user_list: + break + + for user in user_list: + auth0_id = user.get("user_id") + if not auth0_id: + continue + result[auth0_id] = { + "last_login": user.get("last_login"), + "logins_count": user.get("logins_count"), + } + + if len(user_list) < per_page: + break + page += 1 + + return result + + +def load_users_projects_df(include_auth0_login_stats: bool = False): + """ + Load users and projects from the database and merge into a single DataFrame. + + Returns a DataFrame with one row per user–project association, with columns: + - user_id, email, first_name, last_name, auth_id, is_active, is_admin + - project_id, project_name, project_path, project_owner_id, is_public, is_deleted + - can_read, can_write, is_owner + """ + with app.app_context(): + # Query all user–project links with user and project joined + # todo: Update models to use SQLAlchemy 2.0 style Mapped annotations in dbutils/dbmodels.py and remove type: ignore comments + q = ( + db.session.query( + User.id.label("user_id"), + User.email, + User.first_name, + User.last_name, + User.auth_id, + User.is_active, + User.is_admin, + Project.id.label("project_id"), + Project.name.label("project_name"), + Project.path.label("project_path"), + Project.owner.label("project_owner_id"), + Project.is_public, + Project.is_deleted, + Project.description.label("project_description"), + UserProject.can_read, # type: ignore[arg-type] + UserProject.can_write, # type: ignore[arg-type] + UserProject.is_owner, # type: ignore[arg-type] + ) + .join(UserProject, User.id == UserProject.user_id) # type: ignore[arg-type] + .join(Project, Project.id == UserProject.project_id) # type: ignore[arg-type] + .order_by(User.email, Project.name) + ) + rows = q.all() + auth0_login_stats = _fetch_auth0_login_stats() if include_auth0_login_stats else {} + + columns = [ + "user_id", "email", "first_name", "last_name", "auth_id", + "is_active", "is_admin", + "project_id", "project_name", "project_path", "project_owner_id", + "is_public", "is_deleted", "project_description", + "can_read", "can_write", "is_owner", + ] + if include_auth0_login_stats: + columns.extend(["last_login", "logins_count"]) + + if not rows: + return pd.DataFrame(columns=columns) # type: ignore[arg-type] + + records = [] + for r in rows: + record = { + "user_id": r.user_id, + "email": r.email, + "first_name": r.first_name, + "last_name": r.last_name, + "auth_id": r.auth_id, + "is_active": r.is_active, + "is_admin": r.is_admin, + "project_id": r.project_id, + "project_name": r.project_name, + "project_path": r.project_path, + "project_owner_id": r.project_owner_id, + "is_public": r.is_public, + "is_deleted": r.is_deleted, + "project_description": r.project_description, + "can_read": r.can_read, + "can_write": r.can_write, + "is_owner": r.is_owner, + } + if include_auth0_login_stats: + stats = auth0_login_stats.get(r.auth_id, {}) + record["last_login"] = stats.get("last_login") + record["logins_count"] = stats.get("logins_count") + records.append(record) + + return pd.DataFrame(records, columns=columns) # type: ignore[arg-type] + + +def main(): + parser = argparse.ArgumentParser( + description="Collect users and projects from the database and merge into a DataFrame." + ) + parser.add_argument( + "--csv", + metavar="FILE", + help="Save the merged DataFrame to a CSV file", + ) + parser.add_argument( + "--no-print", + action="store_true", + help="Do not print the DataFrame to stdout", + ) + parser.add_argument( + "--with-auth0-login-stats", + action="store_true", + help="Include Auth0 last_login and logins_count fields", + ) + args = parser.parse_args() + + df = load_users_projects_df(include_auth0_login_stats=args.with_auth0_login_stats) + + if not args.no_print: + pd.set_option("display.max_columns", None) + pd.set_option("display.width", None) + pd.set_option("display.max_colwidth", 50) + print(df.to_string()) + + if args.csv: + path = Path(args.csv) + path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(path, index=False) + print(f"\nSaved {len(df)} rows to {path}", file=sys.stderr) + + return df + + +if __name__ == "__main__": + main()