diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 4fc2db8..746ae75 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -2,6 +2,8 @@ name: Docker build and push on: push: + branches: + - 'main' tags: - 'v*' workflow_dispatch: diff --git a/Dockerfile b/Dockerfile index e7d2754..48d8ad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.7-slim-buster +FROM python:3.7 # Add rsync RUN apt update -qq && apt upgrade -y && apt autoremove -y -RUN apt install -y rsync && apt autoremove -y +RUN apt install -y rsync curl && apt autoremove -y EXPOSE 5000 @@ -24,6 +24,7 @@ ENV STATIC_FILES_MIRROR="" # Install pip requirements COPY requirements.txt . +RUN python -m pip install --upgrade pip RUN python -m pip install -r requirements.txt WORKDIR /app diff --git a/deployment/docker-compose/.env.example b/deployment/docker-compose/.env.example new file mode 100644 index 0000000..bd16dbe --- /dev/null +++ b/deployment/docker-compose/.env.example @@ -0,0 +1,26 @@ +POSTGRES_DATA_PATH=./tmp/data/postgres/data +POSTGRES_BACKUP_PATH=./tmp/data/postgres/backups +CACHE_PATH=./tmp/cache +STATIC_PATH=./tmp/static +FLUENTD_LOG_PATH=./tmp/logs/fluentd +FLUENTD_CUSTOM_CONFIG_PATH=./tmp/config +FLUENTD_DOCKER_CONTAINERS_PATH=/var/lib/docker/containers + +POSTGRES_USER=oveda +POSTGRES_PASSWORD= +POSTGRES_DB=oveda + +WEB_TAG=latest +SERVER_NAME= +PREFERRED_URL_SCHEME=https +SECRET_KEY= +SECURITY_PASSWORD_HASH= +MAIL_SERVER= +MAIL_PORT= +MAIL_USERNAME= +MAIL_PASSWORD= +MAIL_DEFAULT_SENDER= +MAIL_USE_TLS=True +GOOGLE_MAPS_API_KEY=AIzaDummy +JWT_PRIVATE_KEY="" +JWT_PUBLIC_JWKS='' \ No newline at end of file diff --git a/deployment/docker-compose/README.md b/deployment/docker-compose/README.md new file mode 100644 index 0000000..9fcc683 --- /dev/null +++ b/deployment/docker-compose/README.md @@ -0,0 +1,23 @@ +# Deployment with Docker compose + +## Configure + +Copy example.env to .env and enter values. + +## Initialize + +```sh +./init.sh +``` + +## Start + +```sh +docker compose up --force-recreate --detach +``` + +## Update app + +```sh +./update.sh +``` diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml new file mode 100644 index 0000000..9dba2e9 --- /dev/null +++ b/deployment/docker-compose/docker-compose.yml @@ -0,0 +1,82 @@ +version: "3.9" +name: "oveda" + +services: + db: + image: postgis/postgis:12-3.1 + restart: always + healthcheck: + test: "pg_isready --username=${POSTGRES_USER} && psql --username=${POSTGRES_USER} --list" + start_period: "5s" + ports: + - 5434:5432 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data + + db-backup: + image: prodrigestivill/postgres-backup-local:12 + restart: always + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: db + POSTGRES_EXTRA_OPTS: "-Z6 -c" + SCHEDULE: "0 0 22 * * *" + volumes: + - ${POSTGRES_BACKUP_PATH}:/backups + depends_on: + db: + condition: service_healthy + + web: + image: danielgrams/gsevpt:${WEB_TAG} + restart: always + healthcheck: + test: "curl -f ${SERVER_NAME}/up" + interval: "60s" + timeout: "5s" + start_period: "5s" + ports: + - "5000:5000" + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + FLASK_APP: main.py + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB} + SECRET_KEY: ${SECRET_KEY} + SECURITY_PASSWORD_HASH: ${SECURITY_PASSWORD_HASH} + MAIL_DEFAULT_SENDER: ${MAIL_DEFAULT_SENDER} + MAIL_PASSWORD: ${MAIL_PASSWORD} + MAIL_PORT: ${MAIL_PORT} + MAIL_SERVER: ${MAIL_SERVER} + MAIL_USE_TLS: ${MAIL_USE_TLS} + MAIL_USERNAME: ${MAIL_USERNAME} + GOOGLE_MAPS_API_KEY: ${GOOGLE_MAPS_API_KEY} + SERVER_NAME: ${SERVER_NAME} + PREFERRED_URL_SCHEME: ${PREFERRED_URL_SCHEME} + GUNICORN_ACCESS_LOG: "-" + STATIC_FILES_MIRROR: /static + CACHE_PATH: tmp + JWT_PRIVATE_KEY: ${JWT_PRIVATE_KEY} + JWT_PUBLIC_JWKS: ${JWT_PUBLIC_JWKS} + volumes: + - ${CACHE_PATH}:/app/project/tmp + - ${STATIC_PATH}:/static + depends_on: + db: + condition: service_healthy + + fluentd: + image: danielgrams/fluentd + restart: always + environment: + FLUENTD_CONF: fluentd-custom.config + volumes: + - ${FLUENTD_LOG_PATH}:/fluentd/log + - ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config:/fluentd/etc/fluentd-custom.config + - ${FLUENTD_DOCKER_CONTAINERS_PATH}:/fluentd/containers diff --git a/deployment/docker-compose/fluentd-custom.config b/deployment/docker-compose/fluentd-custom.config new file mode 100644 index 0000000..d09bf34 --- /dev/null +++ b/deployment/docker-compose/fluentd-custom.config @@ -0,0 +1,60 @@ +# Tail docker logs + + @type tail + read_from_head true + path /fluentd/containers/*/*-json.log + pos_file /fluentd/log/docker.pos + time_format %Y-%m-%dT%H:%M:%S + tag docker.* + format json + + +# Add container id + + @type record_transformer + + container_id ${tag_parts[3]} + + + +# Errors only + + @type grep + + key log + pattern /\[error\]|\| error \|/i + + + + + @type copy + + # Write errors to file + + @type file + path /fluentd/log/docker-error + + + # Tag daily errors + + @type grepcounter + count_interval 14400 # = 4 hours + input_key log + threshold 1 + add_tag_prefix daily_error + + + +# Send daily error mail +# +# @type mail +# host [Host] +# port [Port] +# user [User/Email] +# password [Password] +# from [User/Email] +# to [Recipient] +# subject 'Error alert fluentd' +# message Error occured %s times +# message_out_keys count +# \ No newline at end of file diff --git a/deployment/docker-compose/init.sh b/deployment/docker-compose/init.sh new file mode 100755 index 0000000..71a3ee4 --- /dev/null +++ b/deployment/docker-compose/init.sh @@ -0,0 +1,10 @@ +set -e +source .env + +mkdir -p ${POSTGRES_DATA_PATH} +mkdir -p ${POSTGRES_BACKUP_PATH} +mkdir -p ${CACHE_PATH} +mkdir -p ${STATIC_PATH} +mkdir -p ${FLUENTD_LOG_PATH} +mkdir -p ${FLUENTD_CUSTOM_CONFIG_PATH} +cp ./fluentd-custom.config ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config \ No newline at end of file diff --git a/deployment/docker-compose/update.sh b/deployment/docker-compose/update.sh new file mode 100755 index 0000000..32f7d80 --- /dev/null +++ b/deployment/docker-compose/update.sh @@ -0,0 +1,6 @@ +set -e + +docker compose pull web +docker compose stop web +docker compose run db-backup /backup.sh +docker compose up --detach --force-recreate web \ No newline at end of file diff --git a/doc/development.md b/doc/development.md index 7e195dd..5a2eac4 100644 --- a/doc/development.md +++ b/doc/development.md @@ -68,7 +68,7 @@ pybabel compile -d project/translations ### Build image ```sh -docker build -t gsevpt:latest . +docker build -t danielgrams/gsevpt:latest . ``` ### Run container with existing postgres server diff --git a/project/__init__.py b/project/__init__.py index 763986c..1b2c79a 100644 --- a/project/__init__.py +++ b/project/__init__.py @@ -14,6 +14,11 @@ from flask_wtf.csrf import CSRFProtect from project.custom_session_interface import CustomSessionInterface + +def getenv_bool(name: str, default: str = "False"): + return os.getenv(name, default).lower() in ("true", "1", "t") + + # Create app app = Flask(__name__) app.config["SQLALCHEMY_DATABASE_URI"] = os.environ["DATABASE_URL"] @@ -95,15 +100,15 @@ app.config["WTF_CSRF_CHECK_DEFAULT"] = False # Mail mail_server = os.getenv("MAIL_SERVER") -if mail_server is None: +if not mail_server: app.config["MAIL_SUPPRESS_SEND"] = True app.config["MAIL_DEFAULT_SENDER"] = "test@oveda.de" else: # pragma: no cover app.config["MAIL_SUPPRESS_SEND"] = False app.config["MAIL_SERVER"] = mail_server app.config["MAIL_PORT"] = os.getenv("MAIL_PORT") - app.config["MAIL_USE_TLS"] = os.getenv("MAIL_USE_TLS", True) - app.config["MAIL_USE_SSL"] = os.getenv("MAIL_USE_SSL", False) + app.config["MAIL_USE_TLS"] = getenv_bool("MAIL_USE_TLS", "True") + app.config["MAIL_USE_SSL"] = getenv_bool("MAIL_USE_SSL", "False") app.config["MAIL_USERNAME"] = os.getenv("MAIL_USERNAME") app.config["MAIL_PASSWORD"] = os.getenv("MAIL_PASSWORD") app.config["MAIL_DEFAULT_SENDER"] = os.getenv("MAIL_DEFAULT_SENDER") diff --git a/project/views/root.py b/project/views/root.py index 4dd80a1..5289e3a 100644 --- a/project/views/root.py +++ b/project/views/root.py @@ -5,7 +5,7 @@ from flask import redirect, render_template, request, send_from_directory, url_f from flask_babelex import gettext from markupsafe import Markup -from project import app, cache_path, dump_path, robots_txt_file, sitemap_file +from project import app, cache_path, db, dump_path, robots_txt_file, sitemap_file from project.services.admin import upsert_settings from project.views.utils import track_analytics @@ -32,6 +32,12 @@ def home(): ) +@app.route("/up") +def up(): + db.engine.execute("SELECT 1") + return "OK" + + @app.route("/tos") def tos(): title = gettext("Terms of service")