Deployment with docker compose #376

.github/workflows/docker-build-push.yml

@@ -2,6 +2,8 @@ name: Docker build and push
 
 on:
   push:
+    branches:
+      - 'main'
     tags:
       - 'v*'
   workflow_dispatch:

Dockerfile

@@ -1,8 +1,8 @@
-FROM python:3.7-slim-buster
+FROM python:3.7
 
 # Add rsync
 RUN apt update -qq && apt upgrade -y && apt autoremove -y
-RUN apt install -y rsync && apt autoremove -y
+RUN apt install -y rsync curl && apt autoremove -y
 
 EXPOSE 5000
 
@@ -24,6 +24,7 @@ ENV STATIC_FILES_MIRROR=""
 
 # Install pip requirements
 COPY requirements.txt .
+RUN python -m pip install --upgrade pip
 RUN python -m pip install -r requirements.txt
 
 WORKDIR /app

deployment/docker-compose/.env.example

@@ -0,0 +1,26 @@
+POSTGRES_DATA_PATH=./tmp/data/postgres/data
+POSTGRES_BACKUP_PATH=./tmp/data/postgres/backups
+CACHE_PATH=./tmp/cache
+STATIC_PATH=./tmp/static
+FLUENTD_LOG_PATH=./tmp/logs/fluentd
+FLUENTD_CUSTOM_CONFIG_PATH=./tmp/config
+FLUENTD_DOCKER_CONTAINERS_PATH=/var/lib/docker/containers
+
+POSTGRES_USER=oveda
+POSTGRES_PASSWORD=
+POSTGRES_DB=oveda
+
+WEB_TAG=latest
+SERVER_NAME=
+PREFERRED_URL_SCHEME=https
+SECRET_KEY=
+SECURITY_PASSWORD_HASH=
+MAIL_SERVER=
+MAIL_PORT=
+MAIL_USERNAME=
+MAIL_PASSWORD=
+MAIL_DEFAULT_SENDER=
+MAIL_USE_TLS=True
+GOOGLE_MAPS_API_KEY=AIzaDummy
+JWT_PRIVATE_KEY=""
+JWT_PUBLIC_JWKS=''

deployment/docker-compose/README.md

@@ -0,0 +1,23 @@
+# Deployment with Docker compose
+
+## Configure
+
+Copy example.env to .env and enter values.
+
+## Initialize
+
+```sh
+./init.sh
+```
+
+## Start
+
+```sh
+docker compose up --force-recreate --detach
+```
+
+## Update app
+
+```sh
+./update.sh
+```

deployment/docker-compose/docker-compose.yml

@@ -0,0 +1,82 @@
+version: "3.9"
+name: "oveda"
+
+services:
+  db:
+    image: postgis/postgis:12-3.1
+    restart: always
+    healthcheck:
+      test: "pg_isready --username=${POSTGRES_USER} && psql --username=${POSTGRES_USER} --list"
+      start_period: "5s"
+    ports:
+      - 5434:5432
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    volumes:
+      - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data
+
+  db-backup:
+    image: prodrigestivill/postgres-backup-local:12
+    restart: always
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_HOST: db
+      POSTGRES_EXTRA_OPTS: "-Z6 -c"
+      SCHEDULE: "0 0 22 * * *"
+    volumes:
+      - ${POSTGRES_BACKUP_PATH}:/backups
+    depends_on:
+      db:
+        condition: service_healthy
+
+  web:
+    image: danielgrams/gsevpt:${WEB_TAG}
+    restart: always
+    healthcheck:
+      test: "curl -f ${SERVER_NAME}/up"
+      interval: "60s"
+      timeout: "5s"
+      start_period: "5s"
+    ports:
+      - "5000:5000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      FLASK_APP: main.py
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB}
+      SECRET_KEY: ${SECRET_KEY}
+      SECURITY_PASSWORD_HASH: ${SECURITY_PASSWORD_HASH}
+      MAIL_DEFAULT_SENDER: ${MAIL_DEFAULT_SENDER}
+      MAIL_PASSWORD: ${MAIL_PASSWORD}
+      MAIL_PORT: ${MAIL_PORT}
+      MAIL_SERVER: ${MAIL_SERVER}
+      MAIL_USE_TLS: ${MAIL_USE_TLS}
+      MAIL_USERNAME: ${MAIL_USERNAME}
+      GOOGLE_MAPS_API_KEY: ${GOOGLE_MAPS_API_KEY}
+      SERVER_NAME: ${SERVER_NAME}
+      PREFERRED_URL_SCHEME: ${PREFERRED_URL_SCHEME}
+      GUNICORN_ACCESS_LOG: "-"
+      STATIC_FILES_MIRROR: /static
+      CACHE_PATH: tmp
+      JWT_PRIVATE_KEY: ${JWT_PRIVATE_KEY}
+      JWT_PUBLIC_JWKS: ${JWT_PUBLIC_JWKS}
+    volumes:
+      - ${CACHE_PATH}:/app/project/tmp
+      - ${STATIC_PATH}:/static
+    depends_on:
+      db:
+        condition: service_healthy
+
+  fluentd:
+    image: danielgrams/fluentd
+    restart: always
+    environment:
+      FLUENTD_CONF: fluentd-custom.config
+    volumes:
+      - ${FLUENTD_LOG_PATH}:/fluentd/log
+      - ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config:/fluentd/etc/fluentd-custom.config
+      - ${FLUENTD_DOCKER_CONTAINERS_PATH}:/fluentd/containers

deployment/docker-compose/fluentd-custom.config

@@ -0,0 +1,60 @@
+# Tail docker logs
+<source>
+  @type tail
+  read_from_head true
+  path /fluentd/containers/*/*-json.log
+  pos_file /fluentd/log/docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.*
+  format json
+</source>
+
+# Add container id
+<filter docker.fluentd.containers.*.*.log>
+  @type record_transformer
+  <record>
+    container_id ${tag_parts[3]}
+  </record>
+</filter>
+
+# Errors only
+<filter docker.**>
+  @type grep
+  <regexp>
+    key log
+    pattern /\[error\]|\| error \|/i
+  </regexp>
+</filter>
+
+<match docker.**>
+  @type copy
+
+  # Write errors to file
+  <store>
+    @type file
+    path /fluentd/log/docker-error
+  </store>
+
+  # Tag daily errors
+  <store>
+    @type grepcounter
+    count_interval 14400 # = 4 hours
+    input_key log
+    threshold 1
+    add_tag_prefix daily_error
+  </store>
+</match>
+
+# Send daily error mail
+#<match daily_error.docker.**>
+#  @type mail
+#  host [Host]
+#  port [Port]
+#  user [User/Email]
+#  password [Password]
+#  from [User/Email]
+#  to [Recipient]
+#  subject 'Error alert fluentd'
+#  message Error occured %s times
+#  message_out_keys count
+#</match>

deployment/docker-compose/init.sh

@@ -0,0 +1,10 @@
+set -e
+source .env
+
+mkdir -p ${POSTGRES_DATA_PATH}
+mkdir -p ${POSTGRES_BACKUP_PATH}
+mkdir -p ${CACHE_PATH}
+mkdir -p ${STATIC_PATH}
+mkdir -p ${FLUENTD_LOG_PATH}
+mkdir -p ${FLUENTD_CUSTOM_CONFIG_PATH}
+cp ./fluentd-custom.config ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config

deployment/docker-compose/update.sh

@@ -0,0 +1,6 @@
+set -e
+
+docker compose pull web
+docker compose stop web
+docker compose run db-backup /backup.sh
+docker compose up --detach --force-recreate web

doc/development.md

@@ -68,7 +68,7 @@ pybabel compile -d project/translations
 ### Build image
 
 ```sh
-docker build -t gsevpt:latest .
+docker build -t danielgrams/gsevpt:latest .
 ```
 
 ### Run container with existing postgres server

project/__init__.py

@@ -14,6 +14,11 @@ from flask_wtf.csrf import CSRFProtect
 
 from project.custom_session_interface import CustomSessionInterface
 
+
+def getenv_bool(name: str, default: str = "False"):
+    return os.getenv(name, default).lower() in ("true", "1", "t")
+
+
 # Create app
 app = Flask(__name__)
 app.config["SQLALCHEMY_DATABASE_URI"] = os.environ["DATABASE_URL"]
@@ -95,15 +100,15 @@ app.config["WTF_CSRF_CHECK_DEFAULT"] = False
 # Mail
 mail_server = os.getenv("MAIL_SERVER")
 
-if mail_server is None:
+if not mail_server:
     app.config["MAIL_SUPPRESS_SEND"] = True
     app.config["MAIL_DEFAULT_SENDER"] = "test@oveda.de"
 else:  # pragma: no cover
     app.config["MAIL_SUPPRESS_SEND"] = False
     app.config["MAIL_SERVER"] = mail_server
     app.config["MAIL_PORT"] = os.getenv("MAIL_PORT")
-    app.config["MAIL_USE_TLS"] = os.getenv("MAIL_USE_TLS", True)
-    app.config["MAIL_USE_SSL"] = os.getenv("MAIL_USE_SSL", False)
+    app.config["MAIL_USE_TLS"] = getenv_bool("MAIL_USE_TLS", "True")
+    app.config["MAIL_USE_SSL"] = getenv_bool("MAIL_USE_SSL", "False")
     app.config["MAIL_USERNAME"] = os.getenv("MAIL_USERNAME")
     app.config["MAIL_PASSWORD"] = os.getenv("MAIL_PASSWORD")
     app.config["MAIL_DEFAULT_SENDER"] = os.getenv("MAIL_DEFAULT_SENDER")

project/views/root.py

@@ -5,7 +5,7 @@ from flask import redirect, render_template, request, send_from_directory, url_f
 from flask_babelex import gettext
 from markupsafe import Markup
 
-from project import app, cache_path, dump_path, robots_txt_file, sitemap_file
+from project import app, cache_path, db, dump_path, robots_txt_file, sitemap_file
 from project.services.admin import upsert_settings
 from project.views.utils import track_analytics
 
@@ -32,6 +32,12 @@ def home():
     )
 
 
+@app.route("/up")
+def up():
+    db.engine.execute("SELECT 1")
+    return "OK"
+
+
 @app.route("/tos")
 def tos():
     title = gettext("Terms of service")