From 11122e8ad67359cc46c1c43fde4828b305d17ce7 Mon Sep 17 00:00:00 2001 From: spinach Date: Mon, 24 Feb 2025 03:00:53 -0500 Subject: [PATCH] separating out each compose to minimal service set and adding script to patch together fragmented envs/gen the sql init script --- .gitignore | 6 ++ .gitmodules | 4 +- Makefile | 4 +- {config => caddy}/Caddyfile | 0 caddy/compose.yml | 23 ++++++ {fish => caddy/fish}/resume.pdf | Bin website => caddy/website | 0 docker-compose.yml | 127 ++---------------------------- env | 2 + init-gen.sh => gen.sh | 0 gitea/compose.yml | 41 ++++++++++ {config => gitea}/env | 0 init.py | 122 +++++++++++++++++++++++++++++ mariadb/compose.yml | 134 ++++++++++++++++++++++++++++++++ mariadb/env | 4 + seafile/compose.yml | 134 ++++++++++++++++++++++++++++++++ seafile/env | 20 +++++ 17 files changed, 498 insertions(+), 123 deletions(-) rename {config => caddy}/Caddyfile (100%) create mode 100644 caddy/compose.yml rename {fish => caddy/fish}/resume.pdf (100%) rename website => caddy/website (100%) create mode 100644 env rename init-gen.sh => gen.sh (100%) create mode 100644 gitea/compose.yml rename {config => gitea}/env (100%) create mode 100755 init.py create mode 100644 mariadb/compose.yml create mode 100644 mariadb/env create mode 100644 seafile/compose.yml create mode 100644 seafile/env diff --git a/.gitignore b/.gitignore index 7afdd19..b011484 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ # blocks private configurations priv/ !priv/readme.md +**/priv/ # blocks local docker volumes volumes/ !volumes/readme.md +**/volumes/ + +# prevents env from being leaked anywhere +**/.env + diff --git a/.gitmodules b/.gitmodules index 98426b3..2fe23b4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "website"] - path = website +[submodule "caddy/website"] + path = caddy/website url = https://git.keegandeppe.com/kdeppe/website diff --git a/Makefile b/Makefile index 88405ea..495d63f 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ fetch: git submodule update --init --recursive init: fetch + mkdir -p priv/ + [ -f "priv/env" ] || cp config/env priv/env mkdir -p volumes/caddy/data volumes/caddy/config mkdir -p volumes/gitea/data volumes/gitea/config - chown 1000:1000 volumes/gitea/data volumes/gitea/config + # chown 1000:1000 volumes/gitea/data volumes/gitea/config mkdir -p volumes/seafile/data volumes/seafile/noti mkdir -p volumes/mariadb/db build: fetch diff --git a/config/Caddyfile b/caddy/Caddyfile similarity index 100% rename from config/Caddyfile rename to caddy/Caddyfile diff --git a/caddy/compose.yml b/caddy/compose.yml new file mode 100644 index 0000000..9ebd12e --- /dev/null +++ b/caddy/compose.yml @@ -0,0 +1,23 @@ +services: + caddy: + image: caddy + restart: unless-stopped + # binding ports directly to avoid needing external network + # eventaully replace with auth container + ports: + - "80:80" + - "443:443" + - "443:443/udp" + networks: + caddy + volumes: + - Caddyfile:/etc/caddy/Caddyfile + - website/public:/srv/keegan + - fish:/srv/fish + - volumes/data:/data + - volumes/config:/config + +networks: + caddy: + external: false + driver: bridge diff --git a/fish/resume.pdf b/caddy/fish/resume.pdf similarity index 100% rename from fish/resume.pdf rename to caddy/fish/resume.pdf diff --git a/website b/caddy/website similarity index 100% rename from website rename to caddy/website diff --git a/docker-compose.yml b/docker-compose.yml index 3fd3d72..7dcbcc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,124 +1,11 @@ -services: - caddy: - image: caddy - restart: unless-stopped - ports: - - "80:80" - - "443:443" - - "443:443/udp" - networks: - auth - caddy - volumes: - - config/Caddyfile:/etc/caddy/Caddyfile - - website/public:/srv/keegan - - fish:/srv/fish - - volumes/caddy/data:/data - - volumes/caddy/config:/config - - mariadb: - image: mariadb:10.11 - container_name: mariadb - env-file: "priv/env" - volumes: - - ./volumes/mariadb/db:/var/lib/mysql - - ./priv/init.sql:/script/init.sql - networks: - - gitea-net - - seafile-net - healthcheck: - test: - [ - "CMD", - "/usr/local/bin/healthcheck.sh", - "--connect", - "--mariadbupgrade", - "--innodb_initialized", - ] - interval: 20s - start_period: 30s - timeout: 5s - retries: 10 - - memcached: - image: memcached:1.6.29 - container_name: seafile-memcached - entrypoint: memcached -m 256 - networks: - - seafile-net - - notification-server: - image: seafileltd/notification-server:12.0-latest - container_name: seafile-notification-server - restart: unless-stopped - volumes: - - volumes/seafile/noti:/shared - environment: - - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db} - - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile} - - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty} - - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db} - - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db} - - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty} - - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false} - - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info} - labels: - caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty} - caddy.@ws.0_header: "Connection *Upgrade*" - caddy.@ws.1_header: "Upgrade websocket" - caddy.0_reverse_proxy: "@ws {{upstreams 8083}}" - caddy.1_handle_path: "/notification*" - caddy.1_handle_path.0_rewrite: "* {uri}" - caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}" - depends_on: - db: - condition: service_healthy - networks: - - seafile-net - - seafile: - image: seafileltd/seafile-mc:12.0-latest - container_name: seafile - volumes: - - /opt/seafile-mysql:/shared - env-file: "priv/env" - environment: - - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY} - depends_on: - db: - condition: service_healthy - memcached: - condition: service_started - networks: - - seafile-net - - caddy - - gitea: - image: gitea/gitea:latest-rootless - restart: unless-stopped - env-file: "priv/env" - restart: always - networks: - - gitea - - caddy - volumes: - - ./volumes/gitea/data:/var/lib/gitea - - ./volumes/gitea/config:/etc/gitea - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - # allows ssh pushing via locally stored keys - - /home/git/.ssh:/data/git/.ssh - ports: - - "127.0.0.1:2222:22" - depends_on: - db: - condition: service_healthy +include: + - gitea/compose.yml + - seafile/compose.yml + - mariadb/compose.yml + - caddy/compose.yml volumes: gitea: - caddy_data: - caddy_config: - networks: gitea-net: @@ -128,7 +15,7 @@ networks: caddy: external: false driver: bridge - auth: - external: true + # auth: + # external: true postgres: external: false diff --git a/env b/env new file mode 100644 index 0000000..0a110b1 --- /dev/null +++ b/env @@ -0,0 +1,2 @@ +## Global Settings +TIME_ZONE="America/New_York" diff --git a/init-gen.sh b/gen.sh similarity index 100% rename from init-gen.sh rename to gen.sh diff --git a/gitea/compose.yml b/gitea/compose.yml new file mode 100644 index 0000000..28bcd97 --- /dev/null +++ b/gitea/compose.yml @@ -0,0 +1,41 @@ +services: + + gitea: + image: gitea/gitea:latest-rootless + restart: unless-stopped + env-file: ".env" + restart: always + networks: + - gitea + - caddy + volumes: + - ./volumes/gitea/data:/var/lib/gitea + - ./volumes/gitea/config:/etc/gitea + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + # allows ssh pushing via locally stored keys + - /home/git/.ssh:/data/git/.ssh + ports: + - "127.0.0.1:2222:22" + depends_on: + db: + condition: service_healthy + +volumes: + gitea: + caddy_data: + caddy_config: + + +networks: + gitea-net: + external: false + seafile-net: + external: false + caddy: + external: false + driver: bridge + auth: + external: true + postgres: + external: false diff --git a/config/env b/gitea/env similarity index 100% rename from config/env rename to gitea/env diff --git a/init.py b/init.py new file mode 100755 index 0000000..3ff4850 --- /dev/null +++ b/init.py @@ -0,0 +1,122 @@ +#!/usr/bin/python3 + +import os +import secrets + +## A simple tool go generate an sql init script + +# qconst sql="init.sql" + +# cd priv + +# seafile_user returns a sql command string to create seafile user and db +# def seafile_user(): + # if 'SEAFILE_MYSQL_DB_USER' in# + +# only allow aA-zZ, 0-9 and below "special" characters in passwords +ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\ + + list(range(48,58))\ + + list(range(64,90))\ + + list(range(97,123)) + +# convert ints back to characters +ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS] + + +def gen_pass(l): + """ + gen_pass returns a password of length l + + :param l: is an integer that represents desired password length + """ + password = "" + for i in range(int(l)): + password += secrets.choice(ALLOWED_CHARS) + + return password + +def prompt_fill(key): + prompt = f"{key} was not set in config, please select an option below:\n" + prompt += f"\t1) leave blank (DEAFAULT)\n" + prompt += f"\t2) generate password\n" + prompt += f"\t3) enter value\n" + + choice = input(prompt) + while choice not in "123": + print(f"error: {choice} unrecognized, please enter a number 1-3 or leave blank") + choice = input(prompt) + + if choice == "1" or choice == "": + return "" + + elif choice == "2": + return gen_pass(32) + + elif choice == "3": + # run until user confirms input + user_input = "" + confirm = None + while confirm not in ["", "Y", "y"]: + user_input = input("please enter desired value:") + confirm = input(f"set {key} to {user_input} (Y/n)?") + + return user_input + + + +def sql_init(password): + print(f"CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';") + print(f"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%';") + +def sql_add_user(username, password, *databases): + print(f"CREATE USER {username} IDENTIFIED BY {password};") + for db in databases: + print(f"CREATE DATABASE {db};") + print(f"GRANT ALL PRIVILEGES ON {db} TO {username};") + +def seafile_init(): + print("setting up seafile...") + + f = open("seafile/env") + c = dict() + for line in f: + # skip comments + if line[0] == "#": + continue + if line.isspace(): + continue + + kv = line.strip('\n').split("=") + + key = kv[0] + val = kv[1].strip('"') + + if val == "": + val = prompt_fill(key) + + c[key] = val + + print(c) + + username = c["SEAFILE_MYSQL_DB_USER"] + password = c["SEAFILE_MYSQL_DB_PASSWORD"] + dbs = ["ccnet_db", "seafile_db", "seahub_db"] + + sql_add_user(username, password, *dbs) + +seafile_init() +# def get_env(): +# print(os.environ['PWD']) + +# printf "testing the env sourcing Seafile host = %s\n", "$SEAFILE_MYSQL_DB_HOST" + +# do not overwrite existing file +# test -f "$sql" || exit 1 +# [ -f "$sql" ] && printf "failed: file exists\n" && exit 1 + +# printf "Generating sql script... " +# touch "$sql" + +# # root +# printf "CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';\n" >> "$sql" +# printf "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%'\n" >> "$sql" diff --git a/mariadb/compose.yml b/mariadb/compose.yml new file mode 100644 index 0000000..3fd3d72 --- /dev/null +++ b/mariadb/compose.yml @@ -0,0 +1,134 @@ +services: + caddy: + image: caddy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + - "443:443/udp" + networks: + auth + caddy + volumes: + - config/Caddyfile:/etc/caddy/Caddyfile + - website/public:/srv/keegan + - fish:/srv/fish + - volumes/caddy/data:/data + - volumes/caddy/config:/config + + mariadb: + image: mariadb:10.11 + container_name: mariadb + env-file: "priv/env" + volumes: + - ./volumes/mariadb/db:/var/lib/mysql + - ./priv/init.sql:/script/init.sql + networks: + - gitea-net + - seafile-net + healthcheck: + test: + [ + "CMD", + "/usr/local/bin/healthcheck.sh", + "--connect", + "--mariadbupgrade", + "--innodb_initialized", + ] + interval: 20s + start_period: 30s + timeout: 5s + retries: 10 + + memcached: + image: memcached:1.6.29 + container_name: seafile-memcached + entrypoint: memcached -m 256 + networks: + - seafile-net + + notification-server: + image: seafileltd/notification-server:12.0-latest + container_name: seafile-notification-server + restart: unless-stopped + volumes: + - volumes/seafile/noti:/shared + environment: + - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db} + - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile} + - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty} + - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db} + - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db} + - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty} + - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false} + - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info} + labels: + caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty} + caddy.@ws.0_header: "Connection *Upgrade*" + caddy.@ws.1_header: "Upgrade websocket" + caddy.0_reverse_proxy: "@ws {{upstreams 8083}}" + caddy.1_handle_path: "/notification*" + caddy.1_handle_path.0_rewrite: "* {uri}" + caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}" + depends_on: + db: + condition: service_healthy + networks: + - seafile-net + + seafile: + image: seafileltd/seafile-mc:12.0-latest + container_name: seafile + volumes: + - /opt/seafile-mysql:/shared + env-file: "priv/env" + environment: + - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY} + depends_on: + db: + condition: service_healthy + memcached: + condition: service_started + networks: + - seafile-net + - caddy + + gitea: + image: gitea/gitea:latest-rootless + restart: unless-stopped + env-file: "priv/env" + restart: always + networks: + - gitea + - caddy + volumes: + - ./volumes/gitea/data:/var/lib/gitea + - ./volumes/gitea/config:/etc/gitea + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + # allows ssh pushing via locally stored keys + - /home/git/.ssh:/data/git/.ssh + ports: + - "127.0.0.1:2222:22" + depends_on: + db: + condition: service_healthy + +volumes: + gitea: + caddy_data: + caddy_config: + + +networks: + gitea-net: + external: false + seafile-net: + external: false + caddy: + external: false + driver: bridge + auth: + external: true + postgres: + external: false diff --git a/mariadb/env b/mariadb/env new file mode 100644 index 0000000..79b7448 --- /dev/null +++ b/mariadb/env @@ -0,0 +1,4 @@ +## Mariadb Settings +MYSQL_ROOT_PASSWORD="" +MYSQL_LOG_CONSOLE="true" +MARIADB_AUTO_UPGRADE=1 diff --git a/seafile/compose.yml b/seafile/compose.yml new file mode 100644 index 0000000..3fd3d72 --- /dev/null +++ b/seafile/compose.yml @@ -0,0 +1,134 @@ +services: + caddy: + image: caddy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + - "443:443/udp" + networks: + auth + caddy + volumes: + - config/Caddyfile:/etc/caddy/Caddyfile + - website/public:/srv/keegan + - fish:/srv/fish + - volumes/caddy/data:/data + - volumes/caddy/config:/config + + mariadb: + image: mariadb:10.11 + container_name: mariadb + env-file: "priv/env" + volumes: + - ./volumes/mariadb/db:/var/lib/mysql + - ./priv/init.sql:/script/init.sql + networks: + - gitea-net + - seafile-net + healthcheck: + test: + [ + "CMD", + "/usr/local/bin/healthcheck.sh", + "--connect", + "--mariadbupgrade", + "--innodb_initialized", + ] + interval: 20s + start_period: 30s + timeout: 5s + retries: 10 + + memcached: + image: memcached:1.6.29 + container_name: seafile-memcached + entrypoint: memcached -m 256 + networks: + - seafile-net + + notification-server: + image: seafileltd/notification-server:12.0-latest + container_name: seafile-notification-server + restart: unless-stopped + volumes: + - volumes/seafile/noti:/shared + environment: + - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db} + - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile} + - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty} + - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db} + - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db} + - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty} + - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false} + - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info} + labels: + caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty} + caddy.@ws.0_header: "Connection *Upgrade*" + caddy.@ws.1_header: "Upgrade websocket" + caddy.0_reverse_proxy: "@ws {{upstreams 8083}}" + caddy.1_handle_path: "/notification*" + caddy.1_handle_path.0_rewrite: "* {uri}" + caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}" + depends_on: + db: + condition: service_healthy + networks: + - seafile-net + + seafile: + image: seafileltd/seafile-mc:12.0-latest + container_name: seafile + volumes: + - /opt/seafile-mysql:/shared + env-file: "priv/env" + environment: + - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY} + depends_on: + db: + condition: service_healthy + memcached: + condition: service_started + networks: + - seafile-net + - caddy + + gitea: + image: gitea/gitea:latest-rootless + restart: unless-stopped + env-file: "priv/env" + restart: always + networks: + - gitea + - caddy + volumes: + - ./volumes/gitea/data:/var/lib/gitea + - ./volumes/gitea/config:/etc/gitea + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + # allows ssh pushing via locally stored keys + - /home/git/.ssh:/data/git/.ssh + ports: + - "127.0.0.1:2222:22" + depends_on: + db: + condition: service_healthy + +volumes: + gitea: + caddy_data: + caddy_config: + + +networks: + gitea-net: + external: false + seafile-net: + external: false + caddy: + external: false + driver: bridge + auth: + external: true + postgres: + external: false diff --git a/seafile/env b/seafile/env new file mode 100644 index 0000000..adc9614 --- /dev/null +++ b/seafile/env @@ -0,0 +1,20 @@ +## Seafile Settings +# COMPOSE_FILE='seafile-server.yml,caddy.yml,seadoc.yml' +# COMPOSE_PATH_SEPARATOR=',' + +SEAFILE_VOLUME=volume/seafile/data +SEAFILE_CADDY_VOLUME=volume/caddy/data + +SEAFILE_MYSQL_DB_HOST="mariadb" +SEAFILE_MYSQL_DB_USER="seafile" +SEAFILE_MYSQL_DB_PASSWORD="" + +SEAFILE_JWT_PRIVATE_KEY="" + +SEAFILE_SERVER_HOSTNAME="" +SEAFILE_SERVER_PROTOCOL="https" + +INIT_SEAFILE_ADMIN_EMAIL="" +INIT_SEAFILE_ADMIN_PASSWORD="" + +ENABLE_SEADOC=false