separating out each compose to minimal service set and adding script to patch together fragmented envs/gen the sql init script

.gitignore

@@ -1,7 +1,13 @@
 # blocks private configurations
 priv/
 !priv/readme.md
+**/priv/
 
 # blocks local docker volumes
 volumes/
 !volumes/readme.md
+**/volumes/
+
+# prevents env from being leaked anywhere
+**/.env
+

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "website"]
+[submodule "caddy/website"]
-	path = website
+	path = caddy/website
 	url = https://git.keegandeppe.com/kdeppe/website

Makefile

@@ -2,9 +2,11 @@ fetch:
 	git submodule update --init --recursive
  
 init: fetch
+	mkdir -p priv/
+	[ -f "priv/env" ] || cp config/env priv/env
 	mkdir -p volumes/caddy/data volumes/caddy/config
 	mkdir -p volumes/gitea/data volumes/gitea/config
-	chown 1000:1000 volumes/gitea/data volumes/gitea/config
+	# chown 1000:1000 volumes/gitea/data volumes/gitea/config
 	mkdir -p volumes/seafile/data volumes/seafile/noti
 	mkdir -p volumes/mariadb/db
 build: fetch

caddy/compose.yml

@@ -0,0 +1,23 @@
+services:
+  caddy:
+    image: caddy
+    restart: unless-stopped
+    # binding ports directly to avoid needing external network
+    # eventaully replace with auth container
+    ports:
+      - "80:80"
+      - "443:443"
+      - "443:443/udp"
+    networks:
+      caddy
+    volumes:
+      - Caddyfile:/etc/caddy/Caddyfile
+      - website/public:/srv/keegan
+      - fish:/srv/fish
+      - volumes/data:/data
+      - volumes/config:/config
+  
+networks:
+  caddy:
+    external: false
+    driver: bridge

docker-compose.yml

@@ -1,124 +1,11 @@
-services:
+include:
-  caddy:
+  - gitea/compose.yml
-    image: caddy
+  - seafile/compose.yml
-    restart: unless-stopped
+  - mariadb/compose.yml
-    ports:
+  - caddy/compose.yml
-      - "80:80"
-      - "443:443"
-      - "443:443/udp"
-    networks:
-      auth
-      caddy
-    volumes:
-      - config/Caddyfile:/etc/caddy/Caddyfile
-      - website/public:/srv/keegan
-      - fish:/srv/fish
-      - volumes/caddy/data:/data
-      - volumes/caddy/config:/config
-
-  mariadb:
-    image: mariadb:10.11
-    container_name: mariadb
-    env-file: "priv/env"
-    volumes:
-      - ./volumes/mariadb/db:/var/lib/mysql
-      - ./priv/init.sql:/script/init.sql
-    networks:
-      - gitea-net
-      - seafile-net
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "/usr/local/bin/healthcheck.sh",
-          "--connect",
-          "--mariadbupgrade",
-          "--innodb_initialized",
-        ]
-      interval: 20s
-      start_period: 30s
-      timeout: 5s
-      retries: 10
-
-  memcached:
-    image: memcached:1.6.29
-    container_name: seafile-memcached
-    entrypoint: memcached -m 256
-    networks:
-      - seafile-net
-
-  notification-server:
-    image: seafileltd/notification-server:12.0-latest
-    container_name: seafile-notification-server
-    restart: unless-stopped
-    volumes:
-      - volumes/seafile/noti:/shared
-    environment:
-      - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
-      - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
-      - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
-      - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
-      - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
-      - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty}
-      - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
-      - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
-    labels:
-      caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
-      caddy.@ws.0_header: "Connection *Upgrade*"
-      caddy.@ws.1_header: "Upgrade websocket"
-      caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
-      caddy.1_handle_path: "/notification*"
-      caddy.1_handle_path.0_rewrite: "* {uri}"
-      caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
-    depends_on:
-      db:
-        condition: service_healthy
-    networks:
-      - seafile-net
-          
-  seafile:
-    image: seafileltd/seafile-mc:12.0-latest
-    container_name: seafile
-    volumes:
-      - /opt/seafile-mysql:/shared
-    env-file: "priv/env"
-    environment:
-      - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY}
-    depends_on:
-      db:
-        condition: service_healthy
-      memcached:
-        condition: service_started
-    networks:
-      - seafile-net
-      - caddy
-
-  gitea:
-    image: gitea/gitea:latest-rootless
-    restart: unless-stopped
-    env-file: "priv/env"
-    restart: always
-    networks:
-      - gitea
-      - caddy
-    volumes:
-      - ./volumes/gitea/data:/var/lib/gitea
-      - ./volumes/gitea/config:/etc/gitea
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-        # allows ssh pushing via locally stored keys
-      - /home/git/.ssh:/data/git/.ssh
-    ports:
-      - "127.0.0.1:2222:22"
-    depends_on:
-      db:
-        condition: service_healthy
 
 volumes:
   gitea:
-  caddy_data:
-  caddy_config:
-  
 
 networks:
   gitea-net:
@@ -128,7 +15,7 @@ networks:
   caddy:
     external: false
     driver: bridge
-  auth:
+  # auth:
-    external: true
+  #   external: true
   postgres:
     external: false

env

@@ -0,0 +1,2 @@
+## Global Settings
+TIME_ZONE="America/New_York"

gitea/compose.yml

@@ -0,0 +1,41 @@
+services:
+
+  gitea:
+    image: gitea/gitea:latest-rootless
+    restart: unless-stopped
+    env-file: ".env"
+    restart: always
+    networks:
+      - gitea
+      - caddy
+    volumes:
+      - ./volumes/gitea/data:/var/lib/gitea
+      - ./volumes/gitea/config:/etc/gitea
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+        # allows ssh pushing via locally stored keys
+      - /home/git/.ssh:/data/git/.ssh
+    ports:
+      - "127.0.0.1:2222:22"
+    depends_on:
+      db:
+        condition: service_healthy
+
+volumes:
+  gitea:
+  caddy_data:
+  caddy_config:
+  
+
+networks:
+  gitea-net:
+    external: false
+  seafile-net:
+    external: false
+  caddy:
+    external: false
+    driver: bridge
+  auth:
+    external: true
+  postgres:
+    external: false

init.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+
+import os
+import secrets
+
+## A simple tool go generate an sql init script
+
+# qconst sql="init.sql"
+
+# cd priv
+
+# seafile_user returns a sql command string to create seafile user and db
+# def seafile_user():
+    # if 'SEAFILE_MYSQL_DB_USER' in#  
+
+# only allow aA-zZ, 0-9 and below "special" characters in passwords
+ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\
+                + list(range(48,58))\
+                + list(range(64,90))\
+                + list(range(97,123))
+
+# convert ints back to characters
+ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS]
+
+
+def gen_pass(l):
+    """
+    gen_pass returns a password of length l
+
+    :param l: is an integer that represents desired password length
+    """
+    password = ""
+    for i in range(int(l)):
+        password += secrets.choice(ALLOWED_CHARS)
+    
+    return password
+
+def prompt_fill(key):
+    prompt = f"{key} was not set in config, please select an option below:\n"
+    prompt += f"\t1) leave blank (DEAFAULT)\n"
+    prompt += f"\t2) generate password\n"
+    prompt += f"\t3) enter value\n"
+
+    choice = input(prompt)
+    while choice not in "123":
+        print(f"error: {choice} unrecognized, please enter a number 1-3 or leave blank")
+        choice = input(prompt)
+
+    if choice == "1" or choice == "":
+        return ""
+
+    elif choice == "2":
+        return gen_pass(32)
+
+    elif choice == "3":
+        # run until user confirms input
+        user_input = ""
+        confirm = None
+        while confirm not in ["", "Y", "y"]:
+            user_input = input("please enter desired value:")
+            confirm = input(f"set {key} to {user_input} (Y/n)?")
+
+        return user_input
+
+
+
+def sql_init(password):
+    print(f"CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';")
+    print(f"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%';")
+
+def sql_add_user(username, password, *databases):
+    print(f"CREATE USER {username} IDENTIFIED BY {password};")
+    for db in databases:
+        print(f"CREATE DATABASE {db};")
+        print(f"GRANT ALL PRIVILEGES ON {db} TO {username};")
+
+def seafile_init():
+    print("setting up seafile...")
+
+    f = open("seafile/env")
+    c = dict()
+    for line in f:
+        # skip comments
+        if line[0] == "#":
+            continue
+        if line.isspace():
+            continue
+
+        kv = line.strip('\n').split("=")
+
+        key = kv[0]
+        val = kv[1].strip('"')
+
+        if val == "":
+            val = prompt_fill(key)
+
+        c[key] = val
+
+    print(c)
+
+    username = c["SEAFILE_MYSQL_DB_USER"]
+    password = c["SEAFILE_MYSQL_DB_PASSWORD"]
+    dbs = ["ccnet_db", "seafile_db", "seahub_db"]
+
+    sql_add_user(username, password, *dbs)
+
+seafile_init()
+# def get_env():
+#     print(os.environ['PWD'])
+    
+# printf "testing the env sourcing Seafile host = %s\n", "$SEAFILE_MYSQL_DB_HOST"
+
+# do not overwrite existing file
+# test -f "$sql" || exit 1 
+# [ -f "$sql" ] && printf "failed: file exists\n" && exit 1
+
+# printf "Generating sql script... "
+# touch "$sql"
+
+# # root
+# printf "CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';\n" >> "$sql"
+# printf "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%'\n" >> "$sql"

mariadb/compose.yml

@@ -0,0 +1,134 @@
+services:
+  caddy:
+    image: caddy
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+      - "443:443/udp"
+    networks:
+      auth
+      caddy
+    volumes:
+      - config/Caddyfile:/etc/caddy/Caddyfile
+      - website/public:/srv/keegan
+      - fish:/srv/fish
+      - volumes/caddy/data:/data
+      - volumes/caddy/config:/config
+
+  mariadb:
+    image: mariadb:10.11
+    container_name: mariadb
+    env-file: "priv/env"
+    volumes:
+      - ./volumes/mariadb/db:/var/lib/mysql
+      - ./priv/init.sql:/script/init.sql
+    networks:
+      - gitea-net
+      - seafile-net
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "/usr/local/bin/healthcheck.sh",
+          "--connect",
+          "--mariadbupgrade",
+          "--innodb_initialized",
+        ]
+      interval: 20s
+      start_period: 30s
+      timeout: 5s
+      retries: 10
+
+  memcached:
+    image: memcached:1.6.29
+    container_name: seafile-memcached
+    entrypoint: memcached -m 256
+    networks:
+      - seafile-net
+
+  notification-server:
+    image: seafileltd/notification-server:12.0-latest
+    container_name: seafile-notification-server
+    restart: unless-stopped
+    volumes:
+      - volumes/seafile/noti:/shared
+    environment:
+      - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
+      - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
+      - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
+      - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
+      - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
+      - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty}
+      - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
+      - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
+    labels:
+      caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
+      caddy.@ws.0_header: "Connection *Upgrade*"
+      caddy.@ws.1_header: "Upgrade websocket"
+      caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
+      caddy.1_handle_path: "/notification*"
+      caddy.1_handle_path.0_rewrite: "* {uri}"
+      caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - seafile-net
+          
+  seafile:
+    image: seafileltd/seafile-mc:12.0-latest
+    container_name: seafile
+    volumes:
+      - /opt/seafile-mysql:/shared
+    env-file: "priv/env"
+    environment:
+      - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY}
+    depends_on:
+      db:
+        condition: service_healthy
+      memcached:
+        condition: service_started
+    networks:
+      - seafile-net
+      - caddy
+
+  gitea:
+    image: gitea/gitea:latest-rootless
+    restart: unless-stopped
+    env-file: "priv/env"
+    restart: always
+    networks:
+      - gitea
+      - caddy
+    volumes:
+      - ./volumes/gitea/data:/var/lib/gitea
+      - ./volumes/gitea/config:/etc/gitea
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+        # allows ssh pushing via locally stored keys
+      - /home/git/.ssh:/data/git/.ssh
+    ports:
+      - "127.0.0.1:2222:22"
+    depends_on:
+      db:
+        condition: service_healthy
+
+volumes:
+  gitea:
+  caddy_data:
+  caddy_config:
+  
+
+networks:
+  gitea-net:
+    external: false
+  seafile-net:
+    external: false
+  caddy:
+    external: false
+    driver: bridge
+  auth:
+    external: true
+  postgres:
+    external: false

mariadb/env

@@ -0,0 +1,4 @@
+## Mariadb Settings
+MYSQL_ROOT_PASSWORD=""
+MYSQL_LOG_CONSOLE="true"
+MARIADB_AUTO_UPGRADE=1

seafile/compose.yml

@@ -0,0 +1,134 @@
+services:
+  caddy:
+    image: caddy
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+      - "443:443/udp"
+    networks:
+      auth
+      caddy
+    volumes:
+      - config/Caddyfile:/etc/caddy/Caddyfile
+      - website/public:/srv/keegan
+      - fish:/srv/fish
+      - volumes/caddy/data:/data
+      - volumes/caddy/config:/config
+
+  mariadb:
+    image: mariadb:10.11
+    container_name: mariadb
+    env-file: "priv/env"
+    volumes:
+      - ./volumes/mariadb/db:/var/lib/mysql
+      - ./priv/init.sql:/script/init.sql
+    networks:
+      - gitea-net
+      - seafile-net
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "/usr/local/bin/healthcheck.sh",
+          "--connect",
+          "--mariadbupgrade",
+          "--innodb_initialized",
+        ]
+      interval: 20s
+      start_period: 30s
+      timeout: 5s
+      retries: 10
+
+  memcached:
+    image: memcached:1.6.29
+    container_name: seafile-memcached
+    entrypoint: memcached -m 256
+    networks:
+      - seafile-net
+
+  notification-server:
+    image: seafileltd/notification-server:12.0-latest
+    container_name: seafile-notification-server
+    restart: unless-stopped
+    volumes:
+      - volumes/seafile/noti:/shared
+    environment:
+      - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
+      - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
+      - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
+      - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
+      - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
+      - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty}
+      - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
+      - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
+    labels:
+      caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
+      caddy.@ws.0_header: "Connection *Upgrade*"
+      caddy.@ws.1_header: "Upgrade websocket"
+      caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
+      caddy.1_handle_path: "/notification*"
+      caddy.1_handle_path.0_rewrite: "* {uri}"
+      caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - seafile-net
+          
+  seafile:
+    image: seafileltd/seafile-mc:12.0-latest
+    container_name: seafile
+    volumes:
+      - /opt/seafile-mysql:/shared
+    env-file: "priv/env"
+    environment:
+      - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY}
+    depends_on:
+      db:
+        condition: service_healthy
+      memcached:
+        condition: service_started
+    networks:
+      - seafile-net
+      - caddy
+
+  gitea:
+    image: gitea/gitea:latest-rootless
+    restart: unless-stopped
+    env-file: "priv/env"
+    restart: always
+    networks:
+      - gitea
+      - caddy
+    volumes:
+      - ./volumes/gitea/data:/var/lib/gitea
+      - ./volumes/gitea/config:/etc/gitea
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+        # allows ssh pushing via locally stored keys
+      - /home/git/.ssh:/data/git/.ssh
+    ports:
+      - "127.0.0.1:2222:22"
+    depends_on:
+      db:
+        condition: service_healthy
+
+volumes:
+  gitea:
+  caddy_data:
+  caddy_config:
+  
+
+networks:
+  gitea-net:
+    external: false
+  seafile-net:
+    external: false
+  caddy:
+    external: false
+    driver: bridge
+  auth:
+    external: true
+  postgres:
+    external: false

seafile/env

@@ -0,0 +1,20 @@
+## Seafile Settings
+# COMPOSE_FILE='seafile-server.yml,caddy.yml,seadoc.yml'
+# COMPOSE_PATH_SEPARATOR=','
+
+SEAFILE_VOLUME=volume/seafile/data
+SEAFILE_CADDY_VOLUME=volume/caddy/data
+
+SEAFILE_MYSQL_DB_HOST="mariadb"
+SEAFILE_MYSQL_DB_USER="seafile"
+SEAFILE_MYSQL_DB_PASSWORD=""
+
+SEAFILE_JWT_PRIVATE_KEY=""
+
+SEAFILE_SERVER_HOSTNAME=""
+SEAFILE_SERVER_PROTOCOL="https"
+
+INIT_SEAFILE_ADMIN_EMAIL=""
+INIT_SEAFILE_ADMIN_PASSWORD=""
+
+ENABLE_SEADOC=false