abandoning central DB

bin/bitwarden.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3 
+## helper script to generate bitwarden json objects
+
+import datetime
+
+# json_base = '{
+#   "folders": [
+#     {
+#       "id": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+#       "name": "Folder Name"
+#     },
+#     ...
+#   ],
+#   "items": [
+#     {
+#     "passwordHistory": [
+#         {
+#           "lastUsedDate": "YYYY-MM-00T00:00:00.000Z",
+#           "password": "passwordValue"
+#         }
+#     ],
+#     "revisionDate": "YYYY-MM-00T00:00:00.000Z",
+#     "creationDate": "YYYY-MM-00T00:00:00.000Z",
+#     "deletedDate": null,    
+#     "id": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy",
+#     "organizationId": null,
+#     "folderId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+#     "type": 1,
+#     "reprompt": 0,
+#     "name": "My Gmail Login",
+#     "notes": "This is my gmail login for import.",
+#     "favorite": false,
+#     "fields": [
+#         {
+#           "name": "custom-field-1",
+#           "value": "custom-field-value",
+#           "type": 0
+#         },
+#         ...
+#       ],
+#       "login": {
+#         "uris": [
+#           {
+#             "match": null,
+#             "uri": "https://mail.google.com"
+#           }
+#         ],
+#         "username": "myaccount@gmail.com",
+#         "password": "myaccountpassword",
+#         "totp": otpauth://totp/my-secret-key
+#       },
+#       "collectionIds": null
+#     },
+#     ...
+#   ]
+# }'
+
+now = datetime.datetime.now()
+print(now)

bin/init.py

@@ -0,0 +1,180 @@
+#!/usr/bin/python3
+## A simple tool to configure services and create sql users/databases
+
+import os
+import subprocess
+import secrets
+import argparse
+
+
+# only allow aA-zZ, 0-9 and below "special" characters in passwords
+ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\
+                + list(range(48,58))\
+                + list(range(64,90))\
+                + list(range(97,123))
+
+# convert ints back to characters
+ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS]
+
+
+def gen_pass(l):
+    """gen_pass returns a password of length l from ALLOWED_CHARS"""
+
+    password = ""
+    for i in range(int(l)):
+        password += secrets.choice(ALLOWED_CHARS)
+    
+    return password
+
+def get_var(key):
+    """get_var() gets user input to confirm/replace empty keys"""
+
+    while True:
+        user_input = input(
+                f"{key} is empty, enter value (leave blank for password):\n")
+
+        if user_input == "":
+            prompt = "provided value is empty, choose an option below\n"\
+                + "\t1) generate password\n"\
+                + "\t2) leave blank\n"\
+                + "\t3) back\n"
+            choice = input(prompt) 
+            while choice not in ["1","2","3"]:
+                print(f"error: {choice} unrecognized, please enter 1,2 or 3")
+                choice = input(prompt)
+
+            if choice == "1":
+                return gen_pass(32)
+            if choice == "2":
+                return user_input
+            if choice == "3":
+                continue
+        else:
+            # run until user confirms input
+            confirm = input(f"set {key} to {user_input} (y/N)?")
+            if confirm in ["y", "Y"]:
+                return user_input
+
+def gen_sql_user(username, password, *databases):
+    """gen_sql_user creates the sql queries to add a user and db with perms"""
+
+    sql = f"CREATE USER IF NOT EXISTS {username} IDENTIFIED BY '{password}';\n"
+    for db in databases:
+        sql += f"CREATE DATABASE {db};\n"
+        sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n"
+
+    print(sql)
+
+    return sql
+
+def gen_sql(sql):
+    """gen_sql creates an init.sql file to be run by the database on first launch"""
+
+    if sql == "":
+        return
+
+    subprocess.run(["mkdir", "-p", "mariadb/priv/initdb.d"])
+    f = open(f"mariadb/priv/initdb.d/init.sql", 'w')
+    f.write(sql)
+    f.close()
+
+def parse_env(env_file):
+    """parse_env returns a dictionary of env vars parsed from the base file
+
+    :service: is a string of the service name to parse, must match folder name
+    """
+    f = open(env_file)
+    c = dict()
+    for line in f:
+        # skip comments
+        if line[0] == "#":
+            continue
+        if line.isspace():
+            continue
+
+        kv = line.strip('\n').split("=")
+
+        key = kv[0]
+        val = kv[1].strip('"')
+
+        if val == "":
+            val = get_var(key)
+
+        c[key] = val
+
+    return c
+
+def gen_env(kv):
+    """gen_env takes in a dictionary and returns the formatted env file text as a string
+
+    :kv: is a dictionary of strings mapping env var to value
+        ex) "HOME": "/home/user/"
+    """
+    env = ""
+    for key, value in kv.items():
+        env += f"{key}=\"{value}\"\n"
+
+    return env
+
+def config_service(service, force=False):
+    """config_service processes service env vars to generate private .env file
+
+    :service: is a string of the name of the folder to write the .env to
+        also requires file service/env to source base config from
+    :force: is an optional parameter to overwrite existing file
+        default is False
+    """
+    # setup directories
+
+    subprocess.run(["mkdir", "-p", f"{service}/priv"])
+    priv_file = f"{service}/.env"
+
+
+
+
+    # prevent overwrite
+    if os.path.isfile(priv_file) and not force:
+        print(f"{priv_file} already exists... skipping")
+        return parse_env(priv_file)
+
+    c = parse_env(f"{service}/env")
+
+    f = open(priv_file, 'w')
+    f.write(gen_env(c))
+    f.close()
+
+    return c
+
+def main():
+    parser = argparse.ArgumentParser(
+            description="takes in a service to configure a private .env file and generate sql users/databases")
+
+    parser.add_argument('service', nargs="+", help="service to configure env for")
+    parser.add_argument('-f', '--force', action="store_true", help="overwrite existing files")
+
+    args = parser.parse_args()
+
+    sql = ""
+    for service in args.service:
+        print(f"\nsetting up {service}...")
+        c = config_service(service, args.force)
+        # create mariadb users/dbs
+        if service == "gitea":
+            username = c["GITEA__database__USER"]
+            password = c["GITEA__database__PASSWD"]
+            db = c["GITEA__database__NAME"]
+            sql += gen_sql_user(username, password, db)
+        elif service == "seafile":
+            username = c["SEAFILE_MYSQL_DB_USER"]
+            password = c["SEAFILE_MYSQL_DB_PASSWORD"]
+            dbs = ["ccnet_db", "seafile_db", "seahub_db"]
+            sql += gen_sql_user(username, password, *dbs)
+
+    gen_sql(sql)
+    # finalize sql
+    print("running mariadb to initialize users/dbs. Ctrl+c to cancel after database is setup")
+    subprocess.run(["docker", "compose", "-f", f"mariadb/compose.yml", "up"])
+
+    print("success")
+
+main()

caddy/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       - "443:443"
       - "443:443/udp"
     networks:
-      caddy
+      caddy-net
     volumes:
       - Caddyfile:/etc/caddy/Caddyfile
       - website/public:/srv/keegan
@@ -18,6 +18,6 @@ services:
       - volumes/config:/config
   
 networks:
-  caddy:
+  caddy-net:
     external: false
     driver: bridge

docker-compose.yml

@@ -1,8 +1,8 @@
 include:
-  - gitea/compose.yml
+  - gitea/docker-compose.yml
-  - seafile/compose.yml
+  - seafile/docker-compose.yml
-  - mariadb/compose.yml
+  - mariadb/docker-compose.yml
-  - caddy/compose.yml
+  - caddy/docker-compose.yml
 
 networks:
   gitea-net:

gitea/docker-compose.yml

@@ -3,11 +3,10 @@ services:
   gitea:
     image: gitea/gitea:latest-rootless
     restart: unless-stopped
-    env-file: ".env"
+    env_file: "priv/env"
-    restart: always
     networks:
-      - gitea
+      - gitea-net
-      - caddy
+      - caddy-net
     volumes:
       - ./volumes/gitea/data:/var/lib/gitea
       - ./volumes/gitea/config:/etc/gitea
@@ -24,3 +23,7 @@ services:
 networks:
   gitea-net:
     external: false
+  caddy-net:
+    external: false
+    driver: bridge
+

init.py

@@ -1,169 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import secrets
-
-## A simple tool go generate an sql init script
-
-# qconst sql="init.sql"
-
-# cd priv
-
-# seafile_user returns a sql command string to create seafile user and db
-# def seafile_user():
-    # if 'SEAFILE_MYSQL_DB_USER' in#  
-
-# only allow aA-zZ, 0-9 and below "special" characters in passwords
-ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\
-                + list(range(48,58))\
-                + list(range(64,90))\
-                + list(range(97,123))
-
-# convert ints back to characters
-ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS]
-
-
-def gen_pass(l):
-    """
-    gen_pass returns a password of length l
-
-    :param l: is an integer that represents desired password length
-    """
-    password = ""
-    for i in range(int(l)):
-        password += secrets.choice(ALLOWED_CHARS)
-    
-    return password
-
-def prompt_fill(key):
-    prompt = f"{key} was not set in config, please select an option below:\n"
-    prompt += f"\t1) leave blank (DEAFAULT)\n"
-    prompt += f"\t2) generate password\n"
-    prompt += f"\t3) enter value\n"
-
-    choice = input(prompt)
-    while choice not in "123":
-        print(f"error: {choice} unrecognized, please enter a number 1-3 or leave blank")
-        choice = input(prompt)
-
-    if choice == "1" or choice == "":
-        return ""
-
-    elif choice == "2":
-        return gen_pass(32)
-
-    elif choice == "3":
-        # run until user confirms input
-        user_input = ""
-        confirm = None
-        while confirm not in ["", "Y", "y"]:
-            user_input = input("please enter desired value:")
-            confirm = input(f"set {key} to {user_input} (Y/n)?")
-
-        return user_input
-
-
-
-def sql_init(password):
-    sql = f"CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';\n"
-    sql += f"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%';\n"
-
-    return sql
-
-def sql_add_user(username, password, *databases):
-    sql = f"CREATE USER {username} IDENTIFIED BY '{password}';\n"
-    for db in databases:
-        sql += f"CREATE DATABASE {db};\n"
-        sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n"
-
-    return sql
-
-def parse_env(service):
-    """
-    parse_env returns a dictionary of env vars parsed from the base file
-
-    :service: is a string of the service name to parse, must match folder name
-    """
-    print(f"\nsetting up {service}...")
-
-    f = open(f"{service}/env")
-    c = dict()
-    for line in f:
-        # skip comments
-        if line[0] == "#":
-            continue
-        if line.isspace():
-            continue
-
-        kv = line.strip('\n').split("=")
-
-        key = kv[0]
-        val = kv[1].strip('"')
-
-        if val == "":
-            val = prompt_fill(key)
-
-        c[key] = val
-
-    return c
-
-def gen_env(kv):
-    """
-    gen_env takes in a dictionary and returns the formatted env file
-
-    :kv: is a dictionary of strings
-    """
-    env = ""
-    for key, value in kv.items():
-        env += f"{key}=\"{value}\"\n"
-
-    return env
-
-def service_init(service):
-    priv = f"{service}/.env"
-    sql = ""
-
-    # prevent overwrite
-    if os.path.isfile(priv):
-        print(f"{priv} already exists... skipping")
-        return sql
-
-    c = parse_env(service)
-
-    if service == "mariadb":
-        password = c["MYSQL_ROOT_PASSWORD"]
-        sql = sql_init(password)
-    elif service == "gitea":
-        username = c["GITEA__database__USER"]
-        password = c["GITEA__database__PASSWD"]
-        db = c["GITEA__database__NAME"]
-        sql = sql_add_user(username, password, db)
-    elif service == "seafile":
-        username = c["SEAFILE_MYSQL_DB_USER"]
-        password = c["SEAFILE_MYSQL_DB_PASSWORD"]
-        dbs = ["ccnet_db", "seafile_db", "seahub_db"]
-        sql = sql_add_user(username, password, *dbs)
-    else:
-        print(f"service {service} not regonized!")
-        os.exit(1)
-
-    env = gen_env(c)
-
-    f = open(priv, 'w')
-    f.write(env)
-    f.close()
-
-    return sql
-
-sql=""
-for service in ["mariadb", "gitea", "seafile"]:
-    sql += service_init(service)
-
-# attempt to write sql init script based on parameters
-if os.path.isfile("priv/init.sql"):
-    print("priv/init.sql already exists... skipping")
-    exit
-
-f = open("priv/init.sql", 'w')
-f.write(sql)
-f.close()

mariadb/docker-compose.yml

@@ -2,10 +2,10 @@ services:
   mariadb:
     image: mariadb:10.11
     container_name: mariadb
-    env-file: ".env"
+    env_file: "priv/env"
     volumes:
       - ./volumes/mariadb/db:/var/lib/mysql
-      - ./priv/init.sql:/script/init.sql
+      - ./priv/initdb.d:/config/initdb.d
     networks:
       - gitea-net
       - seafile-net
@@ -22,3 +22,9 @@ services:
       start_period: 30s
       timeout: 5s
       retries: 10
+
+networks:
+  gitea-net:
+    external: false
+  seafile-net:
+    external: false

readme.md

@@ -0,0 +1,6 @@
+# Fork in the road
+
+Originally I was going to use a single main database container and build everything off of that.
+But you know what, this ends up being far more trouble than it seems to be worth.
+If performance becomes an issue, I will revert back to this commit but for now, goodnight my sweet prince.
+

seafile/compose.yml

@@ -1,134 +0,0 @@
-services:
-  caddy:
-    image: caddy
-    restart: unless-stopped
-    ports:
-      - "80:80"
-      - "443:443"
-      - "443:443/udp"
-    networks:
-      auth
-      caddy
-    volumes:
-      - config/Caddyfile:/etc/caddy/Caddyfile
-      - website/public:/srv/keegan
-      - fish:/srv/fish
-      - volumes/caddy/data:/data
-      - volumes/caddy/config:/config
-
-  mariadb:
-    image: mariadb:10.11
-    container_name: mariadb
-    env-file: "priv/env"
-    volumes:
-      - ./volumes/mariadb/db:/var/lib/mysql
-      - ./priv/init.sql:/script/init.sql
-    networks:
-      - gitea-net
-      - seafile-net
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "/usr/local/bin/healthcheck.sh",
-          "--connect",
-          "--mariadbupgrade",
-          "--innodb_initialized",
-        ]
-      interval: 20s
-      start_period: 30s
-      timeout: 5s
-      retries: 10
-
-  memcached:
-    image: memcached:1.6.29
-    container_name: seafile-memcached
-    entrypoint: memcached -m 256
-    networks:
-      - seafile-net
-
-  notification-server:
-    image: seafileltd/notification-server:12.0-latest
-    container_name: seafile-notification-server
-    restart: unless-stopped
-    volumes:
-      - volumes/seafile/noti:/shared
-    environment:
-      - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
-      - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
-      - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
-      - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
-      - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
-      - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty}
-      - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
-      - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
-    labels:
-      caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
-      caddy.@ws.0_header: "Connection *Upgrade*"
-      caddy.@ws.1_header: "Upgrade websocket"
-      caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
-      caddy.1_handle_path: "/notification*"
-      caddy.1_handle_path.0_rewrite: "* {uri}"
-      caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
-    depends_on:
-      db:
-        condition: service_healthy
-    networks:
-      - seafile-net
-          
-  seafile:
-    image: seafileltd/seafile-mc:12.0-latest
-    container_name: seafile
-    volumes:
-      - /opt/seafile-mysql:/shared
-    env-file: "priv/env"
-    environment:
-      - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY}
-    depends_on:
-      db:
-        condition: service_healthy
-      memcached:
-        condition: service_started
-    networks:
-      - seafile-net
-      - caddy
-
-  gitea:
-    image: gitea/gitea:latest-rootless
-    restart: unless-stopped
-    env-file: "priv/env"
-    restart: always
-    networks:
-      - gitea
-      - caddy
-    volumes:
-      - ./volumes/gitea/data:/var/lib/gitea
-      - ./volumes/gitea/config:/etc/gitea
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-        # allows ssh pushing via locally stored keys
-      - /home/git/.ssh:/data/git/.ssh
-    ports:
-      - "127.0.0.1:2222:22"
-    depends_on:
-      db:
-        condition: service_healthy
-
-volumes:
-  gitea:
-  caddy_data:
-  caddy_config:
-  
-
-networks:
-  gitea-net:
-    external: false
-  seafile-net:
-    external: false
-  caddy:
-    external: false
-    driver: bridge
-  auth:
-    external: true
-  postgres:
-    external: false

seafile/docker-compose.yml

@@ -0,0 +1,62 @@
+services:
+  memcached:
+    image: memcached:1.6.29
+    container_name: seafile-memcached
+    entrypoint: memcached -m 256
+    networks:
+      - seafile-net
+
+  # notification-server:
+  #   image: seafileltd/notification-server:12.0-latest
+  #   container_name: seafile-notification-server
+  #   restart: unless-stopped
+  #   volumes:
+  #     - volumes/seafile/noti:/shared
+  #   environment:
+  #     - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
+  #     - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
+  #     - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
+  #     - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
+  #     - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
+  #     - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY:?Variable is not set or empty}
+  #     - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
+  #     - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
+  #   labels:
+  #     caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
+  #     caddy.@ws.0_header: "Connection *Upgrade*"
+  #     caddy.@ws.1_header: "Upgrade websocket"
+  #     caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
+  #     caddy.1_handle_path: "/notification*"
+  #     caddy.1_handle_path.0_rewrite: "* {uri}"
+  #     caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
+  #   depends_on:
+  #     db:
+  #       condition: service_healthy
+  #   networks:
+  #     - seafile-net
+          
+  seafile:
+    image: seafileltd/seafile-mc:12.0-latest
+    container_name: seafile
+    volumes:
+      # - /opt/seafile-mysql:/shared
+      - seafile-vol:/shared
+    env_file: "priv/env"
+    depends_on:
+      db:
+        condition: service_healthy
+      memcached:
+        condition: service_started
+    networks:
+      - seafile-net
+      - caddy-net
+
+volumes:
+  seafile-vol:
+
+networks:
+  seafile-net:
+    external: false
+  caddy-net:
+    external: false
+    driver: bridge