diff --git a/bin/bitwarden.py b/bin/bitwarden.py new file mode 100755 index 0000000..8c711cd --- /dev/null +++ b/bin/bitwarden.py @@ -0,0 +1,59 @@ +#!/usr/bin/python3 +## helper script to generate bitwarden json objects + +import datetime + +# json_base = '{ +# "folders": [ +# { +# "id": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", +# "name": "Folder Name" +# }, +# ... +# ], +# "items": [ +# { +# "passwordHistory": [ +# { +# "lastUsedDate": "YYYY-MM-00T00:00:00.000Z", +# "password": "passwordValue" +# } +# ], +# "revisionDate": "YYYY-MM-00T00:00:00.000Z", +# "creationDate": "YYYY-MM-00T00:00:00.000Z", +# "deletedDate": null, +# "id": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy", +# "organizationId": null, +# "folderId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", +# "type": 1, +# "reprompt": 0, +# "name": "My Gmail Login", +# "notes": "This is my gmail login for import.", +# "favorite": false, +# "fields": [ +# { +# "name": "custom-field-1", +# "value": "custom-field-value", +# "type": 0 +# }, +# ... +# ], +# "login": { +# "uris": [ +# { +# "match": null, +# "uri": "https://mail.google.com" +# } +# ], +# "username": "myaccount@gmail.com", +# "password": "myaccountpassword", +# "totp": otpauth://totp/my-secret-key +# }, +# "collectionIds": null +# }, +# ... +# ] +# }' + +now = datetime.datetime.now() +print(now) diff --git a/bin/init.py b/bin/init.py new file mode 100755 index 0000000..6907657 --- /dev/null +++ b/bin/init.py @@ -0,0 +1,180 @@ +#!/usr/bin/python3 +## A simple tool to configure services and create sql users/databases + +import os +import subprocess +import secrets +import argparse + + +# only allow aA-zZ, 0-9 and below "special" characters in passwords +ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\ + + list(range(48,58))\ + + list(range(64,90))\ + + list(range(97,123)) + +# convert ints back to characters +ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS] + + +def gen_pass(l): + """gen_pass returns a password of length l from ALLOWED_CHARS""" + + password = "" + for i in range(int(l)): + password += secrets.choice(ALLOWED_CHARS) + + return password + +def get_var(key): + """get_var() gets user input to confirm/replace empty keys""" + + while True: + user_input = input( + f"{key} is empty, enter value (leave blank for password):\n") + + if user_input == "": + prompt = "provided value is empty, choose an option below\n"\ + + "\t1) generate password\n"\ + + "\t2) leave blank\n"\ + + "\t3) back\n" + choice = input(prompt) + while choice not in ["1","2","3"]: + print(f"error: {choice} unrecognized, please enter 1,2 or 3") + choice = input(prompt) + + if choice == "1": + return gen_pass(32) + if choice == "2": + return user_input + if choice == "3": + continue + else: + # run until user confirms input + confirm = input(f"set {key} to {user_input} (y/N)?") + if confirm in ["y", "Y"]: + return user_input + +def gen_sql_user(username, password, *databases): + """gen_sql_user creates the sql queries to add a user and db with perms""" + + sql = f"CREATE USER IF NOT EXISTS {username} IDENTIFIED BY '{password}';\n" + for db in databases: + sql += f"CREATE DATABASE {db};\n" + sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n" + + print(sql) + + return sql + +def gen_sql(sql): + """gen_sql creates an init.sql file to be run by the database on first launch""" + + if sql == "": + return + + subprocess.run(["mkdir", "-p", "mariadb/priv/initdb.d"]) + f = open(f"mariadb/priv/initdb.d/init.sql", 'w') + f.write(sql) + f.close() + +def parse_env(env_file): + """parse_env returns a dictionary of env vars parsed from the base file + + :service: is a string of the service name to parse, must match folder name + """ + f = open(env_file) + c = dict() + for line in f: + # skip comments + if line[0] == "#": + continue + if line.isspace(): + continue + + kv = line.strip('\n').split("=") + + key = kv[0] + val = kv[1].strip('"') + + if val == "": + val = get_var(key) + + c[key] = val + + return c + +def gen_env(kv): + """gen_env takes in a dictionary and returns the formatted env file text as a string + + :kv: is a dictionary of strings mapping env var to value + ex) "HOME": "/home/user/" + """ + env = "" + for key, value in kv.items(): + env += f"{key}=\"{value}\"\n" + + return env + +def config_service(service, force=False): + """config_service processes service env vars to generate private .env file + + :service: is a string of the name of the folder to write the .env to + also requires file service/env to source base config from + :force: is an optional parameter to overwrite existing file + default is False + """ + # setup directories + + subprocess.run(["mkdir", "-p", f"{service}/priv"]) + priv_file = f"{service}/.env" + + + + + # prevent overwrite + if os.path.isfile(priv_file) and not force: + print(f"{priv_file} already exists... skipping") + return parse_env(priv_file) + + c = parse_env(f"{service}/env") + + f = open(priv_file, 'w') + f.write(gen_env(c)) + f.close() + + return c + +def main(): + parser = argparse.ArgumentParser( + description="takes in a service to configure a private .env file and generate sql users/databases") + + parser.add_argument('service', nargs="+", help="service to configure env for") + parser.add_argument('-f', '--force', action="store_true", help="overwrite existing files") + + args = parser.parse_args() + + sql = "" + for service in args.service: + print(f"\nsetting up {service}...") + c = config_service(service, args.force) + # create mariadb users/dbs + if service == "gitea": + username = c["GITEA__database__USER"] + password = c["GITEA__database__PASSWD"] + db = c["GITEA__database__NAME"] + sql += gen_sql_user(username, password, db) + elif service == "seafile": + username = c["SEAFILE_MYSQL_DB_USER"] + password = c["SEAFILE_MYSQL_DB_PASSWORD"] + dbs = ["ccnet_db", "seafile_db", "seahub_db"] + sql += gen_sql_user(username, password, *dbs) + + gen_sql(sql) + # finalize sql + print("running mariadb to initialize users/dbs. Ctrl+c to cancel after database is setup") + subprocess.run(["docker", "compose", "-f", f"mariadb/compose.yml", "up"]) + + print("success") + +main() diff --git a/caddy/compose.yml b/caddy/docker-compose.yml similarity index 94% rename from caddy/compose.yml rename to caddy/docker-compose.yml index 9ebd12e..8067812 100644 --- a/caddy/compose.yml +++ b/caddy/docker-compose.yml @@ -9,7 +9,7 @@ services: - "443:443" - "443:443/udp" networks: - caddy + caddy-net volumes: - Caddyfile:/etc/caddy/Caddyfile - website/public:/srv/keegan @@ -18,6 +18,6 @@ services: - volumes/config:/config networks: - caddy: + caddy-net: external: false driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index fe83150..7811c22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ include: - - gitea/compose.yml - - seafile/compose.yml - - mariadb/compose.yml - - caddy/compose.yml + - gitea/docker-compose.yml + - seafile/docker-compose.yml + - mariadb/docker-compose.yml + - caddy/docker-compose.yml networks: gitea-net: diff --git a/gitea/compose.yml b/gitea/docker-compose.yml similarity index 81% rename from gitea/compose.yml rename to gitea/docker-compose.yml index e7aa005..7b370c5 100644 --- a/gitea/compose.yml +++ b/gitea/docker-compose.yml @@ -3,11 +3,10 @@ services: gitea: image: gitea/gitea:latest-rootless restart: unless-stopped - env-file: ".env" - restart: always + env_file: "priv/env" networks: - - gitea - - caddy + - gitea-net + - caddy-net volumes: - ./volumes/gitea/data:/var/lib/gitea - ./volumes/gitea/config:/etc/gitea @@ -24,3 +23,7 @@ services: networks: gitea-net: external: false + caddy-net: + external: false + driver: bridge + diff --git a/init.py b/init.py deleted file mode 100755 index 83e1c3c..0000000 --- a/init.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/python3 - -import os -import secrets - -## A simple tool go generate an sql init script - -# qconst sql="init.sql" - -# cd priv - -# seafile_user returns a sql command string to create seafile user and db -# def seafile_user(): - # if 'SEAFILE_MYSQL_DB_USER' in# - -# only allow aA-zZ, 0-9 and below "special" characters in passwords -ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\ - + list(range(48,58))\ - + list(range(64,90))\ - + list(range(97,123)) - -# convert ints back to characters -ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS] - - -def gen_pass(l): - """ - gen_pass returns a password of length l - - :param l: is an integer that represents desired password length - """ - password = "" - for i in range(int(l)): - password += secrets.choice(ALLOWED_CHARS) - - return password - -def prompt_fill(key): - prompt = f"{key} was not set in config, please select an option below:\n" - prompt += f"\t1) leave blank (DEAFAULT)\n" - prompt += f"\t2) generate password\n" - prompt += f"\t3) enter value\n" - - choice = input(prompt) - while choice not in "123": - print(f"error: {choice} unrecognized, please enter a number 1-3 or leave blank") - choice = input(prompt) - - if choice == "1" or choice == "": - return "" - - elif choice == "2": - return gen_pass(32) - - elif choice == "3": - # run until user confirms input - user_input = "" - confirm = None - while confirm not in ["", "Y", "y"]: - user_input = input("please enter desired value:") - confirm = input(f"set {key} to {user_input} (Y/n)?") - - return user_input - - - -def sql_init(password): - sql = f"CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';\n" - sql += f"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%';\n" - - return sql - -def sql_add_user(username, password, *databases): - sql = f"CREATE USER {username} IDENTIFIED BY '{password}';\n" - for db in databases: - sql += f"CREATE DATABASE {db};\n" - sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n" - - return sql - -def parse_env(service): - """ - parse_env returns a dictionary of env vars parsed from the base file - - :service: is a string of the service name to parse, must match folder name - """ - print(f"\nsetting up {service}...") - - f = open(f"{service}/env") - c = dict() - for line in f: - # skip comments - if line[0] == "#": - continue - if line.isspace(): - continue - - kv = line.strip('\n').split("=") - - key = kv[0] - val = kv[1].strip('"') - - if val == "": - val = prompt_fill(key) - - c[key] = val - - return c - -def gen_env(kv): - """ - gen_env takes in a dictionary and returns the formatted env file - - :kv: is a dictionary of strings - """ - env = "" - for key, value in kv.items(): - env += f"{key}=\"{value}\"\n" - - return env - -def service_init(service): - priv = f"{service}/.env" - sql = "" - - # prevent overwrite - if os.path.isfile(priv): - print(f"{priv} already exists... skipping") - return sql - - c = parse_env(service) - - if service == "mariadb": - password = c["MYSQL_ROOT_PASSWORD"] - sql = sql_init(password) - elif service == "gitea": - username = c["GITEA__database__USER"] - password = c["GITEA__database__PASSWD"] - db = c["GITEA__database__NAME"] - sql = sql_add_user(username, password, db) - elif service == "seafile": - username = c["SEAFILE_MYSQL_DB_USER"] - password = c["SEAFILE_MYSQL_DB_PASSWORD"] - dbs = ["ccnet_db", "seafile_db", "seahub_db"] - sql = sql_add_user(username, password, *dbs) - else: - print(f"service {service} not regonized!") - os.exit(1) - - env = gen_env(c) - - f = open(priv, 'w') - f.write(env) - f.close() - - return sql - -sql="" -for service in ["mariadb", "gitea", "seafile"]: - sql += service_init(service) - -# attempt to write sql init script based on parameters -if os.path.isfile("priv/init.sql"): - print("priv/init.sql already exists... skipping") - exit - -f = open("priv/init.sql", 'w') -f.write(sql) -f.close() diff --git a/mariadb/compose.yml b/mariadb/docker-compose.yml similarity index 75% rename from mariadb/compose.yml rename to mariadb/docker-compose.yml index e09818b..ab80d17 100644 --- a/mariadb/compose.yml +++ b/mariadb/docker-compose.yml @@ -2,10 +2,10 @@ services: mariadb: image: mariadb:10.11 container_name: mariadb - env-file: ".env" + env_file: "priv/env" volumes: - ./volumes/mariadb/db:/var/lib/mysql - - ./priv/init.sql:/script/init.sql + - ./priv/initdb.d:/config/initdb.d networks: - gitea-net - seafile-net @@ -22,3 +22,9 @@ services: start_period: 30s timeout: 5s retries: 10 + +networks: + gitea-net: + external: false + seafile-net: + external: false diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..0e2ae99 --- /dev/null +++ b/readme.md @@ -0,0 +1,6 @@ +# Fork in the road + +Originally I was going to use a single main database container and build everything off of that. +But you know what, this ends up being far more trouble than it seems to be worth. +If performance becomes an issue, I will revert back to this commit but for now, goodnight my sweet prince. + diff --git a/seafile/compose.yml b/seafile/compose.yml deleted file mode 100644 index 3fd3d72..0000000 --- a/seafile/compose.yml +++ /dev/null @@ -1,134 +0,0 @@ -services: - caddy: - image: caddy - restart: unless-stopped - ports: - - "80:80" - - "443:443" - - "443:443/udp" - networks: - auth - caddy - volumes: - - config/Caddyfile:/etc/caddy/Caddyfile - - website/public:/srv/keegan - - fish:/srv/fish - - volumes/caddy/data:/data - - volumes/caddy/config:/config - - mariadb: - image: mariadb:10.11 - container_name: mariadb - env-file: "priv/env" - volumes: - - ./volumes/mariadb/db:/var/lib/mysql - - ./priv/init.sql:/script/init.sql - networks: - - gitea-net - - seafile-net - healthcheck: - test: - [ - "CMD", - "/usr/local/bin/healthcheck.sh", - "--connect", - "--mariadbupgrade", - "--innodb_initialized", - ] - interval: 20s - start_period: 30s - timeout: 5s - retries: 10 - - memcached: - image: memcached:1.6.29 - container_name: seafile-memcached - entrypoint: memcached -m 256 - networks: - - seafile-net - - notification-server: - image: seafileltd/notification-server:12.0-latest - container_name: seafile-notification-server - restart: unless-stopped - volumes: - - volumes/seafile/noti:/shared - environment: - - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db} - - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile} - - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty} - - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db} - - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db} - - JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty} - - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false} - - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info} - labels: - caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty} - caddy.@ws.0_header: "Connection *Upgrade*" - caddy.@ws.1_header: "Upgrade websocket" - caddy.0_reverse_proxy: "@ws {{upstreams 8083}}" - caddy.1_handle_path: "/notification*" - caddy.1_handle_path.0_rewrite: "* {uri}" - caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}" - depends_on: - db: - condition: service_healthy - networks: - - seafile-net - - seafile: - image: seafileltd/seafile-mc:12.0-latest - container_name: seafile - volumes: - - /opt/seafile-mysql:/shared - env-file: "priv/env" - environment: - - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY} - depends_on: - db: - condition: service_healthy - memcached: - condition: service_started - networks: - - seafile-net - - caddy - - gitea: - image: gitea/gitea:latest-rootless - restart: unless-stopped - env-file: "priv/env" - restart: always - networks: - - gitea - - caddy - volumes: - - ./volumes/gitea/data:/var/lib/gitea - - ./volumes/gitea/config:/etc/gitea - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - # allows ssh pushing via locally stored keys - - /home/git/.ssh:/data/git/.ssh - ports: - - "127.0.0.1:2222:22" - depends_on: - db: - condition: service_healthy - -volumes: - gitea: - caddy_data: - caddy_config: - - -networks: - gitea-net: - external: false - seafile-net: - external: false - caddy: - external: false - driver: bridge - auth: - external: true - postgres: - external: false diff --git a/seafile/docker-compose.yml b/seafile/docker-compose.yml new file mode 100644 index 0000000..ff6fd6f --- /dev/null +++ b/seafile/docker-compose.yml @@ -0,0 +1,62 @@ +services: + memcached: + image: memcached:1.6.29 + container_name: seafile-memcached + entrypoint: memcached -m 256 + networks: + - seafile-net + + # notification-server: + # image: seafileltd/notification-server:12.0-latest + # container_name: seafile-notification-server + # restart: unless-stopped + # volumes: + # - volumes/seafile/noti:/shared + # environment: + # - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db} + # - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile} + # - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty} + # - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db} + # - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db} + # - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY:?Variable is not set or empty} + # - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false} + # - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info} + # labels: + # caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty} + # caddy.@ws.0_header: "Connection *Upgrade*" + # caddy.@ws.1_header: "Upgrade websocket" + # caddy.0_reverse_proxy: "@ws {{upstreams 8083}}" + # caddy.1_handle_path: "/notification*" + # caddy.1_handle_path.0_rewrite: "* {uri}" + # caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}" + # depends_on: + # db: + # condition: service_healthy + # networks: + # - seafile-net + + seafile: + image: seafileltd/seafile-mc:12.0-latest + container_name: seafile + volumes: + # - /opt/seafile-mysql:/shared + - seafile-vol:/shared + env_file: "priv/env" + depends_on: + db: + condition: service_healthy + memcached: + condition: service_started + networks: + - seafile-net + - caddy-net + +volumes: + seafile-vol: + +networks: + seafile-net: + external: false + caddy-net: + external: false + driver: bridge