kdeppe/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

abandoning central DB

Browse Source
This commit is contained in:
spinach 2025-02-28 16:46:45 -05:00
parent 95750a7120
commit 90a4b48699
10 changed files with 328 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
bin/bitwarden.py Executable file
View File

@ -0,0 +1,59 @@
#!/usr/bin/python3
## helper script to generate bitwarden json objects
import datetime
# json_base = '{
# "folders": [
# {
# "id": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
# "name": "Folder Name"
# },
# ...
# ],
# "items": [
# {
# "passwordHistory": [
# {
# "lastUsedDate": "YYYY-MM-00T00:00:00.000Z",
# "password": "passwordValue"
# }
# ],
# "revisionDate": "YYYY-MM-00T00:00:00.000Z",
# "creationDate": "YYYY-MM-00T00:00:00.000Z",
# "deletedDate": null,
# "id": "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy",
# "organizationId": null,
# "folderId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
# "type": 1,
# "reprompt": 0,
# "name": "My Gmail Login",
# "notes": "This is my gmail login for import.",
# "favorite": false,
# "fields": [
# {
# "name": "custom-field-1",
# "value": "custom-field-value",
# "type": 0
# },
# ...
# ],
# "login": {
# "uris": [
# {
# "match": null,
# "uri": "https://mail.google.com"
# }
# ],
# "username": "myaccount@gmail.com",
# "password": "myaccountpassword",
# "totp": otpauth://totp/my-secret-key
# },
# "collectionIds": null
# },
# ...
# ]
# }'
now = datetime.datetime.now()
print(now)

180
bin/init.py Executable file
View File

@ -0,0 +1,180 @@
#!/usr/bin/python3
## A simple tool to configure services and create sql users/databases
import os
import subprocess
import secrets
import argparse
# only allow aA-zZ, 0-9 and below "special" characters in passwords
ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\
+ list(range(48,58))\
+ list(range(64,90))\
+ list(range(97,123))
# convert ints back to characters
ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS]
def gen_pass(l):
"""gen_pass returns a password of length l from ALLOWED_CHARS"""
password = ""
for i in range(int(l)):
password += secrets.choice(ALLOWED_CHARS)
return password
def get_var(key):
"""get_var() gets user input to confirm/replace empty keys"""
while True:
user_input = input(
f"{key} is empty, enter value (leave blank for password):\n")
if user_input == "":
prompt = "provided value is empty, choose an option below\n"\
+ "\t1) generate password\n"\
+ "\t2) leave blank\n"\
+ "\t3) back\n"
choice = input(prompt)
while choice not in ["1","2","3"]:
print(f"error: {choice} unrecognized, please enter 1,2 or 3")
choice = input(prompt)
if choice == "1":
return gen_pass(32)
if choice == "2":
return user_input
if choice == "3":
continue
else:
# run until user confirms input
confirm = input(f"set {key} to {user_input} (y/N)?")
if confirm in ["y", "Y"]:
return user_input
def gen_sql_user(username, password, *databases):
"""gen_sql_user creates the sql queries to add a user and db with perms"""
sql = f"CREATE USER IF NOT EXISTS {username} IDENTIFIED BY '{password}';\n"
for db in databases:
sql += f"CREATE DATABASE {db};\n"
sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n"
print(sql)
return sql
def gen_sql(sql):
"""gen_sql creates an init.sql file to be run by the database on first launch"""
if sql == "":
return
subprocess.run(["mkdir", "-p", "mariadb/priv/initdb.d"])
f = open(f"mariadb/priv/initdb.d/init.sql", 'w')
f.write(sql)
f.close()
def parse_env(env_file):
"""parse_env returns a dictionary of env vars parsed from the base file
:service: is a string of the service name to parse, must match folder name
"""
f = open(env_file)
c = dict()
for line in f:
# skip comments
if line[0] == "#":
continue
if line.isspace():
continue
kv = line.strip('\n').split("=")
key = kv[0]
val = kv[1].strip('"')
if val == "":
val = get_var(key)
c[key] = val
return c
def gen_env(kv):
"""gen_env takes in a dictionary and returns the formatted env file text as a string
:kv: is a dictionary of strings mapping env var to value
ex) "HOME": "/home/user/"
"""
env = ""
for key, value in kv.items():
env += f"{key}=\"{value}\"\n"
return env
def config_service(service, force=False):
"""config_service processes service env vars to generate private .env file
:service: is a string of the name of the folder to write the .env to
also requires file service/env to source base config from
:force: is an optional parameter to overwrite existing file
default is False
"""
# setup directories
subprocess.run(["mkdir", "-p", f"{service}/priv"])
priv_file = f"{service}/.env"
# prevent overwrite
if os.path.isfile(priv_file) and not force:
print(f"{priv_file} already exists... skipping")
return parse_env(priv_file)
c = parse_env(f"{service}/env")
f = open(priv_file, 'w')
f.write(gen_env(c))
f.close()
return c
def main():
parser = argparse.ArgumentParser(
description="takes in a service to configure a private .env file and generate sql users/databases")
parser.add_argument('service', nargs="+", help="service to configure env for")
parser.add_argument('-f', '--force', action="store_true", help="overwrite existing files")
args = parser.parse_args()
sql = ""
for service in args.service:
print(f"\nsetting up {service}...")
c = config_service(service, args.force)
# create mariadb users/dbs
if service == "gitea":
username = c["GITEA__database__USER"]
password = c["GITEA__database__PASSWD"]
db = c["GITEA__database__NAME"]
sql += gen_sql_user(username, password, db)
elif service == "seafile":
username = c["SEAFILE_MYSQL_DB_USER"]
password = c["SEAFILE_MYSQL_DB_PASSWORD"]
dbs = ["ccnet_db", "seafile_db", "seahub_db"]
sql += gen_sql_user(username, password, *dbs)
gen_sql(sql)
# finalize sql
print("running mariadb to initialize users/dbs. Ctrl+c to cancel after database is setup")
subprocess.run(["docker", "compose", "-f", f"mariadb/compose.yml", "up"])
print("success")
main()

4
caddy/compose.yml → caddy/docker-compose.yml
View File

@ -9,7 +9,7 @@ services:
- "443:443"
- "443:443/udp"
networks:
caddy
caddy-net
volumes:
- Caddyfile:/etc/caddy/Caddyfile
- website/public:/srv/keegan
@ -18,6 +18,6 @@ services:
- volumes/config:/config
networks:
caddy:
caddy-net:
external: false
driver: bridge

8
docker-compose.yml
View File

@ -1,8 +1,8 @@
include:
- gitea/compose.yml
- seafile/compose.yml
- mariadb/compose.yml
- caddy/compose.yml
- gitea/docker-compose.yml
- seafile/docker-compose.yml
- mariadb/docker-compose.yml
- caddy/docker-compose.yml
networks:
gitea-net:

11
gitea/compose.yml → gitea/docker-compose.yml
View File

@ -3,11 +3,10 @@ services:
gitea:
image: gitea/gitea:latest-rootless
restart: unless-stopped
env-file: ".env"
restart: always
env_file: "priv/env"
networks:
- gitea
- caddy
- gitea-net
- caddy-net
volumes:
- ./volumes/gitea/data:/var/lib/gitea
- ./volumes/gitea/config:/etc/gitea
@ -24,3 +23,7 @@ services:
networks:
gitea-net:
external: false
caddy-net:
external: false
driver: bridge

169
init.py
View File

@ -1,169 +0,0 @@
#!/usr/bin/python3
import os
import secrets
## A simple tool go generate an sql init script
# qconst sql="init.sql"
# cd priv
# seafile_user returns a sql command string to create seafile user and db
# def seafile_user():
# if 'SEAFILE_MYSQL_DB_USER' in#
# only allow aA-zZ, 0-9 and below "special" characters in passwords
ALLOWED_CHARS = [ord('!'), ord('@'), ord('#'), ord('%'), ord('&')]\
+ list(range(48,58))\
+ list(range(64,90))\
+ list(range(97,123))
# convert ints back to characters
ALLOWED_CHARS = [chr(c) for c in ALLOWED_CHARS]
def gen_pass(l):
"""
gen_pass returns a password of length l
:param l: is an integer that represents desired password length
"""
password = ""
for i in range(int(l)):
password += secrets.choice(ALLOWED_CHARS)
return password
def prompt_fill(key):
prompt = f"{key} was not set in config, please select an option below:\n"
prompt += f"\t1) leave blank (DEAFAULT)\n"
prompt += f"\t2) generate password\n"
prompt += f"\t3) enter value\n"
choice = input(prompt)
while choice not in "123":
print(f"error: {choice} unrecognized, please enter a number 1-3 or leave blank")
choice = input(prompt)
if choice == "1" or choice == "":
return ""
elif choice == "2":
return gen_pass(32)
elif choice == "3":
# run until user confirms input
user_input = ""
confirm = None
while confirm not in ["", "Y", "y"]:
user_input = input("please enter desired value:")
confirm = input(f"set {key} to {user_input} (Y/n)?")
return user_input
def sql_init(password):
sql = f"CREATE USER 'root'@'localhost' IDENTIFIED BY 'local';\n"
sql += f"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%%';\n"
return sql
def sql_add_user(username, password, *databases):
sql = f"CREATE USER {username} IDENTIFIED BY '{password}';\n"
for db in databases:
sql += f"CREATE DATABASE {db};\n"
sql += f"GRANT ALL PRIVILEGES ON {db} TO {username};\n"
return sql
def parse_env(service):
"""
parse_env returns a dictionary of env vars parsed from the base file
:service: is a string of the service name to parse, must match folder name
"""
print(f"\nsetting up {service}...")
f = open(f"{service}/env")
c = dict()
for line in f:
# skip comments
if line[0] == "#":
continue
if line.isspace():
continue
kv = line.strip('\n').split("=")
key = kv[0]
val = kv[1].strip('"')
if val == "":
val = prompt_fill(key)
c[key] = val
return c
def gen_env(kv):
"""
gen_env takes in a dictionary and returns the formatted env file
:kv: is a dictionary of strings
"""
env = ""
for key, value in kv.items():
env += f"{key}=\"{value}\"\n"
return env
def service_init(service):
priv = f"{service}/.env"
sql = ""
# prevent overwrite
if os.path.isfile(priv):
print(f"{priv} already exists... skipping")
return sql
c = parse_env(service)
if service == "mariadb":
password = c["MYSQL_ROOT_PASSWORD"]
sql = sql_init(password)
elif service == "gitea":
username = c["GITEA__database__USER"]
password = c["GITEA__database__PASSWD"]
db = c["GITEA__database__NAME"]
sql = sql_add_user(username, password, db)
elif service == "seafile":
username = c["SEAFILE_MYSQL_DB_USER"]
password = c["SEAFILE_MYSQL_DB_PASSWORD"]
dbs = ["ccnet_db", "seafile_db", "seahub_db"]
sql = sql_add_user(username, password, *dbs)
else:
print(f"service {service} not regonized!")
os.exit(1)
env = gen_env(c)
f = open(priv, 'w')
f.write(env)
f.close()
return sql
sql=""
for service in ["mariadb", "gitea", "seafile"]:
sql += service_init(service)
# attempt to write sql init script based on parameters
if os.path.isfile("priv/init.sql"):
print("priv/init.sql already exists... skipping")
exit
f = open("priv/init.sql", 'w')
f.write(sql)
f.close()

10
mariadb/compose.yml → mariadb/docker-compose.yml
View File

@ -2,10 +2,10 @@ services:
mariadb:
image: mariadb:10.11
container_name: mariadb
env-file: ".env"
env_file: "priv/env"
volumes:
- ./volumes/mariadb/db:/var/lib/mysql
- ./priv/init.sql:/script/init.sql
- ./priv/initdb.d:/config/initdb.d
networks:
- gitea-net
- seafile-net
@ -22,3 +22,9 @@ services:
start_period: 30s
timeout: 5s
retries: 10
networks:
gitea-net:
external: false
seafile-net:
external: false

6
readme.md Normal file
View File

@ -0,0 +1,6 @@
# Fork in the road
Originally I was going to use a single main database container and build everything off of that.
But you know what, this ends up being far more trouble than it seems to be worth.
If performance becomes an issue, I will revert back to this commit but for now, goodnight my sweet prince.

134
seafile/compose.yml
View File

@ -1,134 +0,0 @@
services:
caddy:
image: caddy
restart: unless-stopped
ports:
- "80:80"
- "443:443"
- "443:443/udp"
networks:
auth
caddy
volumes:
- config/Caddyfile:/etc/caddy/Caddyfile
- website/public:/srv/keegan
- fish:/srv/fish
- volumes/caddy/data:/data
- volumes/caddy/config:/config
mariadb:
image: mariadb:10.11
container_name: mariadb
env-file: "priv/env"
volumes:
- ./volumes/mariadb/db:/var/lib/mysql
- ./priv/init.sql:/script/init.sql
networks:
- gitea-net
- seafile-net
healthcheck:
test:
[
"CMD",
"/usr/local/bin/healthcheck.sh",
"--connect",
"--mariadbupgrade",
"--innodb_initialized",
]
interval: 20s
start_period: 30s
timeout: 5s
retries: 10
memcached:
image: memcached:1.6.29
container_name: seafile-memcached
entrypoint: memcached -m 256
networks:
- seafile-net
notification-server:
image: seafileltd/notification-server:12.0-latest
container_name: seafile-notification-server
restart: unless-stopped
volumes:
- volumes/seafile/noti:/shared
environment:
- SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
- SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
- SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
- SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
- SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
- JWT_PRIVATE_KEY=${JWT_PRIVATE_KEY:?Variable is not set or empty}
- SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
- NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
labels:
caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
caddy.@ws.0_header: "Connection *Upgrade*"
caddy.@ws.1_header: "Upgrade websocket"
caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
caddy.1_handle_path: "/notification*"
caddy.1_handle_path.0_rewrite: "* {uri}"
caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
depends_on:
db:
condition: service_healthy
networks:
- seafile-net
seafile:
image: seafileltd/seafile-mc:12.0-latest
container_name: seafile
volumes:
- /opt/seafile-mysql:/shared
env-file: "priv/env"
environment:
- JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY}
depends_on:
db:
condition: service_healthy
memcached:
condition: service_started
networks:
- seafile-net
- caddy
gitea:
image: gitea/gitea:latest-rootless
restart: unless-stopped
env-file: "priv/env"
restart: always
networks:
- gitea
- caddy
volumes:
- ./volumes/gitea/data:/var/lib/gitea
- ./volumes/gitea/config:/etc/gitea
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
# allows ssh pushing via locally stored keys
- /home/git/.ssh:/data/git/.ssh
ports:
- "127.0.0.1:2222:22"
depends_on:
db:
condition: service_healthy
volumes:
gitea:
caddy_data:
caddy_config:
networks:
gitea-net:
external: false
seafile-net:
external: false
caddy:
external: false
driver: bridge
auth:
external: true
postgres:
external: false

62
seafile/docker-compose.yml Normal file
View File

@ -0,0 +1,62 @@
services:
memcached:
image: memcached:1.6.29
container_name: seafile-memcached
entrypoint: memcached -m 256
networks:
- seafile-net
# notification-server:
# image: seafileltd/notification-server:12.0-latest
# container_name: seafile-notification-server
# restart: unless-stopped
# volumes:
# - volumes/seafile/noti:/shared
# environment:
# - SEAFILE_MYSQL_DB_HOST=${SEAFILE_MYSQL_DB_HOST:-db}
# - SEAFILE_MYSQL_DB_USER=${SEAFILE_MYSQL_DB_USER:-seafile}
# - SEAFILE_MYSQL_DB_PASSWORD=${SEAFILE_MYSQL_DB_PASSWORD:?Variable is not set or empty}
# - SEAFILE_MYSQL_DB_CCNET_DB_NAME=${SEAFILE_MYSQL_DB_CCNET_DB_NAME:-ccnet_db}
# - SEAFILE_MYSQL_DB_SEAFILE_DB_NAME=${SEAFILE_MYSQL_DB_SEAFILE_DB_NAME:-seafile_db}
# - JWT_PRIVATE_KEY=${SEAFILE_JWT_PRIVATE_KEY:?Variable is not set or empty}
# - SEAFILE_LOG_TO_STDOUT=${SEAFILE_LOG_TO_STDOUT:-false}
# - NOTIFICATION_SERVER_LOG_LEVEL=${NOTIFICATION_SERVER_LOG_LEVEL:-info}
# labels:
# caddy: ${SEAFILE_SERVER_PROTOCOL:-http}://${SEAFILE_SERVER_HOSTNAME:?Variable is not set or empty}
# caddy.@ws.0_header: "Connection *Upgrade*"
# caddy.@ws.1_header: "Upgrade websocket"
# caddy.0_reverse_proxy: "@ws {{upstreams 8083}}"
# caddy.1_handle_path: "/notification*"
# caddy.1_handle_path.0_rewrite: "* {uri}"
# caddy.1_handle_path.1_reverse_proxy: "{{upstreams 8083}}"
# depends_on:
# db:
# condition: service_healthy
# networks:
# - seafile-net
seafile:
image: seafileltd/seafile-mc:12.0-latest
container_name: seafile
volumes:
# - /opt/seafile-mysql:/shared
- seafile-vol:/shared
env_file: "priv/env"
depends_on:
db:
condition: service_healthy
memcached:
condition: service_started
networks:
- seafile-net
- caddy-net
volumes:
seafile-vol:
networks:
seafile-net:
external: false
caddy-net:
external: false
driver: bridge