Skip to content

Commit 6085f4a

Browse files
parrotmacparrotmac
committed
feat: improve development tooling
1 parent 0a62ec6 commit 6085f4a

9 files changed

+518
-230
lines changed

‎.gitignore

+4-1
Original file line numberDiff line numberDiff line change
@@ -136,4 +136,7 @@ creds/
136136

137137
node_modules/
138138
fixtures/
139-
cypress/videos/
139+
cypress/videos/
140+
141+
.env*
142+
.secrets/

‎.tool-versions

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
python 3.10.10
2+
poetry 1.8.2

‎compose.yaml

+8-9
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@ services:
44
image: postgres:16.1-alpine3.19
55
restart: always
66
environment:
7-
POSTGRES_HOST_AUTH_METHOD: trust
8-
# POSTGRES_USER: postgres
9-
# POSTGRES_PASSWORD: postgres
10-
# POSTGRES_DB: postgres
7+
POSTGRES_USER: gary
8+
POSTGRES_PASSWORD: w1shl1st
9+
POSTGRES_DB: gary_db
1110
ports:
12-
- "5900:5432"
13-
redis:
14-
image: redis:alpine3.17
15-
ports:
16-
- "6379:6379"
11+
- "5932:5432"
12+
# valkey:
13+
# image: valkey/valkey:7-alpine3.19
14+
# ports:
15+
# - "6379:6379"

‎development.py

+279
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
import argparse
2+
import gzip
3+
import json
4+
import os
5+
import subprocess
6+
import sys
7+
import tempfile
8+
from pathlib import Path
9+
10+
import boto3
11+
import botocore
12+
from dotenv import load_dotenv
13+
14+
15+
BASE_DIR = Path(__file__).resolve().parent
16+
17+
load_dotenv()
18+
19+
20+
def require_bucket_credentials():
21+
envvars = {
22+
"CAMINO_ARTIFACT_S3_ACCESS_KEY": os.getenv("CAMINO_ARTIFACT_S3_ACCESS_KEY"),
23+
"CAMINO_ARTIFACT_S3_SECRET_KEY": os.getenv("CAMINO_ARTIFACT_S3_SECRET_KEY"),
24+
"CAMINO_ARTIFACT_S3_BUCKET_NAME": os.getenv("CAMINO_ARTIFACT_S3_BUCKET_NAME"),
25+
"CAMINO_ARTIFACT_S3_ENDPOINT": os.getenv("CAMINO_ARTIFACT_S3_ENDPOINT"),
26+
}
27+
is_missing = False
28+
for envvar, value in envvars.items():
29+
if value is None:
30+
is_missing = True
31+
sys.stderr.write(f"Missing required envvar {envvar}\n")
32+
if is_missing:
33+
sys.exit(1)
34+
return envvars
35+
36+
37+
def new_s3_connection():
38+
creds = require_bucket_credentials()
39+
session = boto3.session.Session()
40+
client = session.client(
41+
"s3",
42+
config=botocore.config.Config(s3={"addressing_style": "virtual"}),
43+
region_name="sfo3",
44+
endpoint_url="https://sfo3.digitaloceanspaces.com",
45+
aws_access_key_id=creds["CAMINO_ARTIFACT_S3_ACCESS_KEY"],
46+
aws_secret_access_key=creds["CAMINO_ARTIFACT_S3_SECRET_KEY"],
47+
)
48+
bucket_name = creds["CAMINO_ARTIFACT_S3_BUCKET_NAME"]
49+
return client, bucket_name
50+
51+
def clone_database_from_pgdump_archive(list_sources, from_db, args, database_url):
52+
client, bucket_name = new_s3_connection()
53+
list_kwargs = {
54+
"Bucket": bucket_name,
55+
}
56+
if list_sources or from_db:
57+
list_kwargs["Prefix"] = f"{list_sources or from_db}"
58+
items = client.list_objects(**list_kwargs)["Contents"]
59+
60+
if args._list:
61+
for obj in items:
62+
print("Object: {}".format(obj["Key"]))
63+
if (
64+
args._from
65+
): # To use a specific file, specify the full name of the file instead of just e.g. 'development'
66+
item = items[-1]
67+
item_key = item["Key"]
68+
temp_dir = tempfile.mkdtemp(prefix="db-clone-")
69+
destination_path = Path(temp_dir) / item_key
70+
client.download_file(
71+
Bucket=bucket_name,
72+
Key=item_key,
73+
Filename=str(destination_path),
74+
)
75+
if args.verbose:
76+
print(f"Downloaded {item_key} to {destination_path}")
77+
restore_sql = gzip.decompress(destination_path.read_bytes())
78+
result = subprocess.run(
79+
["/usr/bin/env", "bash", "-c", f"psql {database_url}"],
80+
input=restore_sql,
81+
check=True,
82+
capture_output=True,
83+
)
84+
if args.verbose:
85+
print(result.stdout.decode())
86+
87+
def clone_database_from_live_instance(from_db, database_url):
88+
try:
89+
remote_db_url = json.loads(Path(".secrets/database-credentials.json").read_bytes())["by_key"][from_db]
90+
subprocess.run(
91+
[
92+
"/usr/bin/env",
93+
"bash",
94+
"-c",
95+
f"pg_dump {remote_db_url} | psql {database_url}",
96+
],
97+
check=True,
98+
stdout=sys.stdout,
99+
stderr=sys.stderr,
100+
stdin=sys.stdin,
101+
)
102+
except KeyError:
103+
print(f"Database {from_db} not found in database-credentials.json")
104+
sys.exit(1)
105+
except Exception as e:
106+
print(f"Error reading database-credentials.json: {e}")
107+
sys.exit(1)
108+
109+
secret_files = [
110+
("database-credentials.json", ".secrets/database-credentials.json"),
111+
(".env", ".env"),
112+
]
113+
114+
def main():
115+
parser = argparse.ArgumentParser(description="Development tooling.")
116+
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output.")
117+
component_sp = parser.add_subparsers(dest="component")
118+
119+
secrets_parser = component_sp.add_parser("secrets", help="Manage secrets.")
120+
secrets_action_parser = secrets_parser.add_subparsers(
121+
dest="action", help="Secrets action."
122+
)
123+
secrets_action_parser.add_parser("get", help="Get secrets from 1Password.")
124+
secrets_action_parser.add_parser("upload", help="Upload secrets to 1Password.")
125+
126+
database_parser = component_sp.add_parser("database", help="Manipulate databases.")
127+
database_action_parser = database_parser.add_subparsers(
128+
dest="action", help="Database action."
129+
)
130+
database_cloner = database_action_parser.add_parser(
131+
"clone", help="Clone a database locally."
132+
)
133+
database_cloner.add_argument(
134+
"--from", dest="_from", type=str, help="Which database to clone."
135+
)
136+
database_cloner.add_argument(
137+
"--list", dest="_list", type=str, help="List available clone sources."
138+
)
139+
database_cloner.add_argument(
140+
"--using-archive-strategy",
141+
dest="_using_pgdump_archive",
142+
action="store_true",
143+
help="Restore from a pgdump archive instead of a live instance.",
144+
)
145+
146+
args = parser.parse_args()
147+
148+
match args.component:
149+
case "database":
150+
database_url = os.getenv("DATABASE_URL")
151+
if database_url is None:
152+
sys.stderr.write(f"Missing DATABASE_URL envvar\n")
153+
sys.exit(1)
154+
if args.verbose:
155+
print(f"DATABASE_URL: {database_url}")
156+
match args.action:
157+
case "clone":
158+
if args._using_pgdump_archive:
159+
list_sources = args._list
160+
from_db = args._from
161+
clone_database_from_pgdump_archive(list_sources, from_db, args, database_url)
162+
else:
163+
from_db = args._from
164+
clone_database_from_live_instance(from_db, database_url)
165+
166+
case "secrets":
167+
match args.action:
168+
case "upload":
169+
if input(
170+
"Are you sure you want to upload secrets to 1Password? [y/N] "
171+
).lower() != "y":
172+
print("Aborting.")
173+
sys.exit(1)
174+
175+
for secret_file, secret_path in secret_files:
176+
f = Path(secret_path).read_bytes()
177+
try:
178+
subprocess.run(
179+
[
180+
"/usr/bin/env",
181+
"bash",
182+
"-c",
183+
f"op --account my.1password.com --vault Gary document edit {secret_file}",
184+
],
185+
check=True,
186+
input=f
187+
)
188+
except subprocess.CalledProcessError:
189+
print(f"Failed to get {secret_file}, creating a new document...")
190+
f.seek(0)
191+
subprocess.run(
192+
[
193+
"/usr/bin/env",
194+
"bash",
195+
"-c",
196+
f"op --account my.1password.com --vault Gary document create {secret_file}",
197+
],
198+
check=True,
199+
input=f,
200+
)
201+
print(f"Successfully uploaded {secret_file} to 1Password.")
202+
203+
case "get":
204+
for secret_file, secret_path in secret_files:
205+
leading_dir = os.path.dirname(secret_path)
206+
if leading_dir and not os.path.exists(leading_dir):
207+
os.makedirs(leading_dir)
208+
try:
209+
subprocess.run(
210+
[
211+
"/usr/bin/env",
212+
"bash",
213+
"-c",
214+
f"op --account my.1password.com --vault Gary document get {secret_file} > {secret_path}",
215+
],
216+
check=True,
217+
)
218+
except subprocess.CalledProcessError:
219+
print(f"Failed to get {secret_file}")
220+
sys.exit(1)
221+
222+
# secrets_cache_location = os.path.join(BASE_DIR, ".secrets")
223+
# if not os.path.exists(secrets_cache_location):
224+
# os.makedirs(secrets_cache_location)
225+
# for filename in [
226+
# "database-credentials.json",
227+
# ]:
228+
# # calling directly (not in a sub-shell) to retain current login state
229+
# op_subprocess = subprocess.run(
230+
# [
231+
# "op",
232+
# "--account=my.1password.com",
233+
# "--vault=Gary",
234+
# "document",
235+
# "get",
236+
# filename,
237+
# ],
238+
# check=True,
239+
# stderr=sys.stderr,
240+
# stdout=subprocess.PIPE,
241+
# stdin=sys.stdin,
242+
# )
243+
# op_stdout = op_subprocess.stdout
244+
# if op_stdout.strip() == "":
245+
# raise Exception(
246+
# "op unexpectedly returned an empty document"
247+
# )
248+
249+
# output_fn = Path(f"{secrets_cache_location}/{filename}")
250+
# output_fn.write_bytes(op_stdout)
251+
# for envfile in [
252+
# ".env",
253+
# ]:
254+
# if os.path.exists(envfile):
255+
# if (
256+
# input(
257+
# f"{envfile} already exists, overwrite? [y/N] "
258+
# ).lower()
259+
# != "y"
260+
# ):
261+
# print(f"Skipping {envfile}")
262+
# continue
263+
# subprocess.run(
264+
# [
265+
# "/usr/bin/env",
266+
# "bash",
267+
# "-c",
268+
# f"op --account my.1password.com --vault Gary document get {envfile} > {envfile}",
269+
# ],
270+
# check=True,
271+
# )
272+
case _:
273+
secrets_parser.print_help()
274+
case _:
275+
parser.print_help()
276+
277+
278+
if __name__ == "__main__":
279+
main()

‎gary/settings.py

+3
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ def get_envvar_list(envvar_name, default=[], separator=",", normalize=True):
9191
"django.contrib.auth.middleware.AuthenticationMiddleware",
9292
"django.contrib.messages.middleware.MessageMiddleware",
9393
"django.middleware.clickjacking.XFrameOptionsMiddleware",
94+
"allauth.account.middleware.AccountMiddleware",
9495
]
9596

9697
ROOT_URLCONF = "gary.urls"
@@ -171,6 +172,8 @@ def get_envvar_list(envvar_name, default=[], separator=",", normalize=True):
171172
SOCIALACCOUNT_AUTO_SIGNUP = True
172173
SOCIALACCOUNT_LOGIN_ON_GET = True
173174

175+
LOGIN_REDIRECT_URL = "/"
176+
174177
ACCOUNT_EMAIL_REQUIRED = True
175178
ACCOUNT_AUTHENTICATION_METHOD = 'email'
176179
ACCOUNT_USER_DISPLAY = "gifter.utils.user_display"

‎justfile

+9
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,12 @@
22
db:
33
docker compose down || true
44
docker compose up -d
5+
6+
db-clone environment_name="production":
7+
poetry run python development.py database clone --from {{environment_name}}
8+
9+
secrets:
10+
poetry run python development.py secrets get
11+
12+
upload-secrets:
13+
poetry run python development.py secrets upload

‎manage.py

+3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
import os
44
import sys
55

6+
from dotenv import load_dotenv
7+
8+
load_dotenv()
69

710
def main():
811
"""Run administrative tasks."""

0 commit comments

Comments
 (0)