Merge pull request #29 from neutrons/restructure

Reorganize and modernize repository

Author: glass-ships

.github/workflows/unittest.yml

@@ -41,7 +41,7 @@ jobs:
 
       - name: Start docker containers
         run: |
-          cp ./config/docker-compose.envlocal.yml docker-compose.yml
+          cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml
           docker compose up --build -d
 
       - name: Sleep, wait for containers to start up
@@ -52,10 +52,10 @@ jobs:
 
       - name: Stop the coverage process
         # Stopping the coverage process allows the code coverage to be written to disk
-        run: docker exec live_data_server-livedata-1 /bin/bash -c "pkill coverage"
+        run: docker exec live_data_server-django-1 /bin/bash -c "pkill coverage"
 
       - name: Copy code coverage out of docker container
-        run: docker cp live_data_server-livedata-1:/var/www/livedata/app /tmp/
+        run: docker cp live_data_server-django-1:/var/www/livedata/app /tmp/
 
       - name: Combine and show code coverage
         shell: bash -l {0}

Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /var/www/livedata
 
 COPY docker-entrypoint.sh /usr/bin/
 
-COPY src/live_data_server app
+COPY src app
 RUN mkdir ./static
 
 RUN chmod +x /usr/bin/docker-entrypoint.sh

Makefile

@@ -33,15 +33,18 @@ docker/compose/validate:  ## validate the version of the docker-compose command.
 	@./scripts/docker-compose_validate.sh $(DOCKER_COMPOSE)
 
 docker/compose/local: docker/compose/validate ## compose and start the service locally
-	\cp ./config/docker-compose.envlocal.yml docker-compose.yml
+	\cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml
 	$(DOCKER_COMPOSE) up --build
 
 .PHONY: clean
-clean: ## remove all local compiled Python files
-	rm -f `find . -type f -name '*.py[co]' ` \
-		`find . -type f -name '_version.py'`
+clean: ## remove local files from python installation etc.
+	rm -f `find . -type f -name '*.py[co]' -o -name '_version.py'` \
 	rm -rf `find . -name __pycache__ -o -name "*.egg-info"` \
-		.ruff_cache .pytest_cache
+		.ruff_cache \
+		.pytest_cache \
+		.coverage htmlcov \
+		build dist \
+		# docker-compose.yml
 
 .PHONY: check
 .PHONY: first_install

README.md

@@ -59,7 +59,7 @@ Developer documentation at <https://livedata-ornl.readthedocs.io/en/latest/>
       make docker/compose/local
       ```
 
-      This command will copy `config/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services.
+      This command will copy `deploy-config/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services.
 
       Type `make help` to learn about other macros available as make targets.
       For instance, `make docker/pruneall` will stop all containers, then remove all containers, images, networks, and volumes.

config/docker-compose.envlocal.yml renamed to deploy-config/docker-compose.envlocal.yml

@@ -7,12 +7,12 @@ services:
       - "443:443"
     volumes:
       - web-static:/var/www/livedata/static
-      - ./config/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf
+      - ./deploy-config/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf
     depends_on:
-      livedata:
+      django:
         condition: service_healthy
 
-  livedata:
+  django:
     build:
       context: .
       dockerfile: Dockerfile

config/nginx/docker-entrypoint.sh renamed to deploy-config/nginx/docker-entrypoint.sh

@@ -1,3 +1,7 @@
+upstream livedata {
+  server django:8000;
+}
+
 server {
   listen 80 default_server;
   listen [::]:80 default_server ipv6only=on;
@@ -6,7 +10,7 @@ server {
   client_max_body_size 0;
 
   location / {
-    proxy_pass                          http://livedata:8000;
+    proxy_pass                          http://livedata;
     proxy_set_header  Host              $http_host;
     proxy_set_header  X-Real-IP         $remote_addr;
     proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;

config/nginx/envlocal.conf renamed to deploy-config/nginx/envlocal.conf

@@ -24,4 +24,4 @@ python manage.py createcachetable webcache
 python manage.py ensure_adminuser --username=${DJANGO_SUPERUSER_USERNAME} --email='[email protected]' --password=${DJANGO_SUPERUSER_PASSWORD}
 
 # run application
-${COVERAGE_RUN} gunicorn live_data_server.wsgi:application -w 2 -b :8000 --reload
+${COVERAGE_RUN} gunicorn config.wsgi:application -w 2 -b :8000 --reload

docker-entrypoint.sh

@@ -54,7 +54,7 @@ After the secrets are set, you can start the server with:
 
   make docker/compose/local
 
-This command will copy ``config/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services.
+This command will copy ``deploy-config/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services.
 
 | Run ``make help`` to learn about other macros available as make targets.
 | For instance, ``make docker/pruneall`` will stop all containers, then remove all containers, images, networks, and volumes.

docs/developer/config_for_local_use.rst

@@ -24,9 +24,9 @@ The logs indicate a problem with the certificate files.
 An additional test is to substitute the
 `nginx.conf file for the testing environment <https://code.ornl.gov/sns-hfir-scse/deployments/livedata-deploy/-/blob/main/test/nginx.conf?ref_type=heads>`_
 with the
-`local environment one <https://github.com/neutrons/live_data_server/blob/b903a26232dcbccf9f974d522c6113094a0689f0/config/nginx/envlocal.conf>`_,
+`local environment one <https://github.com/neutrons/live_data_server/blob/next/deploy-config/nginx/envlocal.conf>`_,
 which does not contain SSL certificates. Don't forget to change
-`the server name <https://github.com/neutrons/live_data_server/blob/b903a26232dcbccf9f974d522c6113094a0689f0/config/nginx/envlocal.conf#L4>`_
+`the server name <https://github.com/neutrons/live_data_server/blob/next/deploy-config/nginx/envlocal.conf#L4>`_
 from `"localhost"` to `"testfixture02-test.ornl.gov"`.
 Redeploy after this. If the  http://testfixture02-test.ornl.gov/admin (notice the `http` instead of `https`) app is
 served now, then it's a problem of the secure connection.

docs/developer/troubleshoot/unresponsive.rst

@@ -5,25 +5,25 @@ Updating Data Models
 | There may be times when you need to update the data models used by Django.
 | This can be done by following these steps:
 
-#. Make the necessary changes to the models in ``src/live_data_server/plots/models.py``.
+#. Make the necessary changes to the models in ``src/apps/plots/models.py``.
 #. Generate the Django migration file(s):
 
    .. code-block:: bash
 
-      cd src/live_data_server
+      cd src
       python manage.py makemigrations
 
-The migration(s) will be created in the ``src/live_data_server/plots/migrations/`` directory.
+The migration(s) will be created in the ``src/apps/plots/migrations/`` directory.
 First check the migration(s) to ensure they are correct. If they are, apply the migration(s):
 
-From within the live_data_server Docker container:
+From within the Django app Docker container:
 
 .. code-block:: bash
 
    python manage.py migrate
 
    # or if you are not in the container
-   docker exec -i live_data_server-livedata-1 bash -ic '
+   docker exec -i live_data_server-django-1 bash -ic '
       conda activate livedata
       cd app
       python manage.py migrate

docs/developer/updating_data_models.rst

@@ -1,4 +1,4 @@
 [run]
-source=live_data_server,plots
+source=apps,config
 parallel=True
 relative_files=True

src/live_data_server/.coveragerc renamed to src/.coveragerc

@@ -1,5 +1,5 @@
+from apps.plots.models import DataRun, Instrument, PlotData
 from django.contrib import admin
-from plots.models import DataRun, Instrument, PlotData
 
 
 class PlotDataAdmin(admin.ModelAdmin):

src/live_data_server/__init__.py renamed to src/__init__.py

@@ -2,4 +2,4 @@
 
 
 class PlotsConfig(AppConfig):
-    name = "plots"
+    name = "apps.plots"

src/live_data_server/plots/__init__.py renamed to src/apps/__init__.py

@@ -1,6 +1,6 @@
+from apps.plots.models import DataRun
 from django.core.management.base import BaseCommand
 from django.utils import timezone
-from plots.models import DataRun
 
 
 class Command(BaseCommand):

src/live_data_server/plots/management/__init__.py renamed to src/apps/plots/__init__.py

@@ -16,5 +16,4 @@
     ),
     re_path(r"^(?P<user>[\w]+)/upload_user_data/$", views.upload_user_data, name="upload_user_data"),
     re_path(r"^(?P<instrument>[\w]+)/list/$", views.get_data_list, name="get_data_list"),
-    # re_path(r"^(?P<instrument>[\w]+)/list_extra/$", views.get_data_list, name="get_data_list"),
 ]

src/live_data_server/plots/admin.py renamed to src/apps/plots/admin.py

@@ -8,10 +8,10 @@
 from datetime import datetime
 from typing import Optional
 
+from apps.plots.models import DataRun, Instrument, PlotData
 from django.conf import settings
 from django.http import HttpResponse
 from django.utils import timezone
-from plots.models import DataRun, Instrument, PlotData
 
 
 def generate_key(instrument, run_id):

src/live_data_server/plots/apps.py renamed to src/apps/plots/apps.py

@@ -6,14 +6,14 @@
 import logging
 from datetime import timedelta
 
+from apps.plots.models import DataRun, Instrument, PlotData
 from django.conf import settings
 from django.contrib.auth import authenticate, login
 from django.http import HttpResponse, HttpResponseNotFound, JsonResponse
 from django.shortcuts import get_object_or_404
 from django.utils import dateformat, timezone
 from django.views.decorators.cache import cache_page
 from django.views.decorators.csrf import csrf_exempt
-from plots.models import DataRun, Instrument, PlotData
 
 from . import view_util
 

src/live_data_server/plots/management/commands/__init__.py renamed to src/apps/plots/management/__init__.py

@@ -40,7 +40,7 @@
 # Application definition
 
 INSTALLED_APPS = [
-    "plots.apps.PlotsConfig",
+    "apps.plots",
     "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -61,7 +61,7 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
-ROOT_URLCONF = "live_data_server.urls"
+ROOT_URLCONF = "config.urls"
 
 CORS_ALLOW_ALL_ORIGINS = True
 
@@ -81,7 +81,7 @@
     },
 ]
 
-WSGI_APPLICATION = "live_data_server.wsgi.application"
+WSGI_APPLICATION = "config.wsgi.application"
 
 
 # Database

src/live_data_server/plots/migrations/__init__.py renamed to src/apps/plots/management/commands/__init__.py

@@ -23,5 +23,5 @@
 urlpatterns = [
     re_path(r"^admin/", admin.site.urls),
     re_path(r"^$", RedirectView.as_view(url="/plots/")),
-    re_path(r"^plots/", include("plots.urls", namespace="plots")),
+    re_path(r"^plots/", include("apps.plots.urls", namespace="plots")),
 ]

src/live_data_server/plots/management/commands/ensure_adminuser.py renamed to src/apps/plots/management/commands/ensure_adminuser.py

@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "live_data_server.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 
 application = get_wsgi_application()

src/live_data_server/plots/management/commands/purge_expired_data.py renamed to src/apps/plots/management/commands/purge_expired_data.py

@@ -3,7 +3,7 @@
 import sys
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "live_data_server.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 
     from django.core.management import execute_from_command_line
 

src/live_data_server/plots/migrations/0001_initial.py renamed to src/apps/plots/migrations/0001_initial.py

@@ -6,7 +6,7 @@
 import psycopg
 import requests
 
-TEST_URL = "http://127.0.0.1"
+TEST_URL = "http://127.0.0.1:8000"
 HTTP_OK = requests.status_codes.codes["OK"]
 HTTP_UNAUTHORIZED = requests.status_codes.codes["unauthorized"]
 HTTP_NOT_FOUND = requests.status_codes.codes["NOT_FOUND"]
@@ -117,9 +117,8 @@ def test_expiration_user(self, data_server):
 
     def test_deleting_expired(self):
         """Test the purge_expired_data command"""
-        command = "docker exec -i live_data_server-livedata-1 bash -ic"
+        command = "docker exec -i live_data_server-django-1 bash -ic"
         subcommand = "conda activate livedata && cd app && coverage run manage.py purge_expired_data"
-        # subcommand = "conda activate livedata && cd app && python manage.py purge_expired_data"
         output = subprocess.check_output([*command.split(" "), subcommand])
         print(output)
 

src/live_data_server/plots/migrations/0002_datarun_expiration_date.py renamed to src/apps/plots/migrations/0002_datarun_expiration_date.py

@@ -5,7 +5,7 @@
 import psycopg
 import requests
 
-TEST_URL = "http://127.0.0.1"
+TEST_URL = "http://127.0.0.1:8000"
 HTTP_OK = requests.status_codes.codes["OK"]
 HTTP_UNAUTHORIZED = requests.status_codes.codes["unauthorized"]
 HTTP_NOT_FOUND = requests.status_codes.codes["NOT_FOUND"]