Merge pull request #20 from neutrons/update_python_django_postgresql

Update python django and postgresql

Author: rosswhitfield

Makefile

@@ -1,6 +1,6 @@
 prefix := /var/www/livedata
 app_dir := live_data_server
-DJANGO_COMPATIBLE:=$(shell python -c "import django;t=0 if django.VERSION[1]<9 else 1; print(t)")
+DJANGO_COMPATIBLE:=$(shell python -c "import django;t=0 if django.VERSION[0]<4 else 1; print(t)")
 DJANGO_VERSION:=$(shell python -c "import django;print(django.__version__)")
 
 # command to run docker compose. change this to be what you have installed
@@ -14,13 +14,13 @@ help:
 
 check:  ## Check dependencies
 	@python -c "import django" || echo "\nERROR: Django is not installed: www.djangoproject.com\n"
-	@python -c "import psycopg2" || echo "\nWARNING: psycopg2 is not installed: http://initd.org/psycopg\n"
+	@python -c "import psycopg" || echo "\nWARNING: psycopg is not installed: http://initd.org/psycopg\n"
 	@python -c "import corsheaders" || echo "\nWARNING: django-cors-headers is not installed: https://github.com/ottoyiu/django-cors-headers\n"
 
 ifeq ($(DJANGO_COMPATIBLE),1)
 	@echo "Detected Django $(DJANGO_VERSION)"
 else
-	$(error Detected Django $(DJANGO_VERSION) < 1.9. The web monitor requires at least Django 1.9)
+	$(error Detected Django $(DJANGO_VERSION) < 4. The web monitor requires at least Django 4)
 endif
 
 docker/pruneall: docker/compose/validate  ## stop all containers, then remove all containers, images, networks, and volumes

config/docker-compose.envlocal.yml

@@ -44,8 +44,7 @@ services:
         condition: service_healthy
 
   db:
-    # do not upgrade to version > 9.6.23 unless you also upgrade livedata image
-    image: postgres:9.6.23
+    image: postgres:16
     environment:
       POSTGRES_DB: ${DATABASE_NAME}
       POSTGRES_USER: ${DATABASE_USER}

environment.yml

@@ -2,19 +2,17 @@ name: livedata
 channels:
   - conda-forge
 dependencies:
-  - python=3.7
-  - pip
-  - postgresql=9.5.3
+  - python=3.12
+  - postgresql=16
   - sphinx
   - sphinx_rtd_theme
-  - django=2.1
-  - django-cors-headers
-  - psycopg2
+  - django=4.2
+  - django-cors-headers=4.4
+  - psycopg=3.2
   - gunicorn
   - pytest
   - build
   - versioningit
   - toml
-  - requests<2.31 # can remove this condition once we allow newer versions of openssl
   - pre-commit
   - coverage

src/live_data_server/live_data_server/settings.py

@@ -26,7 +26,7 @@
 DEBUG = bool(os.environ.get("APP_DEBUG", False))
 # CSRF_COOKIE_SECURE = True
 # SESSION_COOKIE_SECURE = True
-CSRF_TRUSTED_ORIGINS = [".ornl.gov", ".sns.gov", "localhost", "127.0.0.1"]
+CSRF_TRUSTED_ORIGINS = ["https://*.ornl.gov", "https://*.sns.gov", "http://localhost", "http://127.0.0.1"]
 
 ALLOWED_HOSTS = [
     "localhost",
@@ -63,7 +63,7 @@
 
 ROOT_URLCONF = "live_data_server.urls"
 
-CORS_ORIGIN_ALLOW_ALL = True
+CORS_ALLOW_ALL_ORIGINS = True
 
 TEMPLATES = [
     {
@@ -89,7 +89,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",  # , 'mysql', 'sqlite3' or 'oracle'.
+        "ENGINE": "django.db.backends.postgresql",  # , 'mysql', 'sqlite3' or 'oracle'.
         "NAME": os.environ.get("DATABASE_NAME"),  # Or path to database file if using sqlite3.
         "USER": os.environ.get("DATABASE_USER"),  # Not used with sqlite3.
         "PASSWORD": os.environ.get("DATABASE_PASS"),  # Not used with sqlite3.

src/live_data_server/plots/urls.py

@@ -1,4 +1,3 @@
-# pylint: disable=invalid-name, line-too-long
 """
 Define url structure
 """

src/live_data_server/plots/view_util.py

@@ -1,4 +1,3 @@
-# pylint: disable=invalid-name, bare-except
 """
 Utility functions to support views.
 """

src/live_data_server/plots/views.py

@@ -1,4 +1,3 @@
-# pylint: disable=unused-argument, invalid-name
 """
 Definition of views
 """

test_client/test_client.py

@@ -1,48 +1,22 @@
-# pylint: disable=invalid-name
 """
 Test client that uploads data
 """
 
 import argparse
 import datetime
 import json
+import os
 import string
-import sys
 
 import numpy as np
 import plotly.graph_objs as go
 import requests
 from plotly.offline import plot
 
-sys.path.insert(0, "../live_data_server")
-
-JSON_DATA = False
-API_USER = "admin"
-API_PWD = "adminadmin"
-INSTRUMENT = "REF_L"
-RUN_NUMBER = 123456
-UPLOAD_URL = "http://127.0.0.1:8888/plots/$instrument/$run_number/upload_plot_data/"
-USER_UPLOAD_URL = "http://127.0.0.1:8888/plots/$user/upload_user_data/"
-
-
-def test_sine_wave(run_number, instrument):
-    """
-    Produce test sine wave plot
-    """
-    now = datetime.datetime.now()
-    x = np.arange(1, now.second)
-    y = np.sin(x)
-    x2 = np.arange(1, now.second)
-    y2 = np.sin(x + 1.0)
-    err = 0.2 * np.ones(len(x))
-    sys.path.append("/opt/postprocessing/postprocessing")
-    from publish_plot import plot1d
-
-    data = [[x, y, err, 2 * err], [x2, y2, err, 2 * err]]
-    data_names = ["data", "offset"]
-    return plot1d(
-        run_number, data_list=data, instrument=instrument, x_title="theta", y_title="sin(theta)", data_names=data_names
-    )
+API_USER = os.environ.get("DJANGO_SUPERUSER_USERNAME")
+API_PWD = os.environ.get("DJANGO_SUPERUSER_PASSWORD")
+UPLOAD_URL = "http://127.0.0.1/plots/$instrument/$run_number/upload_plot_data/"
+USER_UPLOAD_URL = "http://127.0.0.1/plots/$user/upload_user_data/"
 
 
 def get_plot_as_json():
@@ -96,32 +70,23 @@ def get_plot_as_div():
     parser.add_argument("-r", metavar="runid", type=int, help="Run number (int)", dest="runid", required=True)
     parser.add_argument("-d", metavar="description", help="Data description", dest="description", required=False)
     parser.add_argument("-i", metavar="instrument", help="Instrument", dest="instrument", required=True)
-    parser.add_argument("-c", metavar="config", help="Config file", dest="config_file", required=False)
     namespace = parser.parse_args()
-    as_json_data = JSON_DATA if namespace.as_json is None else namespace.as_json
 
     monitor_user = {"username": API_USER, "password": API_PWD}
     url_template = string.Template(UPLOAD_URL)
     url = url_template.substitute(instrument=namespace.instrument, run_number=namespace.runid)
 
-    if as_json_data:
+    if namespace.as_json:
         print("Producing json data")
         files = {"file": json.dumps(get_plot_as_json())}
     else:
         print("Producing html data")
         files = {"file": get_plot_as_div()}
 
-    if namespace.config_file is not None:
-        sys.path.append("/opt/postprocessing/postprocessing")
-        from publish_plot import publish_plot
-
-        request = publish_plot(namespace.instrument, namespace.runid, files=files, config_file=namespace.config_file)
-    else:
-        # request = test_sine_wave(namespace.runid, namespace.instrument)
-        if namespace.description is not None:
-            url_template = string.Template(USER_UPLOAD_URL)
-            url = url_template.substitute(user=namespace.instrument)
-            monitor_user["data_id"] = namespace.description
+    if namespace.description is not None:
+        url_template = string.Template(USER_UPLOAD_URL)
+        url = url_template.substitute(user=namespace.instrument)
+        monitor_user["data_id"] = namespace.description
 
-        request = requests.post(url, data=monitor_user, files=files, verify=False)
+    request = requests.post(url, data=monitor_user, files=files, verify=False)
     print(request.status_code)

tests/test_post_get.py

@@ -3,7 +3,7 @@
 import json
 import os
 
-import psycopg2
+import psycopg
 import requests
 
 TEST_URL = "http://127.0.0.1"
@@ -17,8 +17,8 @@ class TestLiveDataServer:
     @classmethod
     def setup_class(cls):
         """Clean the database before running tests"""
-        conn = psycopg2.connect(
-            database=os.environ.get("DATABASE_NAME"),
+        conn = psycopg.connect(
+            dbname=os.environ.get("DATABASE_NAME"),
             user=os.environ.get("DATABASE_USER"),
             password=os.environ.get("DATABASE_PASS"),
             port=os.environ.get("DATABASE_PORT"),