Sync dev staging mar 19 (#653)

* Harvest dynamic configs pt. 2 (#635)

* Update ckanext-s3filestore branch

* Fix typo

* Update layer manager to use wri release

* Fix Dockerfile

* [ODP-235] Allow private and hidden teams (#537)

* update teams create and edit form

* update organization list api

* update schema

* update api

* remove extras

* fix type errors

* update user_list_wri to support private teams

* fix api call

* remove chained action

* add visibility

* fix breaking changes

* update env

* update env plugin

* fix test

* update test

* add visibility to org during dataset edit

* add test for private teams

* remove logs

* update private team functionality

* Merge branch 'priveateams' of https://github.com/wri/wri-odp into priveateams

* Merge branch 'priveateams' of https://github.com/wri/wri-odp into priveateams

* resolve conflict

* undo changes

* update organization patch and create

* add private attribute to fetched orgs

* update organization_list sql logic

* improve team visbility

* improve error message

* update private team test

* update failing test

* update test

* update settings

* update test

* update test

* update test

* update test

* update test

* update test

* update test

* update docker compose

* add tooltip

* update test

* add padlock to private teams

* add back auth plugin

* Harvest config feb 03 2025 (#637)

* Fix variable replacement in harvest worker configs and switch from /tmp path to /srv/app

* Add whoami output to start_ckan.sh

* [download_event.py] Add missing closing parenthesis

* Set supervisor home by variable

* Fix harvest config permissions

* Use CKAN_IMAGE variable instead of local image name

* Handle sed supervisor changes as root; Add start_ckan sub-script

* Revert sub-script; Always use user ckan for supervisor

* Fix sed commands

* Use root user with ckan home for supervisord

* Revert "Harvest config feb 03 2025 (#637)"

This reverts commit e68155a.

* Harvest dynamic configs pt. 3 (#638)

* Add supercronic for harvest run command (#640)

* Fix alignment

* Fix build

* Fix build

* [ODP-414] add email notification when download-flow fails (#641)

* add email notification when flow fails

* update error notification

* empty

* remove pgclient version

* update libpq

---------

Co-authored-by: Michael Polidori <[email protected]>

* Upgrade to CKAN 2.11 (#632)

* Update setup.py

* Update Members permission to team form + ODP-370 (#642)

* members can not

* update role: Editor can create sub-team and admin moving subteam from public to private should not throw error

* disabled Public option if parent is private

* Trigger CI

---------

Co-authored-by: Luccas Mateus <[email protected]>

* Change setup.py and setup.cfg

* Fix ckanext-wri install (#643)

* Improve resource location design (#646)

* Trigger CI

* [odp-426]: Add edit icon to dashboard entities (#648)

* [odp-426]: Add edit icon to dashboard entities

* update schema

* Add bulk purge collaborator issue patch (#645)

* Increase UWSGI timeouts; Extract URLs from markdown links in learn_more (#649)

* Resource location search improvements (#650)

* add authorized check to group/teams last node (#651)

* Trigger CI

* Rm global from DatafileLocation

* Trigger CI

* enable edit icon for collaborators (#652)

* Fix typo

* Remove dev helm config values and GH Actions file

---------

Co-authored-by: Michael Polidori <[email protected]>
Co-authored-by: Stephen Oni <[email protected]>

Author: 3 people

.github/workflows/main.yml

@@ -112,8 +112,6 @@ jobs:
         run: |
           docker compose -f docker-compose.test.yml --env-file .env.example up --build -d
         working-directory: ./ckan-backend-dev
-      - name: Initialize the Extensions
-        run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Cypress Install and CKAN setup
         if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
@@ -131,8 +129,6 @@ jobs:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:staging-${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example up --build -d frontend
         working-directory: ./ckan-backend-dev
-      - name: Initialize the Extensions
-        run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Print Logs
         if: ${{ env.ENABLE_DEBUG_LOGS == 'true' }}
         env:
@@ -148,6 +144,8 @@ jobs:
           node-version: 18
           runTests: false
           working-directory: ./e2e-tests
+      - name: Initialize the Extensions
+        run: bash ./ckan-backend-dev/ckan/scripts/init-extensions.sh
       - name: Run Integration tests 🧪
         if: ${{ env.ENABLE_INT_TESTS == 'true' }}
         uses: cypress-io/github-action@v6
@@ -195,6 +193,8 @@ jobs:
         working-directory: ./ckan-backend-dev
       - name: Initialize the pending datasets table
         run: docker exec ckan-wri sh -c "ckan -c production.ini pendingdatasetsdb"
+      - name: Initialize the issues table
+        run: docker exec ckan-wri sh -c "ckan -c production.ini issuesdb"
       - name: Run Unit Tests 🧪
         env:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'

ckan-backend-dev/.env.example

@@ -43,7 +43,7 @@ TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
 TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
 
 # CKAN core
-CKAN_VERSION=2.10.0
+CKAN_VERSION=2.11.0
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://ckan-dev:5000
 CKAN_PORT=5000
@@ -84,8 +84,8 @@ DATAPUSHER_REWRITE_RESOURCES=True
 DATAPUSHER_REWRITE_URL=http://ckan-dev:5000
 
 # Extensions
-CKAN__PLUGINS=image_view text_view webpage_view resource_proxy datatables_view datastore datapusher activity s3filestore scheming_datasets scheming_organizations scheming_groups harvest wri wri_harvester auth hierarchy_display hierarchy_form hierarchy_group_form issues envvars
-CKAN__VIEWS__DEFAULT_VIEWS=image_view text_view webpage_view datatables_view
+#CKAN__PLUGINS='text_view image_view webpage_view resource_proxy datatables_view datastore datapusher activity s3filestore scheming_datasets scheming_organizations scheming_groups harvest wri wri_harvester auth hierarchy_display hierarchy_form hierarchy_group_form envvars'
+#CKAN__VIEWS__DEFAULT_VIEWS="image_view text_view webpage_view datatables_view"
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
 CKAN__HARVEST__MQ__PORT=6379

ckan-backend-dev/ckan/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM viderum/ckan:v2.10.3-alpine-3.18
+FROM viderum/ckan:v2.11.0
 
 # Set up environment variables
 ENV APP_DIR=/srv/app
@@ -22,16 +22,22 @@ ENV SRC_EXTENSIONS_DIR=${APP_DIR}/src_extensions
 # to get them mounted in this image at runtime
 USER root
 RUN pip3 install --upgrade setuptools==70.0.0
-# Echo the version of setuptools to check if it is installed correctly
-RUN pip3 install -e 'git+https://github.com/datopian/ckanext-scheming.git@0983735ca163f906693c46f5315ee5abf68d2742#egg=ckanext-scheming' && \
-    pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-scheming/ckan-2.10/test-requirements.txt' && \
+
+RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@5ce30cf2856aee97fa7b49de46847f8119dad355#egg=ckanext-scheming' && \
+    pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-scheming/5ce30cf2856aee97fa7b49de46847f8119dad355/test-requirements.txt' && \
     pip3 install -e 'git+https://github.com/datopian/ckanext-s3filestore.git@wri/cost-splitting-orgs#egg=ckanext-s3filestore' && \
     pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-s3filestore/wri/cost-splitting-orgs/requirements.txt' && \
-    pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-s3filestore/wri/cost-splitting-orgs/dev-requirements.txt' && \
+    #pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-s3filestore/wri/cost-splitting-orgs/dev-requirements.txt' && \
+    # Use latest versions for now due to build errors. TODO: Update to specific newer versions (hopefully these are compatible)
+    pip3 install moto && \
+    pip3 install ckanapi && \
+    pip3 install httpretty && \
     pip3 install -e 'git+https://github.com/datopian/ckanext-auth.git@okta#egg=ckanext-auth' && \
     pip3 install -r 'https://raw.githubusercontent.com/datopian/ckanext-auth/okta/requirements.txt' && \
-    pip3 install -e 'git+https://github.com/ckan/ckanext-hierarchy.git@master#egg=ckanext-hierarchy' && \
+    pip3 install -e 'git+https://github.com/datopian/ckanext-hierarchy.git@wri#egg=ckanext-hierarchy' && \
     pip3 install -e 'git+https://github.com/datopian/[email protected]#egg=ckanext-issues' && \
+    # Required by ckanext-issues, but it's no longer included in any of the requirements files or the subdependencies
+    pip3 install mock && \
     pip3 install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-harvest' && \
     pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-harvest/refs/tags/v1.6.0/requirements.txt'
 
@@ -106,6 +112,9 @@ RUN chown ckan:ckan ${APP_DIR}/start_ckan_development.sh
 
 RUN apk --no-cache add openssl
 
+RUN cat ${APP_DIR}/src/ckan/requirements.txt
+RUN pip3 install -r /srv/app/src/ckan/requirements.txt
+
 RUN openssl genpkey -algorithm RSA -out ${APP_DIR}/jwtRS256.key && \
     openssl rsa -in ${APP_DIR}/jwtRS256.key -pubout -outform PEM -out ${APP_DIR}/jwtRS256.key.pub && \
     chown ckan:ckan ${APP_DIR}/jwtRS256.key && \
@@ -120,6 +129,7 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/
 
 RUN ckan config-tool ${CKAN_INI} "ckan.cors.origin_allow_all = True"
 RUN ckan config-tool ${CKAN_INI} "ckan.root_path = /private-admin/{{LANG}}"
+RUN ckan config-tool ${CKAN_INI} -s app:main "ckan.datapusher.api_token = xxxx-xxxx-xxxx-xxxx"
 
 COPY setup/who.ini ${APP_DIR}/src/ckan/who.ini
 

ckan-backend-dev/ckan/patches/ckan/02_tests_reset_db.patch

@@ -1,18 +1,16 @@
-diff --git a/ckan/ckan/model/__init__.py b/ckan/ckan/model/__init__.py
---- ckan/ckan/model/__init__.py
-+++ ckan/ckan/model/__init__.py
-@@ -219,8 +219,13 @@
-         with warnings.catch_warnings():
+diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py
+index 595ae3876..a03e50c25 100644
+--- a/ckan/model/__init__.py
++++ b/ckan/model/__init__.py
+@@ -237,6 +237,11 @@ class Repository():
              warnings.filterwarnings('ignore', '.*(reflection|tsvector).*')
-             meta.metadata.reflect()
+             meta.metadata.reflect(engine)
 
 +        tables = meta.metadata.sorted_tables
 +        for table in tables:
 +            if table.name == "spatial_ref_sys":
 +                meta.metadata.remove(table)
 +
-         meta.metadata.drop_all()
-         self.tables_created_and_initialised = False
-         log.info('Database tables dropped')
+         with engine.begin() as conn:
+             meta.metadata.drop_all(conn)
 
-

ckan-backend-dev/ckan/patches/ckan/03_fix_dataset_purge_collaborators.patch

@@ -0,0 +1,19 @@
+diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py
+index dc3ce8ec2..1828effca 100644
+--- a/ckan/logic/action/delete.py
++++ b/ckan/logic/action/delete.py
+@@ -152,6 +152,14 @@ def dataset_purge(context: Context, data_dict: DataDict) -> ActionResult.Dataset
+                 model.PackageRelationship.object_package_id == pkg.id)).all():
+         r.purge()
+ 
++    dataset_collaborators = (
++        model.Session.query(model.PackageMember)
++        .filter(model.PackageMember.package_id == id)
++        .all()
++    )
++    for collaborator in dataset_collaborators:
++        collaborator.delete()
++
+     pkg = model.Package.get(id)
+     assert pkg
+     pkg.purge()

ckan-backend-dev/ckan/scripts/init-extensions.sh

@@ -1,4 +1,3 @@
-
 # Initialize Issues and Notification DB
 docker exec ckan-wri sh -c "ckan -c production.ini issuesdb"
 docker exec ckan-wri sh -c "ckan -c production.ini downloadeventdb"

ckan-backend-dev/ckan/setup/prerun.py.override

@@ -93,20 +93,24 @@ def check_solr_connection(retry=None):
 
 
 def init_db():
+    # Output the contents of the CKAN_INI file
+    print("[prerun] CKAN_INI file contents:")
+    with open(ckan_ini, "r") as f:
+        print(f.read())
 
     db_command = ["ckan", "-c", ckan_ini, "db", "init"]
     print("[prerun] Initializing or upgrading db - start")
     try:
         subprocess.check_output(db_command, stderr=subprocess.STDOUT)
         print("[prerun] Initializing or upgrading db - end")
     except subprocess.CalledProcessError as e:
-        if "OperationalError" in e.output:
-            print(e.output)
+        if "OperationalError" in e.output.decode("utf-8"):
+            print(e.output.decode("utf-8"))
             print("[prerun] Database not ready, waiting a bit before exit...")
             time.sleep(5)
             sys.exit(1)
         else:
-            print(e.output)
+            print(e.output.decode("utf-8"))
             raise e
 
 
@@ -144,13 +148,13 @@ def init_datastore_db():
         print(str(e))
 
     except subprocess.CalledProcessError as e:
-        if "OperationalError" in e.output:
-            print(e.output)
+        if "OperationalError" in e.output.decode("utf-8"):
+            print(e.output.decode("utf-8"))
             print("[prerun] Database not ready, waiting a bit before exit...")
             time.sleep(5)
             sys.exit(1)
         else:
-            print(e.output)
+            print(e.output.decode("utf-8"))
             raise e
     finally:
         cursor.close()

ckan-backend-dev/ckan/setup/start_ckan_development.sh.override

@@ -4,6 +4,7 @@
 echo "Looking for local extensions to install..."
 echo "Extension dir contents:"
 ls -la $SRC_EXTENSIONS_DIR
+
 for i in $SRC_EXTENSIONS_DIR/*
 do
     if [ -d $i ];
@@ -46,7 +47,7 @@ echo "Enabling debug mode"
 ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
 
 # Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
-ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
+#ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
 
 # Set up the Secret key used by Beaker and Flask
 # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
@@ -60,10 +61,6 @@ then
     ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
 fi
 
-# Update the plugins setting in the ini file with the values defined in the env var
-echo "Loading the following plugins: $CKAN__PLUGINS"
-ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
-
 # Update test-core.ini DB, SOLR & Redis settings
 echo "Loading test settings into test-core.ini"
 ckan config-tool $SRC_DIR/ckan/test-core.ini \
@@ -73,13 +70,30 @@ ckan config-tool $SRC_DIR/ckan/test-core.ini \
     "solr_url = $TEST_CKAN_SOLR_URL" \
     "ckan.redis.url = $TEST_CKAN_REDIS_URL"
 
+ckan config-tool $CKAN_INI "ckan.plugins = datastore envvars"
+ckan config-tool $CKAN_INI "ckan.views.default_views ="
+
+DEFAULT_VIEWS=$CKAN__VIEWS__DEFAULT_VIEWS
+PLUGINS=$CKAN__PLUGINS
+
+unset CKAN__PLUGINS
+unset CKAN__VIEWS__DEFAULT_VIEWS
+
 # Run the prerun script to init CKAN and create the default admin user
 python3 prerun.py
 
+export CKAN__VIEWS__DEFAULT_VIEWS="$DEFAULT_VIEWS"
+export CKAN__PLUGINS="$PLUGINS"
+
 unset CKAN__DATAPUSHER__API_TOKEN
 echo "Set up ckan.datapusher.api_token in the CKAN config file"
 ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')"
 
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS"
+
 # Run any startup scripts provided by images extending this one
 if [[ -d "/docker-entrypoint.d" ]]
 then

ckan-backend-dev/docker-compose.dev.yml

@@ -96,17 +96,17 @@ services:
       prefect:
         condition: service_healthy
 
-  my-migration:
-    container_name: my-migration
-    build:
-      context: ../migration
-    restart: always
-    environment:
-      - FLOW_DEPLOYMENT_ENV=dev
-      - PREFECT_API_URL=http://prefect:4200/api
-    depends_on:
-      prefect:
-        condition: service_healthy
+  #my-migration:
+  #  container_name: my-migration
+  #  build:
+  #    context: ../migration
+  #  restart: always
+  #  environment:
+  #    - FLOW_DEPLOYMENT_ENV=dev
+  #    - PREFECT_API_URL=http://prefect:4200/api
+  #  depends_on:
+  #    prefect:
+  #      condition: service_healthy
 
   db:
     container_name: ${POSTGRESQL_CONTAINER_NAME}

ckan-backend-dev/solr/Dockerfile

@@ -2,7 +2,7 @@ FROM solr:8
 
 EXPOSE 8983
 
-ARG CKAN_BRANCH="dev-v2.10"
+ARG CKAN_BRANCH="2.11"
 
 ENV SOLR_INSTALL="/opt/solr"
 ENV SOLR_CONFIG_DIR="$SOLR_INSTALL/server/solr/configsets"

ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/get.py

@@ -13,8 +13,7 @@
 from ckan.common import config, asbool, aslist
 from ckan.model import Package
 from sqlalchemy import text, engine
-from shapely import MultiPolygon, Polygon, wkb, wkt
-from shapely import make_valid
+from shapely import wkb, wkt
 import ckan.model as model
 from ckan.logic.action.get import (
     _unpick_search, 
@@ -827,6 +826,7 @@ def group_activity_list_wri(context: Context, data_dict: DataDict):
     return results
 
 
+
 @logic.side_effect_free
 def user_list_wri(context: Context, data_dict: DataDict):
     model = context["model"]
@@ -1274,11 +1274,6 @@ def resource_search(context: Context, data_dict: DataDict):
                 shape = get_shape_from_dataapi(spatial_address, point)
                 if shape:
                     shape = wkt.loads(shape)
-                    bbox = Polygon([(-180, -90), (180, -90), (180, 90), (-180, 90)])
-                    shape = make_valid(shape)
-                    shape = shape.intersection(bbox)
-                    polygons = [geom for geom in shape.geoms if isinstance(geom, (Polygon, MultiPolygon))]
-                    shape = MultiPolygon(polygons)
                     spatial_geom = geoalchemy2.functions.ST_GeomFromText(shape.wkt)
                     location_queries.append(
                         geoalchemy2.functions.ST_Intersects(
@@ -1453,6 +1448,7 @@ def organization_list_for_user(context: Context,
     return orgs_list
 
 
+
 @logic.side_effect_free
 def organization_list(context: Context,
                       data_dict: DataDict) -> ActionResult.OrganizationList:
@@ -1516,6 +1512,7 @@ def organization_list(context: Context,
     return _group_or_org_list(context, data_dict, is_org=True)
 
 
+
 def _group_or_org_list(
         context: Context, data_dict: DataDict, is_org: bool = False):
     model = context['model']
@@ -1844,6 +1841,8 @@ def validate_visibility(context, data_dict):
             raise ValidationError({"message": _("Organization has private visibility and cannot create public datasets")})
 
 
+
+
 @logic.side_effect_free
 def organization_show(context, data_dict):
     data_dict = old_organization_show(context, data_dict)