Skip to content

Commit 146a2e9

Browse files
UsamaSadiqUsamaSadiq
authored
Revert "Deprecate paver wrapper (#111)" (#123)
This reverts commit 75b9a6e.
1 parent 75b9a6e commit 146a2e9

File tree

13 files changed

+180
-24
lines changed

13 files changed

+180
-24
lines changed

‎docker/build/edxapp/ansible_overrides.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ edxapp_npm_production: "no"
1212

1313
EDXAPP_LMS_GUNICORN_EXTRA_CONF: 'reload = True'
1414

15+
EDXAPP_NO_PREREQ_INSTALL: 0
1516
COMMON_SSH_PASSWORD_AUTH: "yes"
1617
EDXAPP_CMS_BASE: "edx.devstack.studio:18010"
1718
EDXAPP_OAUTH_ENFORCE_SECURE: false

‎playbooks/roles/aws_devstack/tasks/main.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
cd edx-repos/devstack/
3838
make dev.pull.large-and-slow
3939
make dev.up.lms
40-
. /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm ci && npm run build && ./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput && python manage.py lms migrate'
40+
. /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm ci && paver update_assets && python manage.py lms migrate'
4141
async: 1800
4242
poll: 10
4343
become_user: ubuntu

‎playbooks/roles/edxapp/defaults/main.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,9 @@ EDXAPP_CMS_STATIC_URL_BASE: "{{ EDXAPP_STATIC_URL_BASE }}"
487487
# does not affect verified students
488488
EDXAPP_PAID_COURSE_REGISTRATION_CURRENCY: ['usd', '$']
489489

490+
# Configure paver tasks in edx-platform to skip Python/Ruby/Node installation
491+
EDXAPP_NO_PREREQ_INSTALL: 1
492+
490493
# whether to setup the python codejail or not
491494
EDXAPP_PYTHON_SANDBOX: true
492495
# this next setting, if true, turns on actual sandbox enforcement. If not true,
@@ -1173,6 +1176,7 @@ edxapp_helper_scripts:
11731176

11741177
edxapp_environment_default:
11751178
LANG: "{{ EDXAPP_LANG }}"
1179+
NO_PREREQ_INSTALL: "{{ EDXAPP_NO_PREREQ_INSTALL }}"
11761180
SKIP_WS_MIGRATIONS: 1
11771181
PATH: "{{ edxapp_deploy_path }}"
11781182
# the settings module for edxapp, DJANGO_SETTINGS_MODULE

‎playbooks/roles/edxapp/tasks/main.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,15 @@
4949
- devstack
5050
- devstack:install
5151

52+
- name: add paver autocomplete to bashrc
53+
lineinfile:
54+
dest: /root/.bashrc
55+
line: "source {{ edxapp_code_dir }}/scripts/paver_autocomplete.sh"
56+
when: devstack is defined and devstack
57+
tags:
58+
- devstack
59+
- devstack:install
60+
5261
- name: create edxapp log dir
5362
file:
5463
path: "{{ edxapp_log_dir }}"

‎playbooks/roles/edxapp/tasks/service_variant_config.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -312,8 +312,8 @@
312312
- gather_static_assets
313313
- assets
314314

315-
# Build static assets if possible
316-
- name: "gather static assets"
315+
# Gather assets using paver if possible
316+
- name: "gather static assets with paver and pull translations"
317317
command: "{{ COMMON_BIN_DIR }}/edxapp-update-assets"
318318
when: celery_worker is not defined and not devstack
319319
tags:

‎playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,11 @@
33
{% if edxapp_staticfiles_storage_overrides %}
44
{% for override in edxapp_staticfiles_storage_overrides %}
55
sudo -E -H -u {{ edxapp_user }} \
6-
env "PATH=$PATH" \
7-
npm install \
8-
&& npm run webpack \
9-
&& npm run compile-sass -- --theme-dir /edx/var/edx-themes/edx-themes/edx-platform --theme-dir /edx/app/edxapp/edx-platform/themes \
10-
&& {{ edxapp_venv_bin }}/python manage.py lms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS \
11-
&& {{ edxapp_venv_bin }}/python manage.py cms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS
6+
env "PATH=$PATH" "STATICFILES_STORAGE={{ override }}" \
7+
{{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings=$EDX_PLATFORM_SETTINGS
128
{% endfor %}
139
{% else %}
1410
sudo -E -H -u {{ edxapp_user }} \
1511
env "PATH=$PATH" \
16-
npm install \
17-
&& npm run webpack \
18-
&& npm run compile-sass -- --theme-dir /edx/var/edx-themes/edx-themes/edx-platform --theme-dir /edx/app/edxapp/edx-platform/themes \
19-
&& {{ edxapp_venv_bin }}/python manage.py lms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS \
20-
&& {{ edxapp_venv_bin }}/python manage.py cms collectstatic --noinput --settings=$EDX_PLATFORM_SETTINGS
12+
{{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings $EDX_PLATFORM_SETTINGS
2113
{% endif %}

‎playbooks/roles/test_build_server/defaults/main.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
---
2+
#
3+
# edX Configuration
4+
#
5+
# github: https://github.com/openedx/configuration
6+
# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS
7+
# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
8+
# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT
9+
#
10+
##
11+
# Defaults for role test_build_server
12+
#
13+
14+
#
15+
# vars are namespace with the module name.
16+
#
17+
test_build_server_user: jenkins
18+
test_build_server_repo_path: /home/jenkins
19+
TEST_EDX_PLATFORM_VERSION: master

‎playbooks/roles/test_build_server/files/test-development-environment.sh

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
################################################################################
4+
# This executes a small subset of the edx-platform tests. It is intended as
5+
# a means of testing newly provisioned AMIs for our jenkins workers.
6+
#
7+
# The two main things that happen here:
8+
# 1. The setup from edx-platform/scripts/all-tests.sh, the script that is
9+
# run by the jenkins workers to kick off tests.
10+
# 2. The paver command for tests, coverage and quality reports are run.
11+
# For the tests, it runs only a small number of test cases for each
12+
# test suite.
13+
###############################################################################
14+
15+
# Doing this rather than copying the file into the scripts folder so that
16+
# this file doesn't get cleaned out by the 'git clean' in all-tests.sh.
17+
cd edx-platform-clone
18+
19+
# This will run all of the setup it usually runs, but none of the
20+
# tests because TEST_SUITE isn't defined.
21+
export PYTHON_VERSION=3.8
22+
source scripts/jenkins-common.sh
23+
24+
case "$1" in
25+
"unit")
26+
27+
# Now we can run a subset of the tests via paver.
28+
# Run some of the common/lib unit tests
29+
paver test_lib -t common/lib/xmodule/xmodule/tests/test_stringify.py
30+
31+
# Generate some coverage reports
32+
# Since `TEST_SUITE` is not set, change the coverage file written by the
33+
# previous test to a generic one.
34+
cp reports/common_lib_xmodule.coverage reports/.coverage
35+
paver coverage
36+
37+
# Run some of the djangoapp unit tests
38+
paver test_system -t lms/djangoapps/courseware/tests/tests.py
39+
paver test_system -t cms/djangoapps/course_creators/tests/test_views.py
40+
;;
41+
42+
"js")
43+
44+
# Run some of the javascript unit tests
45+
paver test_js_run -s lms-coffee
46+
;;
47+
48+
"quality")
49+
# Generate quality reports
50+
paver run_quality
51+
;;
52+
53+
*)
54+
echo "args required"
55+
exit 1
56+
esac

‎playbooks/roles/test_build_server/meta/main.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
#
3+
# edX Configuration
4+
#
5+
# github: https://github.com/openedx/configuration
6+
# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS
7+
# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
8+
# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT
9+
#
10+
##
11+
# Role includes for role test_build_server
12+
#
13+
# Example:
14+
#
15+
# dependencies:
16+
# - {
17+
# role: my_role
18+
# my_role_var0: "foo"
19+
# my_role_var1: "bar"
20+
# }
21+
22+
#### INTENTIONALLY LEFT BLANK ####
23+
24+
# Since this is a test role, it should not install anything extra onto the
25+
# target machine, thus altering the system under test. Be careful when
26+
# adding dependencies.

‎playbooks/roles/test_build_server/tasks/main.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
---
2+
#
3+
# edX Configuration
4+
#
5+
# github: https://github.com/openedx/configuration
6+
# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS
7+
# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
8+
# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT
9+
#
10+
#
11+
#
12+
# Tasks for role test_build_server
13+
#
14+
# Overview:
15+
#
16+
#
17+
# Dependencies:
18+
#
19+
#
20+
# Example play:
21+
#
22+
23+
- name: Create clone of edx-platform
24+
git:
25+
repo: "https://github.com/openedx/edx-platform.git"
26+
dest: "{{ test_build_server_repo_path }}/edx-platform-clone"
27+
version: "{{ TEST_EDX_PLATFORM_VERSION }}"
28+
become_user: "{{ test_build_server_user }}"
29+
30+
- name: get xargs limit
31+
shell: "xargs --show-limits"
32+
33+
- name: Copy test-development-environment.sh to somewhere the jenkins user can access it
34+
copy:
35+
src: test-development-environment.sh
36+
dest: "{{ test_build_server_repo_path }}"
37+
mode: 0755
38+
39+
- name: Validate build environment
40+
shell: "bash test-development-environment.sh {{ item }}"
41+
args:
42+
chdir: "{{ test_build_server_repo_path }}/"
43+
environment:
44+
PYTHON_VERSION: "3.8"
45+
become_user: "{{ test_build_server_user }}"
46+
with_items:
47+
- "unit"
48+
- "js"

‎playbooks/vagrant-analytics.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
devstack: true
88
disable_edx_services: true
99
mongo_enable_journal: false
10+
EDXAPP_NO_PREREQ_INSTALL: 0
1011
COMMON_SSH_PASSWORD_AUTH: "yes"
1112
EDXAPP_LMS_BASE: 127.0.0.1:8000
1213
EDXAPP_OAUTH_ENFORCE_SECURE: false

‎util/jenkins/app-container-provisioner.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,10 @@ chown :www-data /edx/etc/${app_service_name}.yml
8484
8585
if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then
8686
# run migrations
87-
docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database default
88-
docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database default --noinput
89-
docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database student_module_history
90-
docker run --network=host --rm -u='www-data' -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput
87+
docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database default
88+
docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database default --noinput
89+
docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database student_module_history
90+
docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput
9191
else
9292
# Run app migrations
9393
docker run --network=host --rm -u='www-data' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py migrate
@@ -123,7 +123,7 @@ fi
123123
if [[ ${app_service_name} == 'lms' ]]; then
124124
# temporary hack, create npm-install.log file
125125
touch /edx/app/edxapp/edx-platform/test_root/log/npm-install.log
126-
docker run --network=host --rm -u='root' -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/${app_service_name}.yml -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -e STATIC_ROOT_LMS=/edx/var/edxapp/staticfiles -e STATIC_ROOT_CMS=/edx/var/edxapp/staticfiles/studio -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /var/tmp/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/${app_name}:/edx/var/${app_name} -v /edx/app/edxapp/edx-platform/test_root/log/npm-install.log:/edx/app/edxapp/edx-platform/test_root/log/npm-install.log -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest npm run build && ./manage.py lms collectstatic --noinput --settings=docker-production && ./manage.py cms collectstatic --noinput --settings=docker-production
126+
docker run --network=host --rm -u='root' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/${app_service_name}.yml -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /var/tmp/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/${app_name}:/edx/var/${app_name} -v /edx/app/edxapp/edx-platform/test_root/log/npm-install.log:/edx/app/edxapp/edx-platform/test_root/log/npm-install.log -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest paver update_assets --debug-collect --settings=docker-production
127127
fi
128128
129129
# Generate docker-compose file for app service

0 commit comments

Comments
 (0)