Local automated upgrade testing

.dockerignore

@@ -17,3 +17,7 @@ securedrop/static/gen
 securedrop/static/.webassets-cache
 securedrop/tests/log/*
 securedrop/tests/pages-layout/
+**/.molecule
+build/*.deb
+**/*.box
+admin/.tox

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+    config.py,
+    .venv/,
+    journalist_gui/journalist_gui/updaterUI.py,
+    journalist_gui/journalist_gui/resources_rc.py,
+    .python3

.gitignore

@@ -147,3 +147,7 @@ raw-test-output/
 
 # Ignore visual studio code folder
 .vscode
+
+#Ignore vagrant staging files
+*.box
+*.img

Makefile

@@ -55,7 +55,7 @@ docs: ## Build project documentation in live reload for editing
 
 .PHONY: flake8
 flake8: ## Validates PEP8 compliance for Python source files.
-	flake8 --exclude='config.py,.venv/,journalist_gui/journalist_gui/updaterUI.py,journalist_gui/journalist_gui/resources_rc.py'
+	flake8
 
 .PHONY: app-lint
 app-lint: ## Tests pylint lint rule compliance.
@@ -143,6 +143,10 @@ self-signed-https-certs: ## Generates self-signed certs for TESTING the HTTPS co
 	@echo "Generating self-signed HTTPS certs for testing..."
 	@./devops/generate-self-signed-https-certs.sh
 
+.PHONY: vagrant-package
+vagrant-package: ## Package up a vagrant box of the last stable SD release
+	@devops/scripts/vagrant_package.sh
+
 # Explaination of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
 # 2. Use sed-like syntax to remove the make targets

devops/scripts/vagrant_package.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+molecule test -s vagrant_packager && \
+# Unfortunately since we need to prompt the user for sudo creds..
+# I had to break the actual vagrant package logic outside of molecule
+molecule/vagrant_packager/package.py && \
+molecule destroy -s vagrant_packager

install_files/ansible-base/group_vars/all/securedrop

@@ -65,3 +65,8 @@ enable_ssh_over_tor: true
 # force a reboot. Needed because of the de-coupled nature of
 # the many roles of the current prod playbook
 securedrop_cond_reboot_file: /tmp/sd-reboot-now
+
+# If you bump this, also remember to bump in molecule/builder/tests/vars.yml
+securedrop_pkg_grsec:
+  ver: "4.4.115"
+  depends: "linux-image-3.14.79-grsec,linux-image-4.4.115-grsec"

install_files/ansible-base/roles/build-generic-pkg/defaults/main.yml

@@ -2,3 +2,13 @@
 # Destination directory on Ansible controller where built deb packages
 # will be stored after fetching from build host.
 securedrop_local_build: "../../build"
+
+securedrop_generic_build_path: "/tmp/build/"
+
+securedrop_app_rsync_opts:
+  - "--chmod=u=rwX,g=rX,o=rX"
+  - "--chown=root:root"
+  - "--exclude=*.git"
+  - "--exclude=*aths"
+  - "--exclude=*.deb"
+  - "--exclude=*.j2"

install_files/ansible-base/roles/build-generic-pkg/tasks/main.yml

@@ -1,15 +1,46 @@
 ---
+- name: Reduce weird path re-usage
+  set_fact:
+    package_path: "{{ role_path }}/../../../{{ package_name }}/"
+    build_path: "{{ securedrop_generic_build_path }}/{{ package_name }}"
+
+- name: Ensure build directory in-place
+  file:
+    state: directory
+    path: "{{ securedrop_generic_build_path }}"
+
 - name: Copy app code to build directory.
   synchronize:
     # All Debian package control files for FPF-maintained packages reside in
     # `install_files/<package_name>` in the SecureDrop git repository.
-    src: "{{ role_path }}/../../../{{ package_name }}/"
-    dest: "/tmp/build/{{ package_name }}/"
+    src: "{{ package_path }}"
+    dest: "{{ build_path }}/"
     recursive: yes
     delete: yes
     use_ssh_args: "{{ 'yes' if amazon_builder else omit }}"
     rsync_opts: "{{ securedrop_app_rsync_opts }}"
 
+- name: Find any jinja templates
+  find:
+    paths: "{{ package_path }}"
+    patterns: "*.j2"
+    recurse: yes
+  delegate_to: localhost
+  become: no
+  register: jinja_files_found
+
+- name: Create any necessary parent directories for jinja files
+  file:
+    path: "{{ build_path }}/{{ item.path | dirname |regex_replace('^\\/.*'+package_name, '') }}"
+    state: directory
+  with_items: "{{ jinja_files_found.files }}"
+
+- name: Template out any jinja files found and copy over
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ build_path }}/{{ item.path | regex_replace('^\\/.*'+package_name, '') | regex_replace ('\\.j2$','') }}"
+  with_items: "{{ jinja_files_found.files }}"
+
 - name: run bash script to build generic packages
   script: "build_generic_package.sh {{ package_name }}"
 

install_files/ansible-base/roles/build-ossec-deb-pkg/library/ossec_urls.py

@@ -102,7 +102,7 @@ def main():
     ossec_version = module.params['ossec_version']
     try:
         ossec_config = OSSECURLs(ossec_version=ossec_version)
-    except:
+    except:  # noqa: E722
         msg = ("Failed to find checksum information for OSSEC v{}."
                "Ensure you have the proper release specified, "
                "and check the download page to confirm: "

install_files/securedrop-grsec/DEBIAN/control → install_files/securedrop-grsec/DEBIAN/control.j2

@@ -1,9 +1,9 @@
 Package: securedrop-grsec
 Source: securedrop-grsec
-Version: 4.4.115
+Version: {{ securedrop_pkg_grsec.ver }}
 Architecture: amd64
 Maintainer: SecureDrop Team <[email protected]>
-Depends: linux-image-3.14.79-grsec,linux-image-4.4.115-grsec
+Depends: {{ securedrop_pkg_grsec.depends }}
 Section: admin
 Priority: optional
 Homepage: https://securedrop.org

molecule/ansible-config/molecule.yml

@@ -22,8 +22,8 @@ provisioner:
       callback_whitelist: "profile_tasks, timer"
   inventory:
     links:
-      group_vars: ../../../install_files/ansible-base/group_vars
-      host_vars: ../../../install_files/ansible-base/host_vars
+      group_vars: ../../install_files/ansible-base/group_vars
+      host_vars: ../../install_files/ansible-base/host_vars
   env:
     ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles
 scenario:

molecule/aws/molecule.yml

@@ -31,8 +31,8 @@ provisioner:
       callback_whitelist: "profile_tasks, timer"
   inventory:
     links:
-      group_vars: ../../../install_files/ansible-base/group_vars
-      host_vars: ../../../install_files/ansible-base/host_vars
+      group_vars: ../../install_files/ansible-base/group_vars
+      host_vars: ../../install_files/ansible-base/host_vars
   env:
     ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles
 scenario:

molecule/builder/molecule.yml

@@ -13,8 +13,7 @@ provisioner:
   name: ansible
   inventory:
     links:
-      # Path is relative to .molecule folder
-      group_vars: ../../../install_files/ansible-base/group_vars
+      group_vars: ../../install_files/ansible-base/group_vars
   env:
     ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles
     ANSIBLE_ACTION_PLUGINS: ../../install_files/ansible-base/action_plugins

molecule/builder/tests/test_securedrop_deb_package.py

@@ -38,6 +38,7 @@ def get_deb_packages():
             ossec_version=securedrop_test_vars.ossec_version,
             keyring_version=securedrop_test_vars.keyring_version,
             config_version=securedrop_test_vars.config_version,
+            grsec_version=securedrop_test_vars.grsec_version
             )
 
     deb_packages = [d.format(**substitutions) for d
@@ -289,3 +290,18 @@ def test_grsec_metapackage(host, deb):
         # Post-install kernel hook for managing PaX flags must exist.
         assert re.search("^.*\./etc/kernel/postinst.d/paxctl-grub$",
                          c.stdout, re.M)
+
+
+@pytest.mark.parametrize("deb", deb_packages)
+def test_jinja_files_not_present(host, deb):
+    """
+    Make sure that jinja (.j2) files were not copied over
+    as-is into the debian packages.
+    """
+
+    deb_package = host.file(deb.format(
+        securedrop_test_vars.securedrop_version))
+
+    c = host.run("dpkg-deb --contents {}".format(deb_package.path))
+    # There shouldn't be any files with a .j2 ending
+    assert not re.search("^.*\.j2$", c.stdout, re.M)

molecule/builder/tests/vars.yml

@@ -3,6 +3,7 @@ securedrop_version: "0.7.0~rc1"
 ossec_version: "2.8.2"
 keyring_version: "0.1.1"
 config_version: "0.1.1"
+grsec_version: "4.4.115"
 
 # These values will be interpolated with values populated above
 # via helper functions in the tests.
@@ -26,6 +27,7 @@ build_deb_packages:
   - /tmp/build/ossec-agent-{ossec_version}-amd64.deb
   - /tmp/build/securedrop-keyring-{keyring_version}+{securedrop_version}-amd64.deb
   - /tmp/build/securedrop-config-{config_version}+{securedrop_version}-amd64.deb
+  - /tmp/build/securedrop-grsec-{grsec_version}-amd64.deb
 
 lintian_tags:
   # - non-standard-file-perm

molecule/upgrade/ansible-override-vars.yml

@@ -0,0 +1,24 @@
+---
+install_local_packages: false
+primary_network_iface: eth0
+ssh_users: vagrant
+securedrop_app_install_from_repo: true
+allow_direct_access: true
+ssh_listening_address: 0.0.0.0
+
+monitor_ip: "{{ hostvars['mon-staging']['ansible_'+primary_network_iface].ipv4.address }}"
+monitor_hostname: "{{ hostvars['mon-staging'].ansible_hostname }}"
+app_ip: "{{ hostvars['app-staging']['ansible_'+primary_network_iface].ipv4.address }}"
+app_hostname: "{{ hostvars['app-staging'].ansible_hostname }}"
+
+etc_hosts:
+  app-staging:
+    - reg: ^127.0.0.1
+      line: 127.0.0.1   app-staging localhost
+    - reg: securedrop-monitor-server-alias$
+      line: "{{ monitor_ip }}  mon-staging securedrop-monitor-server-alias"
+  mon-staging:
+    - reg: ^127.0.0.1\s+mon
+      line: 127.0.0.1   mon-staging localhost
+    - reg: app-staging$
+      line: "{{ app_ip }}  app-staging"

molecule/upgrade/apt.yml

@@ -0,0 +1,74 @@
+---
+
+- name: Configure apt-server
+  hosts: aptservers
+  become: yes
+  tasks:
+    - name: Establish QA strategy
+      set_fact:
+        QA_APTTEST: "{{ lookup('env','QA_APTTEST')|bool }}"
+
+    - import_tasks: local_apt_mirror.yml
+    - import_tasks: local_apt_with_debs.yml
+      when: not QA_APTTEST
+
+  vars:
+    molecule_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}"
+    dpkg_dir: /var/repos/debs
+    rep_dist: trusty
+    rep_component: main
+    rep_arch: i386 amd64
+    release_file: "/var/repos/base/dists/{{ rep_dist }}/Release"
+    nginx_sites:
+      default:
+        - listen 80
+        - root "/var/repos/base"
+        - location / { autoindex on; }
+        - location /gpg { alias /var/repos/base/; }
+      encrypted:
+        - listen 443 ssl
+        - server_name apt.freedom.press
+        - ssl_certificate /etc/ssl/certs/apt_freedom_press.pem
+        - ssl_certificate_key /etc/ssl/private/apt_freedom_press.priv
+        - root "/var/repos/base"
+        - location / { {{ "proxy_pass https://apt-test.freedom.press;" if QA_APTTEST else "autoindex on;" }} }
+
+- name: Configure apt-server
+  hosts: securedrop
+  gather_facts: false
+  become: yes
+  tasks:
+    - block:
+        - name: Redirect to local QA Apt server
+          set_fact:
+            apt_key_bits:
+              - url: "http://{{ hostvars['apt']['ansible_eth0'].ipv4.address }}/gpg/apt-test.pub"
+                id: 6D65484B
+              - url: "http://{{ hostvars['apt']['ansible_eth0'].ipv4.address }}/gpg/apt-test-fpf.pub"
+                id: 4A3BE4A92211B03C
+            apt_test_etc_line: "{{ hostvars['apt']['ansible_eth0'].ipv4.address }}  apt.freedom.press"
+
+        - name: Install testing apt key to keyring
+          apt_key:
+            id: "{{ item.id|default(omit) }}"
+            url: "{{ item.url|default(omit) }}"
+            data: "{{ item.data|default(omit) }}"
+            state: present
+          with_items: "{{ apt_key_bits }}"
+
+        - name: Redirect apt.freedom.press to local apt server
+          lineinfile:
+            path: /etc/hosts
+            regexp: apt.freedom.press$
+            line: "{{ apt_test_etc_line }}"
+
+        - name: Add apt-test CA to CA trusted store
+          copy:
+            src: cacert.pub
+            dest: /usr/local/share/ca-certificates/fpf_test_ca.crt
+          notify: update ca
+      tags: apt
+
+  handlers:
+    - name: update ca
+      command: update-ca-certificates

molecule/upgrade/create.yml

@@ -0,0 +1,59 @@
+---
+- import_playbook: sd_clone.yml
+
+- name: Create
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+  tasks:
+
+    - name: Create molecule instance(s)
+      molecule_vagrant:
+        instance_name: "{{ item.name }}"
+        instance_interfaces: "{{ item.interfaces | default(omit) }}"
+        instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}"
+
+        platform_box: "{{ item.box }}"
+        platform_box_version: "{{ item.box_version | default(omit) }}"
+        platform_box_url: "{{ 'file://'+playbook_dir+'/' if item.box_url|default(false) else '' }}{{ item.box_url | default(omit) }}"
+
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        provider_memory: "{{ item.memory | default(omit) }}"
+        provider_cpus: "{{ item.cpus | default(omit) }}"
+        provider_override_args: "{{ item.provider_override_args | default(omit) }}"
+        provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}"
+        force_stop: yes
+
+        state: up
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config dict
+      set_fact:
+        instance_conf_dict: {
+          'instance': "{{ item.Host }}",
+          'address': "{{ item.HostName }}",
+          'user': "{{ item.User }}",
+          'port': "{{ item.Port }}",
+          'identity_file': "{{ item.IdentityFile }}", }
+      with_items: "{{ server.results }}"
+      register: instance_config_dict
+      when: server.changed | bool
+
+    - name: Convert instance config dict to a list
+      set_fact:
+        instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}"
+      when: server.changed | bool
+
+    - name: Dump instance config
+      copy:
+        # NOTE(retr0h): Workaround for Ansible 2.2.
+        #               https://github.com/ansible/ansible/issues/20885
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool