.github/workflows/ci.yaml

@@ -0,0 +1,49 @@
+name: continuous integration
+on:
+  - push
+  - pull_request
+
+jobs:
+  tests:
+    name: tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['2.x', 'pypy2']
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: setup riak
+        run: |
+          utils/setup_travis_riak.sh 2.1.1
+      - name: setup redis
+        uses: shogo82148/actions-setup-redis@v1
+        with:
+          redis-version: '6.x'
+      - name: setup nodejs
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8'
+      - name: install deps
+        run: |
+          pip install wheel
+          pip install -r requirements.pip
+      - name: check riak
+        # Hopefully enough time has passed that riak is now up and running.
+        run: |
+          $HOME/riak/bin/riak version
+          $HOME/riak/bin/riak-admin member-status
+      - name: run tests
+        env:
+          VUMI_TEST_ASSERT_CLOSED: 'true'
+          VUMI_TEST_TIMEOUT: '20'
+          VUMITEST_REDIS_DB: '1'
+        run: |
+          export VUMI_TEST_NODE_PATH="$(which node)"
+          echo "node path: ${VUMI_TEST_NODE_PATH}"
+          echo "node version: $(${VUMI_TEST_NODE_PATH} --version)"
+          trial vumi

.travis.yml

@@ -1,26 +1,26 @@
-sudo: false # We want container builds.
+# Default (xenial) has antique openssl that cryptography no longer supports.
+dist: bionic
 
 language: python
 python:
   - "2.7"
   - "3.6"
-node_js:
-  - "0.10"
 matrix:
   include:
     # Test against the oldest version of Twisted that we claim to support.
-    # Also test against Riak 2.1.1.
+    # Also test against Riak 1.4.12.
     # This is a separate matrix inclusion to avoid spawning unnecessary builds.
     # Although there are two different things we're testing against here, they
     # are orthogonal and any failures should be easily attributable to either
     # Twisted version or Riak version without adding an additional build job.
-    # Twisted 13.2 requires PyCrypto for twisted.conch.ssh support.
     - python: "2.7"
-      env: TWISTED_VERSION="Twisted==13.2.0" RIAK_VERSION="2.1.1" PYCRYPTO_VERSION="PyCrypto==2.6.1"
+      env: TWISTED_VERSION="Twisted==16.4.0" RIAK_VERSION="1.4.12"
     # Test on pypy without coverage, because it's unnecessary and very slow.
-    # Also, we hit an obscure GC bug in pypy<=2.6.0 so we need at least 2.6.1.
-    - python: "pypy"
-      env: PYPY_VERSION="5.6.0" NO_COVERAGE=1
+    # The exact versions of pypy available seem to change over time, but pypy2
+    # gets us an older version. Hopefully pypy2.7-7.3.1 will continue working
+    # for a while.
+    - python: "pypy2.7-7.3.1"
+      env: NO_COVERAGE=1
     - python: "3.6"
       env: RUN_TESTS="false"
   allow_failures:
@@ -36,24 +36,26 @@ services:
 
 before_install:
   # Default values for our build envvars.
-  - export RIAK_VERSION="${RIAK_VERSION-1.4.12}"
+  - export RIAK_VERSION="${RIAK_VERSION-2.1.1}"
   - export TWISTED_VERSION="${TWISTED_VERSION-Twisted}"
+  - export NODEJS_VERSION="${NODEJS_VERSION-lts/carbon}"
   - export RUN_TESTS="${RUN_TESTS-true}"
-  # If necessary, set up an appropriate version of pypy.
-  - if [ ! -z "$PYPY_VERSION" ]; then source utils/setup-pypy-travis.sh; fi
-  - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi
   # Set up an appropriate version of Riak.
   - utils/setup_travis_riak.sh "${RIAK_VERSION}"
+  # Set up an appropriate version of nodejs.
+  - nvm install $NODEJS_VERSION
 install:
   # Travis seems to have pip 6.x, which doesn't build and cache wheels.
   - "pip install 'pip>=7.1.0'"
   - "pip install ${TWISTED_VERSION}"
-  # If requested, install PyCrypto
-  - if [ ! -z "$PYCRYPTO_VERSION" ]; then pip install "$PYCRYPTO_VERSION"; fi
   - "pip install -r requirements.pip"
   - "pip install coveralls"
 
 before_script:
+  # Dump the riak console log in case in contains anything we need to see.
+  - echo '   vvv riak console log vvv'
+  - cat $HOME/riak/log/console.log
+  - echo '   ^^^ riak console log ^^^'
   # To see what version of Riak we're running and check that it's happy.
   - $HOME/riak/bin/riak version
   - $HOME/riak/bin/riak-admin member-status

docs/conf.py

@@ -53,7 +53,7 @@
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.17'
+release = '0.6.19'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="vumi",
-    version="0.6.17",
+    version="0.6.19",
     url='http://github.com/praekelt/vumi',
     license='BSD',
     description="Super-scalable messaging engine for the delivery of SMS, "
@@ -36,7 +36,7 @@
     install_requires=[
         cryptography,  # See above for pypy-version-dependent requirement.
         'zope.interface',
-        'Twisted>=13.2.0',
+        'Twisted>=16.4.0',  # Minimum version supported by recent treq.
         'txAMQP>=0.6.2',
         'PyYAML',
         'iso8601',
@@ -45,14 +45,14 @@
         'service_identity',
         'txssmi>=0.3.0',
         'wokkel',
-        'redis>=2.10.0',
+        'redis>=2.10.0,<3.0.0',  # 3.x has a new api.
         'txredis',
         'python-smpp>=0.1.5',
         'pytz',
         'riak>=2.1',
         'txJSON-RPC==0.5',
         'txTwitter>=0.1.4a',
-        'treq<16.12.0',
+        'treq<20.4.1',  # https://github.com/twisted/treq/issues/303
         'confmodel>=0.2.0',
         'hyperloglog',
     ],

utils/setup_travis_riak.sh

@@ -16,4 +16,10 @@ fi
 
 tar xjf $RIAK_DOWNLOAD -C $HOME/
 
+# Fix jvm opts if necessary.
+if [ -f $HOME/riak/etc/riak.conf ]; then
+    sed -i'.bak' 's/^search.solr.jvm_options = .*$/search.solr.jvm_options = -Xms1g -Xmx1g -XX:+UseCompressedOops/' $HOME/riak/etc/riak.conf
+    rm $HOME/riak/etc/riak.conf.bak
+fi
+
 $HOME/riak/bin/riak start

vumi/__init__.py

@@ -2,4 +2,4 @@
 Vumi scalable text messaging engine.
 """
 
-__version__ = "0.6.17"
+__version__ = "0.6.19"

vumi/application/tests/test_sandbox.py

@@ -770,7 +770,7 @@ def test_handle_get_for_unknown_key(self):
 
     @inlineCallbacks
     def test_handle_delete(self):
-        self.create_metric('foo', json.dumps('bar'))
+        yield self.create_metric('foo', json.dumps('bar'))
         yield self.r_server.set('count#test_id', '1')
         reply = yield self.dispatch_command('delete', key='foo')
         self.check_reply(reply, success=True, existed=True)
@@ -790,14 +790,14 @@ def test_handle_incr_create(self):
 
     @inlineCallbacks
     def test_handle_incr_existing(self):
-        self.create_metric('foo', '2')
+        yield self.create_metric('foo', '2')
         reply = yield self.dispatch_command('incr', key='foo', amount=2)
         self.check_reply(reply, success=True, value=4)
         yield self.check_metric('foo', '4', 1)
 
     @inlineCallbacks
     def test_handle_incr_existing_non_int(self):
-        self.create_metric('foo', 'a')
+        yield self.create_metric('foo', 'a')
         reply = yield self.dispatch_command('incr', key='foo', amount=2)
         self.check_reply(reply, success=False)
         self.assertTrue(reply['reason'])
@@ -990,7 +990,7 @@ def fail_next(self, error):
     def succeed_next(self, body, code=200, headers={}):
 
         default_headers = {
-            'Content-Length': len(body),
+            'Content-Length': str(len(body)),
         }
         default_headers.update(headers)
 
@@ -1282,7 +1282,7 @@ def test_handle_post_files(self):
     @inlineCallbacks
     def test_data_limit_exceeded_using_header(self):
         self.http_request_succeed('', headers={
-            'Content-Length': self.resource.DEFAULT_DATA_LIMIT + 1,
+            'Content-Length': str(self.resource.DEFAULT_DATA_LIMIT + 1),
         })
         reply = yield self.dispatch_command(
             'get', url='https://www.example.com',)

vumi/transports/mtn_nigeria/xml_over_tcp.py

@@ -384,7 +384,9 @@ def serialize_body(cls, packet_type, params):
             root.appendChild(el)
 
         data = root.toxml()
-        return data.decode('utf8').encode(cls.ENCODING, 'xmlcharrefreplace')
+        if not isinstance(data, unicode):
+            data = data.decode('utf8')
+        return data.encode(cls.ENCODING, 'xmlcharrefreplace')
 
     @classmethod
     def serialize_packet(cls, session_id, packet_type, params):

vumi/transports/vumi_bridge/tests/test_vumi_bridge.py

@@ -179,9 +179,9 @@ def test_receive_bad_event(self):
         self.assertEquals(status['message'], 'Bad event received from Vumi Go')
 
     @inlineCallbacks
-    def test_weak_cacerts_installed(self):
+    def test_cacerts_installed(self):
         yield self.get_configured_transport()
-        self.assertEqual(os.environ["SSL_CERT_FILE"], certifi.old_where())
+        self.assertEqual(os.environ["SSL_CERT_FILE"], certifi.where())
 
     @inlineCallbacks
     def test_sending_messages(self):

vumi/transports/vumi_bridge/vumi_bridge.py

@@ -231,11 +231,7 @@ def teardown_transport(self):
             self.redis.close_manager()
 
     def setup_cacerts(self):
-        # TODO: This installs an older CA certificate chain that allows
-        #       some weak CA certificates. We should switch to .where() when
-        #       Vumi Go's certificate doesn't rely on older intermediate
-        #       certificates.
-        os.environ["SSL_CERT_FILE"] = certifi.old_where()
+        os.environ["SSL_CERT_FILE"] = certifi.where()
 
     def get_transport_url(self, suffix=''):
         """