Merge pull request #1998 from karlcow/1995/1

Fix #1995 - Adds proper unicode handling for Form data

Author: Mike Taylor

tests/test_form.py

@@ -16,6 +16,7 @@ class TestForm(unittest.TestCase):
 
     def setUp(self):
         """Set up."""
+        self.maxDiff = None
         webcompat.app.config['TESTING'] = True
         self.app = webcompat.app.test_client()
 
@@ -28,6 +29,12 @@ def test_normalize_url(self):
         r = form.normalize_url('http://example.com')
         self.assertEqual(r, 'http://example.com')
 
+        r = form.normalize_url(u'愛')
+        self.assertEqual(r, u'http://愛')
+
+        r = form.normalize_url(u'http://愛')
+        self.assertEqual(r, u'http://愛')
+
         r = form.normalize_url('https://example.com')
         self.assertEqual(r, 'https://example.com')
 
@@ -136,3 +143,8 @@ def test_build_formdata(self):
         expected = {'body': u'<!-- @browser: None -->\n<!-- @ua_header: None -->\n<!-- @reported_with: None -->\n\n**URL**: None\n\n**Browser / Version**: None\n**Operating System**: None\n**Tested Another Browser**: Unknown\n\n**Problem type**: Unknown\n**Description**: None\n**Steps to Reproduce**:\nNone\n\n\n\n_From [webcompat.com](https://webcompat.com/) with \u2764\ufe0f_', 'title': 'None - unknown'}  # nopep8
         self.assertIs(type(actual), dict)
         self.assertEqual(actual, expected)
+        # testing with unicode strings.
+        expected = {'body': u'<!-- @browser: None -->\n<!-- @ua_header: None -->\n<!-- @reported_with: None -->\n\n**URL**: \u611b\n\n**Browser / Version**: None\n**Operating System**: None\n**Tested Another Browser**: Unknown\n\n**Problem type**: Unknown\n**Description**: None\n**Steps to Reproduce**:\nNone\n\n\n\n_From [webcompat.com](https://webcompat.com/) with \u2764\ufe0f_', 'title': u'\u611b - unknown'}  # nopep8
+        form_object = {'url': u'愛'}
+        actual = form.build_formdata(form_object)
+        self.assertEqual(actual, expected)

webcompat/form.py

@@ -145,17 +145,17 @@ def normalize_url(url):
         # if url starts with a bad scheme, parsed.netloc will be empty,
         # so we use parsed.path instead
         path = parsed.path.lstrip('/')
-        url = '{}://{}'.format(parsed.scheme, path)
+        url = u'{}://{}'.format(parsed.scheme, path)
         if parsed.query:
             url += '?' + parsed.query
         if parsed.fragment:
             url += '#' + parsed.fragment
     elif not parsed.scheme:
         # We assume that http is missing not https
         if url.startswith("//"):
-            url = "http://{}".format(url[2:])
+            url = u"http://{}".format(url[2:])
         else:
-            url = 'http://{}'.format(url)
+            url = u'http://{}'.format(url)
     return url
 
 
@@ -221,15 +221,16 @@ def build_formdata(form_object):
     NOTE: Add milestone "needstriage" when create new issue
     """
     # Do domain extraction for adding to the summary/title
+    # form_object always returns a unicode string
     url = form_object.get('url')
     normalized_url = normalize_url(url)
     domain = domain_name(normalized_url)
     problem_summary = get_problem_summary(form_object.get('problem_category'))
 
     if domain:
-        summary = '{0} - {1}'.format(domain, problem_summary)
+        summary = u'{0} - {1}'.format(domain, problem_summary)
     else:
-        summary = '{0} - {1}'.format(normalized_url, problem_summary)
+        summary = u'{0} - {1}'.format(normalized_url, problem_summary)
 
     metadata_keys = ['browser', 'ua_header', 'reported_with']
     extra_label = form_object.get('extra_label', None)
@@ -265,7 +266,7 @@ def build_formdata(form_object):
 """.format(**formdata)
     # Add the image, if there was one.
     if form_object.get('image_upload') is not None:
-        body += '\n\n![Screenshot of the site issue]({image_url})'.format(
+        body += u'\n\n![Screenshot of the site issue]({image_url})'.format(
             image_url=form_object.get('image_upload').get('url'))
     # Append "from webcompat.com" message to bottom (for GitHub issue viewers)
     body += u'\n\n{0}'.format(GITHUB_HELP)