Issue #2659 - Move console logs saving to upload endpoint

Author: ksy36

tests/unit/test_console_logs.py

@@ -4,12 +4,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-'''Tests for image upload API endpoint.'''
+'''Tests for console logs upload.'''
 
 import json
 import os.path
 import sys
 import unittest
+from werkzeug.datastructures import MultiDict
 
 # Add webcompat module to import path
 sys.path.append(os.path.realpath(os.pardir))
@@ -50,6 +51,12 @@ def cleanup(filepath):
     os.remove(filepath)
 
 
+def form_request(data):
+    d = MultiDict()
+    d['console_logs'] = data
+    return d
+
+
 class TestConsoleUploads(unittest.TestCase):
     def setUp(self):
         app.config['TESTING'] = True
@@ -60,30 +67,35 @@ def tearDown(self):
         pass
 
     def test_get(self):
-        '''Test that /console_logs/ doesn't let you GET.'''
-        rv = self.test_client.get('/console_logs/')
+        '''Test that /upload/ doesn't let you GET.'''
+        rv = self.test_client.get('/upload/')
         self.assertEqual(rv.status_code, 404)
 
     def test_console_log_upload(self):
         rv = self.test_client.post(
-            '/console_logs/', data=json.dumps(LOG))
+            '/upload/', data=form_request(json.dumps(LOG)))
         self.assertEqual(rv.status_code, 201)
 
         response = json.loads(rv.data)
         self.assertTrue('url' in response)
         cleanup(get_path(self, response.get('url')))
 
+    def test_console_log_bad_format(self):
         rv = self.test_client.post(
-            '/console_logs/', data='{test}')
+            '/upload/', data=form_request('{test}'))
         self.assertEqual(rv.status_code, 400)
 
         rv = self.test_client.post(
-            '/console_logs/', data='')
-        self.assertEqual(rv.status_code, 400)
+            '/upload/', data=form_request(''))
+        self.assertEqual(rv.status_code, 501)
+
+        rv = self.test_client.post(
+            '/upload/', data=json.dumps(LOG))
+        self.assertEqual(rv.status_code, 501)
 
     def test_console_log_file_contents(self):
         rv = self.test_client.post(
-            '/console_logs/', data=json.dumps(LOG))
+            '/upload/', data=form_request(json.dumps(LOG)))
         response = json.loads(rv.data)
         filepath = get_path(self, response.get('url'))
         actual = get_json_contents(filepath)

webcompat/__init__.py

@@ -39,9 +39,8 @@
 from webcompat.api.endpoints import api  # noqa
 from webcompat.api.uploads import uploads  # noqa
 from webcompat.error_handlers import error_handlers  # noqa
-from webcompat.api.console_logs import console_logs  # noqa
 
-for blueprint in [api, error_handlers, uploads, console_logs]:
+for blueprint in [api, error_handlers, uploads]:
     app.register_blueprint(blueprint)
 
 

webcompat/api/console_logs.py

@@ -4,7 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-'''Flask Blueprint for image uploads.'''
+'''Flask Blueprint for file uploads.'''
 
 import base64
 import datetime
@@ -22,13 +22,29 @@
 from werkzeug.exceptions import RequestEntityTooLarge
 
 from webcompat import app
+from webcompat.helpers import get_data_from_request
 
 uploads = Blueprint('uploads', __name__, url_prefix='/upload',
                     template_folder='../templates')
 JSON_MIME = 'application/json'
 
 
-class Upload(object):
+class BaseUpload(object):
+    def __init__(self):
+        today = datetime.date.today()
+        self.year = str(today.year)
+        self.month = str(today.month)
+        self.file_id = str(uuid.uuid4())
+
+    def get_file_path(self, year, month, file_id, ext, thumb=False):
+        thumb_string = ''
+        if thumb:
+            thumb_string = '-thumb'
+        file_name = file_id + thumb_string + '.' + ext
+        return os.path.join(year, month, file_name)
+
+
+class ImageUpload(BaseUpload):
 
     '''Class that abstracts over saving image and screenshot uploads.
 
@@ -39,17 +55,15 @@ class Upload(object):
     ALLOWED_FORMATS = ('jpg', 'jpeg', 'jpe', 'png', 'gif', 'bmp')
 
     def __init__(self, imagedata):
+        super().__init__()
         self.image_object = self.to_image_object(imagedata)
-        # computing the parameters to be used
-        today = datetime.date.today()
-        self.year = str(today.year)
-        self.month = str(today.month)
-        self.image_id = str(uuid.uuid4())
         self.file_ext = self.get_file_ext()
-        self.image_path = self.img_path(self.month, self.year, self.image_id,
-                                        thumb=False)
-        self.thumb_path = self.img_path(self.month, self.year, self.image_id,
-                                        thumb=True)
+        self.image_path = self.get_file_path(self.year, self.month,
+                                             self.file_id, self.file_ext,
+                                             thumb=False)
+        self.thumb_path = self.get_file_path(self.year, self.month,
+                                             self.file_id, self.file_ext,
+                                             thumb=True)
 
     def to_image_object(self, imagedata):
         '''Method to return a Pillow Image object from the raw imagedata.'''
@@ -70,14 +84,6 @@ def to_image_object(self, imagedata):
             # Not a valid format
             abort(415)
 
-    def img_path(self, month, year, image_id, thumb=False):
-        '''Return the right image path.'''
-        thumb_string = ''
-        if thumb:
-            thumb_string = '-thumb'
-        image_name = image_id + thumb_string + '.' + self.file_ext
-        return os.path.join(year, month, image_name)
-
     def get_file_ext(self):
         '''Method to return the file extension, as determined by Pillow.
 
@@ -125,33 +131,69 @@ def save(self):
         self.image_object.thumbnail(size, Image.HAMMING)
         self.image_object.save(thumb_dest, **save_parameters)
 
+    def get_file_info(self):
+        return {
+            'filename': self.get_filename(self.image_path),
+            'url': self.get_url(self.image_path),
+            'thumb_url': self.get_url(self.thumb_path)
+        }
+
+
+class LogUpload(BaseUpload):
+    def __init__(self, data):
+        super().__init__()
+        self.data = self.process_data(data)
+        self.file_path = self.get_file_path(self.year, self.month,
+                                            self.file_id, 'json')
+
+    def process_data(self, data):
+        try:
+            return json.loads(data)
+        except ValueError:
+            abort(400)
+
+    def get_url(self, file_path):
+        return os.path.splitext(file_path)[0]
+
+    def save(self):
+        file_dest = app.config['UPLOADS_DEFAULT_DEST'] + self.file_path
+        dest_dir = os.path.dirname(file_dest)
+
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+
+        with open(file_dest, 'w', encoding='utf-8') as f:
+            json.dump(self.data, f, ensure_ascii=False)
+
+    def get_file_info(self):
+        return {
+            'url': self.get_url(self.file_path)
+        }
+
 
 @uploads.route('/', methods=['POST'])
 def upload():
-    '''Endpoint to upload an image.
+    '''Endpoint to upload an image or a json with console logs.
 
-    If the image asset passes validation, it's saved as:
+    If the file asset passes validation, it's saved as:
         UPLOADS_DEFAULT_DEST + /year/month/random-uuid.ext
 
     Returns a JSON string that contains the filename and url.
     '''
-    if 'image' in request.files and request.files['image'].filename:
-        imagedata = request.files['image']
-    elif 'image' in request.form:
-        imagedata = request.form['image']
-    else:
+    is_image, data = get_data_from_request(request)
+    if not data:
         # We don't know what you're trying to do, but it ain't gonna work.
         abort(501)
 
     try:
-        upload = Upload(imagedata)
+        if is_image:
+            upload = ImageUpload(data)
+        else:
+            upload = LogUpload(data)
+
         upload.save()
-        data = {
-            'filename': upload.get_filename(upload.image_path),
-            'url': upload.get_url(upload.image_path),
-            'thumb_url': upload.get_url(upload.thumb_path)
-        }
-        return (json.dumps(data), 201, {'content-type': JSON_MIME})
+        file_info = upload.get_file_info()
+        return json.dumps(file_info), 201, {'content-type': JSON_MIME}
     except (TypeError, IOError):
         abort(415)
     except RequestEntityTooLarge:

webcompat/api/uploads.py

@@ -32,7 +32,7 @@
 from wtforms.validators import Regexp
 
 from webcompat import app
-from webcompat.api.uploads import Upload
+from webcompat.api.uploads import ImageUpload
 from webcompat.helpers import get_browser
 from webcompat.helpers import get_os
 from webcompat.helpers import get_str_value
@@ -221,7 +221,8 @@ class IssueForm(FlaskForm):
     # any changes here need to be updated in form.html.
     image = FileField('Attach a screenshot image',
                       [Optional(),
-                       FileAllowed(Upload.ALLOWED_FORMATS, image_message)])
+                       FileAllowed(ImageUpload.ALLOWED_FORMATS,
+                       image_message)])
     details = HiddenField()
     reported_with = HiddenField()
     ua_header = HiddenField()

webcompat/form.py

@@ -412,7 +412,7 @@ def wrapped_func(*args, **kwargs):
                 file_path = FIXTURES_PATH + request.path + "." + checksum
             else:
                 file_path = FIXTURES_PATH + request.path
-            if '.json' not in file_path:
+            if not file_path.endswith('.json'):
                 file_path = file_path + '.json'
             if not os.path.exists(file_path):
                 print('Fixture expected at: {fix}'.format(fix=file_path))
@@ -761,3 +761,14 @@ def get_extra_labels(form):
             extra_labels = ['form-v2-experiment']
 
     return extra_labels
+
+
+def get_data_from_request(request):
+    if 'image' in request.files and request.files['image'].filename:
+        return True, request.files['image']
+    elif 'image' in request.form:
+        return True, request.form['image']
+    elif 'console_logs' in request.form:
+        return False, request.form['console_logs']
+    else:
+        return False, None

webcompat/helpers.py

@@ -655,11 +655,15 @@ BugForm.prototype.uploadConsoleLogs = function() {
     return dfd.resolve();
   }
 
+  var formdata = new FormData();
+  formdata.append("console_logs", JSON.stringify(details.consoleLog));
+
   return $.ajax({
-    contentType: "application/json",
-    data: JSON.stringify(details.consoleLog),
+    contentType: false,
+    processData: false,
+    data: formdata,
     method: "POST",
-    url: "/console_logs/",
+    url: "/upload/",
     success: function(response) {
       var path = location.origin + "/console_logs/";
       this.consoleLogsInput.val(path + response.url);

webcompat/static/js/lib/bugform.js

@@ -1199,11 +1199,15 @@ BugForm.prototype.uploadConsoleLogs = function() {
     return dfd.resolve();
   }
 
+  var formdata = new FormData();
+  formdata.append("console_logs", JSON.stringify(details.consoleLog));
+
   return $.ajax({
-    contentType: "application/json",
-    data: JSON.stringify(details.consoleLog),
+    contentType: false,
+    processData: false,
+    data: formdata,
     method: "POST",
-    url: "/console_logs/",
+    url: "/upload/",
     success: function(response) {
       var path = location.origin + "/console_logs/";
       this.consoleLogsInput.val(path + response.url);

webcompat/static/js/lib/issue-wizard-bugform.js

@@ -2,6 +2,7 @@
 <html lang="en">
 <head>
   <title>Console logs | webcompat.com</title>
+  <meta name="robots" content="noindex">
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta name="description" content="Open source community for web compatibility.  Report bugs from websites or for browsers and help us move the web forward.">