Add a compile_json job to replace compile_firmware. (#67)

* switch to qmk list-keyboards

* add a compile_json job that will replace compile_firmware

* fix the qmk list-keyboards parsing

* Add a skeleton for keymap.json

Author: skullydazed

qmk_commands.py

@@ -31,6 +31,15 @@
 LUFA_GIT_BRANCH = os.environ.get('LUFA_GIT_BRANCH', 'master')
 LUFA_GIT_URL = os.environ.get('LUFA_GIT_URL', 'https://github.com/qmk/lufa')
 
+KEYMAP_DOCUMENTATION = """"This file is a QMK Configurator export. You can import this at <https://config.qmk.fm>. It can also be used directly with QMK's source code.
+
+To setup your QMK environment check out the tutorial: <https://docs.qmk.fm/#/newbs>
+
+You can convert this file to a keymap.c using this command: `qmk json2c {keymap}`
+
+You can compile this keymap using this command: `qmk compile {keymap}`"
+"""
+
 ZIP_EXCLUDES = {
     'qmk_firmware': ['qmk_firmware/.build/*', 'qmk_firmware/.git/*', 'qmk_firmware/lib/chibios/.git', 'qmk_firmware/lib/chibios-contrib/.git'],
     'chibios': ['chibios/.git/*'],
@@ -302,3 +311,17 @@ def write_version_txt():
     hash = check_output(['git', 'rev-parse', 'HEAD'], universal_newlines=True)
     version_txt = Path('version.txt')
     version_txt.write_text(hash + '\n')
+
+
+def keymap_skeleton():
+    """Returns the minimal structure needed for a keymap.json.
+    """
+    return {
+            'version': 1,
+            'notes': '',
+            'keyboard': None,
+            'keymap': None,
+            'layout': None,
+            'layers': [],
+            'documentation': KEYMAP_DOCUMENTATION,
+    }

qmk_compiler.py

@@ -3,7 +3,7 @@
 from io import BytesIO
 from os import chdir, environ, path, remove
 from subprocess import check_output, CalledProcessError, STDOUT
-from time import strftime
+from time import strftime, time
 from traceback import format_exc
 
 from rq import get_current_job
@@ -80,7 +80,7 @@ def compile_keymap(job, result):
         result['firmware_filename'] = find_firmware_file()
 
     except CalledProcessError as build_error:
-        logging.error('Could not build firmware (%s): %s', build_error.cmd, build_error.output)
+        print('Could not build firmware (%s): %s' % (build_error.cmd, build_error.output))
         result['returncode'] = build_error.returncode
         result['cmd'] = build_error.cmd
         result['output'] = build_error.output
@@ -106,7 +106,6 @@ def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
         'author': '',
         'notes': '',
         'version': 1,
-        'source_ip': source_ip,
         'documentation': 'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`' % (keymap_json_file,)
     })
     result = {
@@ -130,7 +129,7 @@ def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
 
         # Sanity checks
         if not path.exists('qmk_firmware/keyboards/' + keyboard):
-            logging.error('Unknown keyboard: %s', keyboard)
+            print('Unknown keyboard: %s' % (keyboard,))
             return {'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None}
 
         # If this keyboard needs a submodule check it out
@@ -161,8 +160,71 @@ def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
     return result
 
 
+@job('default', connection=redis, timeout=900)
+def compile_json(keyboard_keymap_data, source_ip=None):
+    """Compile a keymap.
+
+    Arguments:
+
+        keyboard_keymap_data
+            A configurator export file that's been deserialized
+
+        source_ip
+            The IP that submitted the compile job
+    """
+    result = {
+        'returncode': -2,
+        'output': '',
+        'firmware': None,
+        'firmware_filename': '',
+        'source_ip': source_ip,
+        'output': 'Unknown error',
+    }
+    try:
+        for key in ('keyboard', 'layout', 'keymap'):
+            result[key] = keyboard_keymap_data[key]
+
+        result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace('/', '-'), result['keymap'].replace('/', '-'))
+        result['keymap_json'] = json.dumps(keyboard_keymap_data)
+        result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
+
+        kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard'])
+        job = get_current_job()
+        result['id'] = job.id
+        checkout_qmk()
+
+        # Sanity checks
+        if not path.exists('qmk_firmware/keyboards/' + result['keyboard']):
+            print('Unknown keyboard: %s' % (result['keyboard'],))
+            return {'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None}
+
+        # If this keyboard needs a submodule check it out
+        if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
+            checkout_lufa()
+
+        if kb_data.get('protocol') == 'ChibiOS':
+            checkout_chibios()
+
+        # Write the keymap file
+        with open(path.join('qmk_firmware', result['keymap_archive']), 'w') as fd:
+            fd.write(result['keymap_json'] + '\n')
+
+        # Compile the firmware
+        store_firmware_source(result)
+        remove(result['source_archive'])
+        compile_keymap(job, result)
+        store_firmware_binary(result)
+
+    except Exception as e:
+        result['returncode'] = -3
+        result['exception'] = e.__class__.__name__
+        result['stacktrace'] = format_exc()
+
+    return result
+
+
 @job('default', connection=redis)
 def ping():
     """Write a timestamp to redis to make sure at least one worker is running ok.
     """
-    return redis.set('qmk_api_last_ping', strftime('"%Y-%m-%d %H:%M:%SZ"'))
+    return redis.set('qmk_api_last_ping', time())

update_kb_redis.py

@@ -67,9 +67,8 @@ def list_keyboards():
     """
     chdir('qmk_firmware')
     try:
-        keyboards = check_output(('make', 'list-keyboards'), stderr=STDOUT, universal_newlines=True)
+        keyboards = check_output(('qmk', 'list-keyboards'), universal_newlines=True)
         keyboards = keyboards.strip()
-        keyboards = keyboards.split('\n')[-1]
     finally:
         chdir('..')
     return keyboards.split()