[CLI] Add c2json (#8817)

* Basic keymap parsing finally works

* Add 'keymap.json' creation to the qmk.keymap module

* Add tests and fix formatting

* Fix/exclude flake8 errors

* Convert keymap.c to valid keymap.json

* Fix some errors

* Add tests

* Finalize keymap.json creation, add json template

* Add docs

* Move pygments to the standard requirements

* Add support for nameless layers, fix tests

* Fix things after rebase

* Add missing 'keymap' value.

* Fix missing layer numbers from advanced keycodes

Buckwich noticed that if the advanced keycode / layer toggling key
contains a number, it goes missing.
Now we properly handle them.
Thx for noticing!

* Apply suggestions from code review

* fixup tests

Co-authored-by: Zach White <skullydazed@drpepper.org>
Co-authored-by: skullY <skullydazed@gmail.com>

docs/cli_commands.md

@@ -167,6 +167,17 @@ Creates a keymap.c from a QMK Configurator export.
 qmk json2c [-o OUTPUT] filename
 ```
 
+## `qmk c2json`
+
+Creates a keymap.json from a keymap.c.  
+**Note:** Parsing C source files is not easy, therefore this subcommand may not work your keymap. In some cases not using the C pre-processor helps.
+
+**Usage**:
+
+```
+qmk c2json [--no-cpp] [-o OUTPUT] filename
+```
+
 ## `qmk list-keyboards`
 
 This command lists all the keyboards currently defined in `qmk_firmware`

keyboards/handwired/onekey/keymaps/pytest_nocpp/keymap.c

@@ -0,0 +1,23 @@
+#include QMK_KEYBOARD_H
+#include "audio.h"
+
+/* THIS FILE WAS GENERATED AND IS EXPERIMENTAL!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+	[0] = LAYOUT(KC_ENTER)
+};
+
+void encoder_update_user(uint8_t index, bool clockwise) {
+	if (index == 0) {
+		if (clockwise) {
+			tap_code(KC_UP);
+		} else {
+			tap_code(KC_DOWN);
+		}
+	}
+
+};

keyboards/handwired/onekey/pytest/templates/keymap.json

@@ -0,0 +1,3 @@
+{
+  "documentation": "This file is a keymap.json file for handwired/onekey/pytest"
+}

lib/python/qmk/cli/__init__.py

@@ -6,6 +6,7 @@ import sys
 
 from milc import cli
 
+from . import c2json
 from . import cformat
 from . import compile
 from . import config

lib/python/qmk/cli/c2json.py

@@ -0,0 +1,62 @@
+"""Generate a keymap.json from a keymap.c file.
+"""
+import json
+import sys
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('--no-cpp', arg_only=True, action='store_false', help='Do not use \'cpp\' on keymap.c')
+@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.argument('-kb', '--keyboard', arg_only=True, required=True, help='The keyboard\'s name')
+@cli.argument('-km', '--keymap', arg_only=True, required=True, help='The keymap\'s name')
+@cli.argument('filename', arg_only=True, help='keymap.c file')
+@cli.subcommand('Creates a keymap.json from a keymap.c file.')
+def c2json(cli):
+    """Generate a keymap.json from a keymap.c file.
+
+    This command uses the `qmk.keymap` module to generate a keymap.json from a keymap.c file. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    cli.args.filename = qmk.path.normpath(cli.args.filename)
+
+    # Error checking
+    if not cli.args.filename.exists():
+        cli.log.error('C file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    if str(cli.args.filename) == '-':
+        # TODO(skullydazed/anyone): Read file contents from STDIN
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the keymap.c
+    keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp)
+
+    # Generate the keymap.json
+    try:
+        keymap_json = qmk.keymap.generate(keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'], type='json', keymap=keymap_json['keymap'])
+    except KeyError:
+        cli.log.error('Something went wrong. Try to use --no-cpp.')
+        sys.exit(1)
+
+    if cli.args.output:
+        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
+        if cli.args.output.exists():
+            cli.args.output.replace(cli.args.output.name + '.bak')
+        cli.args.output.write_text(json.dumps(keymap_json))
+
+        if not cli.args.quiet:
+            cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(json.dumps(keymap_json))

lib/python/qmk/cli/doctor.py

@@ -58,7 +58,7 @@ def parse_gcc_version(version):
     return {
         'major': int(m.group(1)),
         'minor': int(m.group(2)) if m.group(2) else 0,
-        'patch': int(m.group(3)) if m.group(3) else 0
+        'patch': int(m.group(3)) if m.group(3) else 0,
     }
 
 

lib/python/qmk/commands.py

@@ -7,7 +7,6 @@ import subprocess
 import shlex
 import shutil
 
-from milc import cli
 import qmk.keymap
 
 
@@ -84,6 +83,4 @@ def run(command, *args, **kwargs):
         safecmd = ' '.join(safecmd)
         command = [os.environ['SHELL'], '-c', safecmd]
 
-    cli.log.debug('Running command: %s', command)
-
     return subprocess.run(command, *args, **kwargs)

lib/python/qmk/tests/test_cli_commands.py

@@ -1,10 +1,11 @@
-import subprocess
+from subprocess import STDOUT, PIPE
+
 from qmk.commands import run
 
 
 def check_subcommand(command, *args):
     cmd = ['bin/qmk', command] + list(args)
-    result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    result = run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
     return result
 
 
@@ -28,6 +29,11 @@ def test_compile():
     check_returncode(result)
 
 
+def test_compile_json():
+    result = check_subcommand('compile', '-kb', 'handwired/onekey/pytest', '-km', 'default_json')
+    check_returncode(result)
+
+
 def test_flash():
     result = check_subcommand('flash', '-kb', 'handwired/onekey/pytest', '-km', 'default', '-n')
     check_returncode(result)
@@ -153,3 +159,15 @@ def test_info_matrix_render():
     assert 'LAYOUT_ortho_1x1' in result.stdout
     assert '│0A│' in result.stdout
     assert 'Matrix for "LAYOUT_ortho_1x1"' in result.stdout
+
+
+def test_c2json():
+    result = check_subcommand("c2json", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/default/keymap.c")
+    check_returncode(result)
+    assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}'
+
+
+def test_c2json_nocpp():
+    result = check_subcommand("c2json", "--no-cpp", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/pytest_nocpp/keymap.c")
+    check_returncode(result)
+    assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_ENTER"]]}'

lib/python/qmk/tests/test_qmk_keymap.py

@@ -6,14 +6,34 @@ def test_template_onekey_proton_c():
     assert templ == qmk.keymap.DEFAULT_KEYMAP_C
 
 
+def test_template_onekey_proton_c_json():
+    templ = qmk.keymap.template('handwired/onekey/proton_c', type='json')
+    assert templ == {'keyboard': 'handwired/onekey/proton_c'}
+
+
 def test_template_onekey_pytest():
     templ = qmk.keymap.template('handwired/onekey/pytest')
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n'
 
 
+def test_template_onekey_pytest_json():
+    templ = qmk.keymap.template('handwired/onekey/pytest', type='json')
+    assert templ == {'keyboard': 'handwired/onekey/pytest', "documentation": "This file is a keymap.json file for handwired/onekey/pytest"}
+
+
 def test_generate_onekey_pytest():
     templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n'
 
 
+def test_generate_onekey_pytest_json():
+    templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']], type='json', keymap='default')
+    assert templ == {"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]}
+
+
+def test_parse_keymap_c():
+    parsed_keymap_c = qmk.keymap.parse_keymap_c('keyboards/handwired/onekey/keymaps/default/keymap.c')
+    assert parsed_keymap_c == {'layers': [{'name': '0', 'layout': 'LAYOUT_ortho_1x1', 'keycodes': ['KC_A']}]}
+
+
 # FIXME(skullydazed): Add a test for qmk.keymap.write that mocks up an FD.

requirements.txt

@@ -4,3 +4,4 @@ appdirs
 argcomplete
 colorama
 hjson
+pygments