QMK CLI and JSON keymap support (#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

.editorconfig

@@ -16,6 +16,10 @@ insert_final_newline = true
 trim_trailing_whitespace = false
 indent_size = 4
 
+[{qmk,*.py}]
+charset = utf-8
+max_line_length = 200
+
 # Make these match what we have in .gitattributes
 [*.mk]
 end_of_line = lf

.gitignore

@@ -70,3 +70,6 @@ util/Win_Check_Output.txt
 secrets.tar
 id_rsa_*
 /.vs
+
+# python things
+__pycache__

bin/qmk

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""CLI wrapper for running QMK commands.
+"""
+import os
+import subprocess
+import sys
+from glob import glob
+from time import strftime
+from importlib import import_module
+from importlib.util import find_spec
+
+# Add the QMK python libs to our path
+script_dir = os.path.dirname(os.path.realpath(__file__))
+qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
+python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
+sys.path.append(python_lib_dir)
+
+# Change to the root of our checkout
+os.environ['ORIG_CWD'] = os.getcwd()
+os.chdir(qmk_dir)
+
+# Make sure our modules have been setup
+with open('requirements.txt', 'r') as fd:
+    for line in fd.readlines():
+        line = line.strip().replace('<', '=').replace('>', '=')
+
+        if line[0] == '#':
+            continue
+
+        if '#' in line:
+            line = line.split('#')[0]
+
+        module = line.split('=')[0] if '=' in line else line
+        if not find_spec(module):
+            print('Your QMK build environment is not fully setup!\n')
+            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            exit(255)
+
+# Figure out our version
+command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
+result = subprocess.run(command, text=True, capture_output=True)
+
+if result.returncode == 0:
+    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+else:
+    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+
+# Setup the CLI
+import milc
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}ψ{style_reset_all}'
+
+# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
+# This means we can't accept arguments to the qmk script itself.
+script_name = os.path.basename(sys.argv[0])
+if script_name == 'qmk':
+    if len(sys.argv) == 1:
+        milc.cli.log.error('No subcommand specified!\n')
+
+    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
+        milc.cli.echo('usage: qmk <subcommand> [...]')
+        milc.cli.echo('\nsubcommands:')
+        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
+        for subcommand in sorted(subcommands):
+            subcommand = os.path.basename(subcommand).split('-', 1)[1]
+            milc.cli.echo('\t%s', subcommand)
+        milc.cli.echo('\nqmk <subcommand> --help for more information')
+        exit(1)
+
+    if sys.argv[1] in ['-V', '--version']:
+        milc.cli.echo(os.environ['QMK_VERSION'])
+        exit(0)
+
+    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
+    del sys.argv[1]
+
+# Look for which module to import
+if script_name == 'qmk':
+    milc.cli.print_help()
+    exit(0)
+elif not script_name.startswith('qmk-'):
+    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
+else:
+    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
+    subcommand.insert(1, 'cli')
+    subcommand = '.'.join(subcommand)
+
+    try:
+        import_module(subcommand)
+    except ModuleNotFoundError as e:
+        if e.__class__.__name__ != subcommand:
+            raise
+
+        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
+        exit(1)
+
+if __name__ == '__main__':
+    milc.cli()

bin/qmk-compile-json

@@ -0,0 +1 @@
+qmk

bin/qmk-doctor

@@ -0,0 +1 @@
+qmk

bin/qmk-hello

@@ -0,0 +1 @@
+qmk

bin/qmk-json-keymap

@@ -0,0 +1 @@
+qmk

build_json.mk

@@ -0,0 +1,27 @@
+# Look for a json keymap file
+ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+endif
+
+# Generate the keymap.c
+ifneq ("$(KEYMAP_JSON)","")
+    _ = $(shell bin/qmk-json-keymap -f $(KEYMAP_JSON) -o $(KEYMAP_C))
+endif

build_keyboard.mk

@@ -98,31 +98,38 @@ MAIN_KEYMAP_PATH_3 := $(KEYBOARD_PATH_3)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_5)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_4)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_3)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_2)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_1)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-else ifneq ($(LAYOUTS),)
-    include build_layout.mk
-else
-    $(error Could not find keymap)
-    # this state should never be reached
+# Check for keymap.json first, so we can regenerate keymap.c
+include build_json.mk
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
+    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_5)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_4)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_3)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_2)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_1)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+    else ifneq ($(LAYOUTS),)
+        # If we haven't found a keymap yet fall back to community layouts
+        include build_layout.mk
+    else
+        $(error Could not find keymap)
+        # this state should never be reached
+    endif
 endif
 
 ifeq ($(strip $(CTPC)), yes)
@@ -313,7 +320,6 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
-
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!
@@ -323,7 +329,7 @@ ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
 
-# # project specific files
+# project specific files
 SRC += $(KEYBOARD_SRC) \
     $(KEYMAP_C) \
     $(QUANTUM_SRC)

docs/_summary.md

@@ -8,6 +8,7 @@
 
 * [QMK Basics](README.md)
   * [QMK Introduction](getting_started_introduction.md)
+  * [QMK CLI](cli.md)
   * [Contributing to QMK](contributing.md)
   * [How to Use Github](getting_started_github.md)
   * [Getting Help](getting_started_getting_help.md)
@@ -34,6 +35,8 @@
   * [Keyboard Guidelines](hardware_keyboard_guidelines.md)
   * [Config Options](config_options.md)
   * [Keycodes](keycodes.md)
+  * [Coding Conventions - C](coding_conventions_c.md)
+  * [Coding Conventions - Python](coding_conventions_python.md)
   * [Documentation Best Practices](documentation_best_practices.md)
   * [Documentation Templates](documentation_templates.md)
   * [Glossary](reference_glossary.md)
@@ -41,6 +44,7 @@
   * [Useful Functions](ref_functions.md)
   * [Configurator Support](reference_configurator_support.md)
   * [info.json Format](reference_info_json.md)
+  * [Python Development](python_development.md)
 
 * [Features](features.md)
   * [Basic Keycodes](keycodes_basic.md)

docs/cli.md

@@ -0,0 +1,31 @@
+# QMK CLI
+
+This page describes how to setup and use the QMK CLI.
+
+# Overview
+
+The QMK CLI makes building and working with QMK keyboards easier. We have provided a number of commands to help you work with QMK:
+
+* `qmk compile-json`
+
+# Setup
+
+Simply add the `qmk_firmware/bin` directory to your `PATH`. You can run the `qmk` commands from any directory.
+
+```
+export PATH=$PATH:$HOME/qmk_firmware/bin
+```
+
+You may want to add this to your `.profile`, `.bash_profile`, `.zsh_profile`, or other shell startup scripts.
+
+# Commands
+
+## `qmk compile-json`
+
+This command allows you to compile JSON files you have downloaded from <https://config.qmk.fm>.
+
+**Usage**:
+
+```
+qmk compile-json mine.json
+```

docs/coding_conventions_c.md

@@ -0,0 +1,58 @@
+# Coding Conventions (C)
+
+Most of our style is pretty easy to pick up on, but right now it's not entirely consistent. You should match the style of the code surrounding your change, but if that code is inconsistent or unclear use the following guidelines:
+
+* We indent using four (4) spaces (soft tabs)
+* We use a modified One True Brace Style
+  * Opening Brace: At the end of the same line as the statement that opens the block
+  * Closing Brace: Lined up with the first character of the statement that opens the block
+  * Else If: Place the closing brace at the beginning of the line and the next opening brace at the end of the same line.
+  * Optional Braces: Always include optional braces.
+    * Good: if (condition) { return false; }
+    * Bad: if (condition) return false;
+* We encourage use of C style comments: `/* */`
+  * Think of them as a story describing the feature
+  * Use them liberally to explain why particular decisions were made.
+  * Do not write obvious comments
+  * If you not sure if a comment is obvious, go ahead and include it.
+* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
+* We use `#pragma once` at the start of header files rather than old-style include guards (`#ifndef THIS_FILE_H`, `#define THIS_FILE_H`, ..., `#endif`)
+* We accept both forms of preprocessor if's: `#ifdef DEFINED` and `#if defined(DEFINED)`
+  * If you are not sure which to prefer use the `#if defined(DEFINED)` form.
+  * Do not change existing code from one style to the other, except when moving to a multiple condition `#if`.
+  * Do not put whitespace between `#` and `if`.
+  * When deciding how (or if) to indent directives keep these points in mind:
+    * Readability is more important than consistency.
+    * Follow the file's existing style. If the file is mixed follow the style that makes sense for the section you are modifying.
+    * When choosing to indent you can follow the indention level of the surrounding C code, or preprocessor directives can have their own indent level. Choose the style that best communicates the intent of your code.
+
+Here is an example for easy reference:
+
+```c
+/* Enums for foo */
+enum foo_state {
+  FOO_BAR,
+  FOO_BAZ,
+};
+
+/* Returns a value */
+int foo(void) {
+  if (some_condition) {
+    return FOO_BAR;
+  } else {
+    return -1;
+  }
+}
+```
+
+# Auto-formatting with clang-format
+
+[Clang-format](https://clang.llvm.org/docs/ClangFormat.html) is part of LLVM and can automatically format your code for you, because ain't nobody got time to do it manually. We supply a configuration file for it that applies most of the coding conventions listed above. It will only change whitespace and newlines, so you will still have to remember to include optional braces yourself.
+
+Use the [full LLVM installer](http://llvm.org/builds/) to get clang-format on Windows, or use `sudo apt install clang-format` on Ubuntu.
+
+If you run it from the command-line, pass `-style=file` as an option and it will automatically find the .clang-format configuration file in the QMK root directory.
+
+If you use VSCode, the standard C/C++ plugin supports clang-format, alternatively there is a [separate extension](https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat) for it.
+
+Some things (like LAYOUT macros) are destroyed by clang-format, so either don't run it on those files, or wrap the sensitive code in `// clang-format off` and `// clang-format on`.

docs/contributing.md

@@ -54,62 +54,10 @@ Never made an open source contribution before? Wondering how contributions work
 
 # Coding Conventions
 
-Most of our style is pretty easy to pick up on, but right now it's not entirely consistent. You should match the style of the code surrounding your change, but if that code is inconsistent or unclear use the following guidelines:
+Most of our style is pretty easy to pick up on. If you are familiar with either C or Python you should not have too much trouble with our local styles.
 
-* We indent using four (4) spaces (soft tabs)
-* We use a modified One True Brace Style
-  * Opening Brace: At the end of the same line as the statement that opens the block
-  * Closing Brace: Lined up with the first character of the statement that opens the block
-  * Else If: Place the closing brace at the beginning of the line and the next opening brace at the end of the same line.
-  * Optional Braces: Always include optional braces.
-    * Good: if (condition) { return false; }
-    * Bad: if (condition) return false;
-* We encourage use of C style comments: `/* */`
-  * Think of them as a story describing the feature
-  * Use them liberally to explain why particular decisions were made.
-  * Do not write obvious comments
-  * If you not sure if a comment is obvious, go ahead and include it.
-* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
-* We use `#pragma once` at the start of header files rather than old-style include guards (`#ifndef THIS_FILE_H`, `#define THIS_FILE_H`, ..., `#endif`)
-* We accept both forms of preprocessor if's: `#ifdef DEFINED` and `#if defined(DEFINED)`
-  * If you are not sure which to prefer use the `#if defined(DEFINED)` form.
-  * Do not change existing code from one style to the other, except when moving to a multiple condition `#if`.
-  * Do not put whitespace between `#` and `if`.
-  * When deciding how (or if) to indent directives keep these points in mind:
-    * Readability is more important than consistency.
-    * Follow the file's existing style. If the file is mixed follow the style that makes sense for the section you are modifying.
-    * When choosing to indent you can follow the indention level of the surrounding C code, or preprocessor directives can have their own indent level. Choose the style that best communicates the intent of your code.
-
-Here is an example for easy reference:
-
-```c
-/* Enums for foo */
-enum foo_state {
-  FOO_BAR,
-  FOO_BAZ,
-};
-
-/* Returns a value */
-int foo(void) {
-  if (some_condition) {
-    return FOO_BAR;
-  } else {
-    return -1;
-  }
-}
-```
-
-# Auto-formatting with clang-format
-
-[Clang-format](https://clang.llvm.org/docs/ClangFormat.html) is part of LLVM and can automatically format your code for you, because ain't nobody got time to do it manually. We supply a configuration file for it that applies most of the coding conventions listed above. It will only change whitespace and newlines, so you will still have to remember to include optional braces yourself.
-
-Use the [full LLVM installer](http://llvm.org/builds/) to get clang-format on Windows, or use `sudo apt install clang-format` on Ubuntu.
-
-If you run it from the command-line, pass `-style=file` as an option and it will automatically find the .clang-format configuration file in the QMK root directory.
-
-If you use VSCode, the standard C/C++ plugin supports clang-format, alternatively there is a [separate extension](https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat) for it.
-
-Some things (like LAYOUT macros) are destroyed by clang-format, so either don't run it on those files, or wrap the sensitive code in `// clang-format off` and `// clang-format on`.
+* [Coding Conventions - C](coding_conventions_c.md)
+* [Coding Conventions - Python](coding_conventions_python.md)
 
 # General Guidelines
 

docs/python_development.md

@@ -0,0 +1,45 @@
+# Python Development in QMK
+
+This document gives an overview of how QMK has structured its python code. You should read this before working on any of the python code.
+
+## Script directories
+
+There are two places scripts live in QMK: `qmk_firmware/bin` and `qmk_firmware/util`. You should use `bin` for any python scripts that utilize the `qmk` wrapper. Scripts that are standalone and not run very often live in `util`.
+
+We discourage putting anything into `bin` that does not utilize the `qmk` wrapper. If you think you have a good reason for doing so please talk to us about your use case.
+
+## Python Modules
+
+Most of the QMK python modules can be found in `qmk_firmware/lib/python`. This is the path that we append to `sys.path`.
+
+We have a module hierarchy under that path:
+
+* `qmk_firmware/lib/python`
+    * `milc.py` - The CLI library we use. Will be pulled out into its own module in the future.
+    * `qmk` - Code associated with QMK
+        * `cli` - Modules that will be imported for CLI commands.
+        * `errors.py` - Errors that can be raised within QMK apps
+        * `keymap.py` - Functions for working with keymaps
+
+## CLI Scripts
+
+We have a CLI wrapper that you should utilize for any user facing scripts. We think it's pretty easy to use and it gives you a lot of nice things for free.
+
+To use the wrapper simply place a module into `qmk_firmware/lib/python/qmk/cli`, and create a symlink to `bin/qmk` named after your module. Dashes in command names will be converted into dots so you can use hierarchy to manage commands.
+
+When `qmk` is run it checks to see how it was invoked. If it was invoked as `qmk` the module name is take from `sys.argv[1]`. If it was invoked as `qmk-<module-name>` then everything after the first dash is taken as the module name. Dashes and underscores are converted to dots, and then `qmk.cli` is prepended before the module is imported.
+
+The module uses `@cli.entrypoint()` and `@cli.argument()` decorators to define an entrypoint, which is where execution starts.
+
+## Example CLI Script
+
+We have provided a QMK Hello World script you can use as an example. To run it simply run `qmk hello` or `qmk-hello`. The source code is listed below.
+
+```
+from milc import cli
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Python Hello World.')
+def main(cli):
+    cli.echo('Hello, %s!', cli.config.general.name)
+```

keyboards/clueboard/66_hotswap/keymaps/json/keymap.json

@@ -0,0 +1 @@
+{"keyboard":"clueboard/66_hotswap/gen1","keymap":"default_66","layout":"LAYOUT","layers":[["KC_GESC","KC_1","KC_2","KC_3","KC_4","KC_5","KC_6","KC_7","KC_8","KC_9","KC_0","KC_MINS","KC_EQL","KC_BSPC","KC_PGUP","KC_TAB","KC_Q","KC_W","KC_E","KC_R","KC_T","KC_Y","KC_U","KC_I","KC_O","KC_P","KC_LBRC","KC_RBRC","KC_BSLS","KC_PGDN","KC_CAPS","KC_A","KC_S","KC_D","KC_F","KC_G","KC_H","KC_J","KC_K","KC_L","KC_SCLN","KC_QUOT","KC_ENT","KC_LSFT","KC_Z","KC_X","KC_C","KC_V","KC_B","KC_N","KC_M","KC_COMM","KC_DOT","KC_SLSH","KC_RSFT","KC_UP","KC_LCTL","KC_LGUI","KC_LALT","KC_SPC","KC_SPC","KC_RALT","KC_RGUI","MO(1)","KC_RCTL","KC_LEFT","KC_DOWN","KC_RGHT"],["KC_GRV","KC_F1","KC_F2","KC_F3","KC_F4","KC_F5","KC_F6","KC_F7","KC_F8","KC_F9","KC_F10","KC_F11","KC_F12","KC_DEL","BL_INC","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_MPRV","KC_MPLY","KC_MNXT","KC_NO","KC_MUTE","BL_DEC","KC_NO","KC_NO","MO(2)","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_PGUP","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","MO(1)","KC_NO","KC_HOME","KC_PGDN","KC_END"],["KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_TOGG","BL_INC","KC_NO","KC_NO","KC_NO","KC_NO","RESET","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_DEC","KC_NO","KC_NO","MO(2)","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_STEP","KC_NO","KC_NO","MO(1)","KC_NO","KC_NO","KC_NO","KC_NO"]],"author":"","notes":""}

lib/python/qmk/cli/compile/json.py

@@ -0,0 +1,44 @@
+"""Create a keymap directory from a configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', help='Configurator JSON export')
+@cli.entrypoint('Compile a QMK Configurator export.')
+def main(cli):
+    """Compile a QMK Configurator export.
+
+    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        exit(1)
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    # Compile the keymap
+    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)

lib/python/qmk/cli/doctor.py

@@ -0,0 +1,47 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import shutil
+import platform
+import os
+
+from milc import cli
+
+
+@cli.entrypoint('Basic QMK environment checks')
+def main(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Run the binaries to make sure they work
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+
+    cli.log.info('QMK Doctor is Checking your environment')
+
+    ok = True
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            ok = False
+
+    OS = platform.system()
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS")
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}linux")
+        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+        if os.system(test) == 0:
+            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows")
+
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')

lib/python/qmk/cli/hello.py

@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Hello World.')
+def main(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.general.name)

lib/python/qmk/cli/json/keymap.py

@@ -0,0 +1,54 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+import sys
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', help='File to write to')
+@cli.argument('filename', help='Configurator JSON file')
+@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
+def main(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)

lib/python/qmk/errors.py

@@ -0,0 +1,6 @@
+class NoSuchKeyboardError(Exception):
+    """Raised when we can't find a keyboard/keymap directory.
+    """
+
+    def __init__(self, message):
+        self.message = message

lib/python/qmk/keymap.py

@@ -0,0 +1,100 @@
+"""Functions that help you work with QMK keymaps.
+"""
+import json
+import logging
+import os
+from traceback import format_exc
+
+import qmk.path
+from qmk.errors import NoSuchKeyboardError
+
+# The `keymap.c` template to use when a keyboard doesn't have its own
+DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+
+/* THIS FILE WAS GENERATED!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+__KEYMAP_GOES_HERE__
+};
+"""
+
+
+def template(keyboard):
+    """Returns the `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
+    text will be used instead of `DEFAULT_KEYMAP_C`.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
+    """
+    template_name = 'keyboards/%s/templates/keymap.c' % keyboard
+
+    if os.path.exists(template_name):
+        with open(template_name, 'r') as fd:
+            return fd.read()
+
+    return DEFAULT_KEYMAP_C
+
+
+def generate(keyboard, layout, layers):
+    """Returns a keymap.c for the specified keyboard, layout, and layers.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    keymap_c = template(keyboard, keymap)
+
+    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        keymap
+            The name of the keymap
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_c = generate(keyboard, layout, layers)
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_dir = os.path.join(keymap_path, keymap)
+    keymap_file = os.path.join(keymap_dir, 'keymap.c')
+
+    if not os.path.exists(keymap_dir):
+        os.makedirs(keymap_dir)
+
+    with open(keymap_file, 'w') as keymap_fd:
+        keymap_fd.write(keymap_c)
+
+    return keymap_file

lib/python/qmk/path.py

@@ -0,0 +1,32 @@
+"""Functions that help us work with files and folders.
+"""
+import os
+
+
+def keymap(keyboard):
+    """Locate the correct directory for storing a keymap.
+
+    Args:
+        keyboard
+            The name of the keyboard. Example: clueboard/66/rev3
+    """
+    for directory in ['.', '..', '../..', '../../..', '../../../..', '../../../../..']:
+        basepath = os.path.normpath(os.path.join('keyboards', keyboard, directory, 'keymaps'))
+
+        if os.path.exists(basepath):
+            return basepath
+
+    logging.error('Could not find keymaps directory!')
+    raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard)
+
+
+def normpath(path):
+    """Returns the fully resolved absolute path to a file.
+
+    This function will return the absolute path to a file as seen from the
+    directory the script was called from.
+    """
+    if path and path[0] == '/':
+        return os.path.normpath(path)
+
+    return os.path.normpath(os.path.join(os.environ['ORIG_CWD'], path))

requirements.txt

@@ -0,0 +1,5 @@
+# Python requirements
+# milc FIXME(skullydazed): Included in the repo for now.
+argcomplete
+colorama
+#halo

util/freebsd_install.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+util_dir=$(dirname "$0")
 pkg update
 pkg install -y \
 	git \
@@ -17,3 +18,4 @@ pkg install -y \
 	arm-none-eabi-newlib \
 	diffutils \
 	python3
+pip3 install -r ${util_dir}/../requirements.txt