fix after rebase

This allows us to skip validation when the file has not been changed
since the last time it was validated.

.github/labeler.yml

@@ -22,7 +22,6 @@ keymap:
 via:
   - keyboards/**/keymaps/via/*
 cli:
-  - bin/qmk
   - requirements.txt
   - lib/python/**/*
 python:

.github/workflows/cli.yml

@@ -8,7 +8,6 @@ on:
   pull_request:
     paths:
     - 'lib/python/**'
-    - 'bin/qmk'
     - 'requirements.txt'
     - '.github/workflows/cli.yml'
 

Makefile

@@ -30,11 +30,7 @@ endif
 endif
 
 # Determine which qmk cli to use
-ifeq (,$(shell which qmk))
-    QMK_BIN = bin/qmk
-else
-    QMK_BIN = qmk
-endif
+QMK_BIN := qmk
 
 # avoid 'Entering|Leaving directory' messages
 MAKEFLAGS += --no-print-directory

bin/qmk

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-"""CLI wrapper for running QMK commands.
-"""
-import os
-import sys
-from pathlib import Path
-
-# Add the QMK python libs to our path
-script_dir = Path(os.path.realpath(__file__)).parent
-qmk_dir = script_dir.parent
-python_lib_dir = Path(qmk_dir / 'lib' / 'python').resolve()
-sys.path.append(str(python_lib_dir))
-
-# Setup the CLI
-import milc  # noqa
-
-milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}Ψ{style_reset_all}'
-
-
-@milc.cli.entrypoint('QMK Helper Script')
-def qmk_main(cli):
-    """The function that gets run when no subcommand is provided.
-    """
-    cli.print_help()
-
-
-def main():
-    """Setup our environment and then call the CLI entrypoint.
-    """
-    # Change to the root of our checkout
-    os.environ['ORIG_CWD'] = os.getcwd()
-    os.environ['DEPRECATED_BIN_QMK'] = '1'
-    os.chdir(qmk_dir)
-
-    print('Warning: The bin/qmk script is being deprecated. Please install the QMK CLI: python3 -m pip install qmk', file=sys.stderr)
-
-    # Import the subcommands
-    import milc.subcommand.config  # noqa
-    import qmk.cli  # noqa
-
-    # Execute
-    return_code = milc.cli()
-
-    if return_code is False:
-        exit(1)
-
-    elif return_code is not True and isinstance(return_code, int):
-        if return_code < 0 or return_code > 255:
-            milc.cli.log.error('Invalid return_code: %d', return_code)
-            exit(255)
-
-        exit(return_code)
-
-    exit(0)
-
-
-if __name__ == '__main__':
-    main()

build_keyboard.mk

@@ -115,6 +115,7 @@ include $(INFO_RULES_MK)
 # Check for keymap.json first, so we can regenerate keymap.c
 include build_json.mk
 
+# Pull in keymap level rules.mk
 ifeq ("$(wildcard $(KEYMAP_PATH))", "")
     # Look through the possible keymap folders until we find a matching keymap.c
     ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
@@ -345,6 +346,7 @@ ifeq ("$(USER_NAME)","")
 endif
 USER_PATH := users/$(USER_NAME)
 
+# Pull in user level rules.mk
 -include $(USER_PATH)/rules.mk
 ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
@@ -356,6 +358,23 @@ endif
 # Disable features that a keyboard doesn't support
 -include disable_features.mk
 
+# Pull in post_rules.mk files from all our subfolders
+ifneq ("$(wildcard $(KEYBOARD_PATH_1)/post_rules.mk)","")
+    include $(KEYBOARD_PATH_1)/post_rules.mk
+endif
+ifneq ("$(wildcard $(KEYBOARD_PATH_2)/post_rules.mk)","")
+    include $(KEYBOARD_PATH_2)/post_rules.mk
+endif
+ifneq ("$(wildcard $(KEYBOARD_PATH_3)/post_rules.mk)","")
+    include $(KEYBOARD_PATH_3)/post_rules.mk
+endif
+ifneq ("$(wildcard $(KEYBOARD_PATH_4)/post_rules.mk)","")
+    include $(KEYBOARD_PATH_4)/post_rules.mk
+endif
+ifneq ("$(wildcard $(KEYBOARD_PATH_5)/post_rules.mk)","")
+    include $(KEYBOARD_PATH_5)/post_rules.mk
+endif
+
 ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif

docs/cli_commands.md

@@ -118,54 +118,6 @@ This command lets you configure the behavior of QMK. For the full `qmk config` d
 qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN]
 ```
 
-## `qmk console`
-
-This command lets you connect to keyboard consoles to get debugging messages. It only works if your keyboard firmware has been compiled with `CONSOLE_ENABLE=yes`.
-
-**Usage**:
-
-```
-qmk console [-d <pid>:<vid>[:<index>]] [-l] [-n] [-t] [-w <seconds>]
-```
-
-**Examples**:
-
-Connect to all available keyboards and show their console messages:
-
-```
-qmk console
-```
-
-List all devices:
-
-```
-qmk console -l
-```
-
-Show only messages from clueboard/66/rev3 keyboards:
-
-```
-qmk console -d C1ED:2370
-```
-
-Show only messages from the second clueboard/66/rev3:
-
-```
-qmk console -d C1ED:2370:2
-```
-
-Show timestamps and VID:PID instead of names:
-
-```
-qmk console -n -t
-```
-
-Disable bootloader messages:
-
-```
-qmk console --no-bootloaders
-```
-
 ## `qmk doctor`
 
 This command examines your environment and alerts you to potential build or flash problems. It can fix many of them if you want it to.

docs/contributing.md

@@ -105,7 +105,7 @@ enum my_keycodes {
 
 Before opening a pull request, you can preview your changes if you have set up the development environment by running this command from the `qmk_firmware/` folder:
 
-    ./bin/qmk docs
+    qmk docs
 
 or if you only have Python 3 installed:
 

docs/de/cli.md

@@ -51,25 +51,6 @@ Wir suchen nach Freiwilligen, die ein `qmk`-Package für weitere Betriebssysteme
 * Installiere mit einem [virtualenv](https://virtualenv.pypa.io/en/latest/).
 * Weise den User an, die Umgebungs-Variable `QMK_HOME` zu setzen, um die Firmware-Quelle anders einzustellen als `~/qmk_firmware`.
 
-# Lokale CLI
-
-Wenn Du die globale CLI nicht verwenden möchtest, beinhaltet `qmk_firmware` auch eine lokale CLI. Du kannst sie hier finden: `qmk_firmware/bin/qmk`. Du kannst den `qmk`-Befehl aus irgendeinem Datei-Verzeichnis ausführen und es wird immer auf dieser Kopie von `qmk_firmware` arbeiten.
-
-**Beispiel**:
-
-```
-$ ~/qmk_firmware/bin/qmk hello
-Ψ Hello, World!
-```
-
-## Einschränkungen der lokalen CLI
-
-Hier ein Vergleich mit der globalen CLI:
-
-* Die lokale CLI unterstützt kein `qmk setup` oder `qmk clone`.
-* Die lokale CLI arbeitet immer innerhalb der selben `qmk_firmware`-Verzeichnisstruktur, auch wenn Du mehrere Repositories geklont hast.
-* Die lokale CLI läuft nicht in einer virtualenv. Daher ist es möglich, dass Abhängigkeiten (dependencies) miteinander in Konflikt kommen/stehen.
-
 # CLI-Befehle
 
 ## `qmk compile`

docs/fr-fr/cli.md

@@ -48,25 +48,6 @@ Nous recherchons des gens pour créer et maintenir un paquet `qmk` pour plus de
 * Installez en utilisant un virtualenv
 * Expliquez à l'utilisateur de définir la variable d'environnement `QMK_Home` pour "check out" les sources du firmware à un autre endroit que `~/qmk_firmware`.
 
-# CLI locale
-
-Si vous ne voulez pas utiliser la CLI globale, il y a une CLI locale empaquetée avec `qmk_firmware`. Vous pouvez le trouver dans `qmk_firmware/bin/qmk`. Vous pouvez lancer la commande `qmk` depuis n'importe quel répertoire et elle fonctionnera toujours sur cette copie de `qmk_firmware`.
-
-**Exemple**:
-
-```
-$ ~/qmk_firmware/bin/qmk hello
-Ψ Hello, World!
-```
-
-## Limitations de la CLI locale
-
-Il y a quelques limitations à la CLI locale comparé à la globale:
-
-* La CLI locale ne supporte pas `qmk setup` ou `qmk clone`
-* La CLI locale n'opère pas sur le même arbre `qmk_firmware`, même si vous avez plusieurs dépôts clonés.
-* La CLI locale ne s'exécute pas dans un virtualenv, donc il y a des risques que des dépendances seront en conflit
-
 # Les commandes CLI
 
 ## `qmk compile`

docs/hardware_keyboard_guidelines.md

@@ -144,10 +144,38 @@ The `rules.mk` file can also be placed in a sub-folder, and its reading order is
         * `keyboards/top_folder/sub_1/sub_2/sub_3/sub_4/rules.mk`
           * `keyboards/top_folder/keymaps/a_keymap/rules.mk`
           * `users/a_user_folder/rules.mk`
+        * `keyboards/top_folder/sub_1/sub_2/sub_3/sub_4/post_rules.mk`
+      * `keyboards/top_folder/sub_1/sub_2/sub_3/post_rules.mk`
+    * `keyboards/top_folder/sub_1/sub_2/post_rules.mk`
+  * `keyboards/top_folder/sub_1/post_rules.mk`
+* `keyboards/top_folder/post_rules.mk`
 * `common_features.mk`
 
 Many of the settings written in the `rules.mk` file are interpreted by `common_features.mk`, which sets the necessary source files and compiler options.
 
+The `post_rules.mk` file can interpret `features` of a keyboard-level before `common_features.mk`.  For example, when your designed keyboard has the option to implement backlighting or underglow using rgblight.c, writing the following in the `post_rules.mk` makes it easier for the user to configure the `rules.mk`.
+
+* `keyboards/top_folder/keymaps/a_keymap/rules.mk`
+  ```makefile
+  # Please set the following according to the selection of the hardware implementation option.
+  RGBLED_OPTION_TYPE = backlight   ## none, backlight or underglow
+  ```
+* `keyboards/top_folder/post_rules.mk`
+  ```makefile
+  ifeq ($(filter $(strip $(RGBLED_OPTION_TYPE))x, nonex backlightx underglowx x),)
+     $(error unknown RGBLED_OPTION_TYPE value "$(RGBLED_OPTION_TYPE)")
+  endif
+
+  ifeq ($(strip $(RGBLED_OPTION_TYPE)),backlight)
+    RGBLIGHT_ENABLE = yes
+    OPT_DEFS += -DRGBLED_NUM=30
+  endif
+  ifeq ($(strip $(RGBLED_OPTION_TYPE)),underglow)
+    RGBLIGHT_ENABLE = yes
+    OPT_DEFS += -DRGBLED_NUM=6
+  endif
+  ```
+
 ?> See `build_keyboard.mk` and `common_features.mk` for more details.
 
 ### `<keyboard_name.c>`

keyboards/dztech/dz65rgb/keymaps/jumper149/readme.md

@@ -3,13 +3,13 @@
 Run commands in the root directory of this repository.
 
 ```
-./bin/qmk compile && sudo dfu-programmer atmega32u4 erase && sudo dfu-programmer atmega32u4 flash ./dztech_dz65rgb_v2_jumper149.hex && sudo dfu-programmer atmega32u4 reset
+qmk compile && sudo dfu-programmer atmega32u4 erase && sudo dfu-programmer atmega32u4 flash ./dztech_dz65rgb_v2_jumper149.hex && sudo dfu-programmer atmega32u4 reset
 ```
 
 ## build
 
 ```
-./bin/qmk compile
+qmk compile
 ```
 
 ## flash

lib/python/qmk/c_parse.py

@@ -1,7 +1,8 @@
 """Functions for working with config.h files.
 """
-from pathlib import Path
 import re
+from functools import lru_cache
+from pathlib import Path
 
 from milc import cli
 
@@ -12,18 +13,21 @@ single_comment_regex = re.compile(r'\s+/[/*].*$')
 multi_comment_regex = re.compile(r'/\*(.|\n)*?\*/', re.MULTILINE)
 
 
+@lru_cache(maxsize=0)
 def strip_line_comment(string):
     """Removes comments from a single line string.
     """
     return single_comment_regex.sub('', string)
 
 
+@lru_cache(maxsize=0)
 def strip_multiline_comment(string):
     """Removes comments from a single line string.
     """
     return multi_comment_regex.sub('', string)
 
 
+@lru_cache(maxsize=0)
 def c_source_files(dir_names):
     """Returns a list of all *.c, *.h, and *.cpp files for a given list of directories
 
@@ -38,6 +42,7 @@ def c_source_files(dir_names):
     return files
 
 
+@lru_cache(maxsize=0)
 def find_layouts(file):
     """Returns list of parsed LAYOUT preprocessor macros found in the supplied include file.
     """
@@ -144,6 +149,7 @@ def _default_key(label=None):
     return new_key
 
 
+@lru_cache(maxsize=0)
 def _parse_layout_macro(layout_macro):
     """Split the LAYOUT macro into its constituent parts
     """
@@ -154,6 +160,7 @@ def _parse_layout_macro(layout_macro):
     return macro_name, layout, matrix
 
 
+@lru_cache(maxsize=0)
 def _parse_matrix_locations(matrix, file, macro_name):
     """Parse raw matrix data into a dictionary keyed by the LAYOUT identifier.
     """

lib/python/qmk/cli/__init__.py

@@ -35,7 +35,6 @@ subcommands = [
     'qmk.cli.chibios.confmigrate',
     'qmk.cli.clean',
     'qmk.cli.compile',
-    'qmk.cli.console',
     'qmk.cli.docs',
     'qmk.cli.doctor',
     'qmk.cli.fileformat',

lib/python/qmk/cli/clean.py

@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import _find_make
 from milc import cli
 
 
@@ -11,4 +11,6 @@ from milc import cli
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean']
+
+    cli.run(make_cmd, capture_output=False, stdin=DEVNULL)

lib/python/qmk/cli/compile.py

@@ -2,23 +2,24 @@
 
 You can compile a keymap already in the repo or using a QMK Configurator export.
 """
-from subprocess import DEVNULL
-
 from argcomplete.completers import FilesCompleter
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.commands import do_compile
+from qmk.keyboard import keyboard_completer, is_keyboard_target
 from qmk.keymap import keymap_completer
+from qmk.metadata import true_values, false_values
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-t', '--target', help="The make target to run. By default it compiles the keyboard only.")
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
-@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.subcommand('Compile a QMK Firmware.')
@@ -31,47 +32,31 @@ def compile(cli):
 
     If a keyboard and keymap are provided this command will build a firmware based on that.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
-        cli.run(command, capture_output=False, stdin=DEVNULL)
+    # If -f has been specified without a keyboard target, assume -kb all
+    keyboard = cli.config.compile.keyboard or ''
 
-    # Build the environment vars
-    envs = {}
-    for env in cli.args.env:
-        if '=' in env:
-            key, value = env.split('=', 1)
-            envs[key] = value
-        else:
-            cli.log.warning('Invalid environment variable: %s', env)
+    if cli.args.filter and not cli.args.keyboard:
+        cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.')
+        keyboard = 'all'
 
-    # Determine the compile command
-    command = None
+    if cli.args.filename and cli.args.filter:
+        cli.log.warning('Ignoring --filter because a keymap.json was provided.')
 
-    if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
-        user_keymap = parse_configurator_json(cli.args.filename)
-        command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
+    filters = {}
 
-    else:
-        if cli.config.compile.keyboard and cli.config.compile.keymap:
-            # Generate the make command for a specific keyboard/keymap.
-            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
+    for filter in cli.args.filter:
+        if '=' in filter:
+            key, value = filter.split('=', 1)
 
-        elif not cli.config.compile.keyboard:
-            cli.log.error('Could not determine keyboard!')
-        elif not cli.config.compile.keymap:
-            cli.log.error('Could not determine keymap!')
+            if value in true_values:
+                value = True
+            elif value in false_values:
+                value = False
+            elif value.isdigit():
+                value = int(value)
+            elif '.' in value and value.replace('.').isdigit():
+                value = float(value)
 
-    # Compile the firmware, if we're able to
-    if command:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-        if not cli.args.dry_run:
-            cli.echo('\n')
-            # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
-            compile = cli.run(command, capture_output=False, text=False)
-            return compile.returncode
+            filters[key] = value
 
-    else:
-        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
-        return False
+    return do_compile(keyboard, cli.config.compile.keymap, cli.config.compile.parallel, cli.config.compile.target, filters)

lib/python/qmk/cli/doctor/check.py

@@ -26,7 +26,6 @@ ESSENTIAL_BINARIES = {
     'arm-none-eabi-gcc': {
         'version_arg': '-dumpversion'
     },
-    'bin/qmk': {},
 }
 
 

lib/python/qmk/cli/flash.py

@@ -3,15 +3,13 @@
 You can compile a keymap already in the repo or using a QMK Configurator export.
 A bootloader must be specified.
 """
-from subprocess import DEVNULL
-
 from argcomplete.completers import FilesCompleter
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.commands import do_compile
+from qmk.keyboard import keyboard_completer, is_keyboard_target
 
 
 def print_bootloader_help():
@@ -36,7 +34,7 @@ def print_bootloader_help():
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -54,55 +52,10 @@ def flash(cli):
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
-        cli.run(command, capture_output=False, stdin=DEVNULL)
-
-    # Build the environment vars
-    envs = {}
-    for env in cli.args.env:
-        if '=' in env:
-            key, value = env.split('=', 1)
-            envs[key] = value
-        else:
-            cli.log.warning('Invalid environment variable: %s', env)
-
-    # Determine the compile command
-    command = ''
-
     if cli.args.bootloaders:
         # Provide usage and list bootloaders
-        cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+        cli.print_usage()
         print_bootloader_help()
         return False
 
-    if cli.args.filename:
-        # Handle compiling a configurator JSON
-        user_keymap = parse_configurator_json(cli.args.filename)
-        keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-        command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-        cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
-
-    else:
-        if cli.config.flash.keyboard and cli.config.flash.keymap:
-            # Generate the make command for a specific keyboard/keymap.
-            command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-        elif not cli.config.flash.keyboard:
-            cli.log.error('Could not determine keyboard!')
-        elif not cli.config.flash.keymap:
-            cli.log.error('Could not determine keymap!')
-
-    # Compile the firmware, if we're able to
-    if command:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-        if not cli.args.dry_run:
-            cli.echo('\n')
-            compile = cli.run(command, capture_output=False, stdin=DEVNULL)
-            return compile.returncode
-
-    else:
-        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-        return False
+    return do_compile(cli.config.flash.keyboard, cli.config.flash.keymap, cli.config.flash.parallel, cli.config.flash.bootloader)

lib/python/qmk/cli/format/python.py

@@ -11,15 +11,15 @@ def format_python(cli):
     """Format python code according to QMK's style.
     """
     edit = '--diff' if cli.args.dry_run else '--in-place'
-    yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python']
+    yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'lib/python']
     try:
         cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL)
-        cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.')
+        cli.log.info('Python code in `lib/python` is correctly formatted.')
         return True
 
     except CalledProcessError:
         if cli.args.dry_run:
-            cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!')
+            cli.log.error('Python code in `lib/python` is incorrectly formatted!')
         else:
             cli.log.error('Error formatting python code!')
 

lib/python/qmk/cli/lint.py

@@ -1,72 +1,129 @@
 """Command to look over a keyboard/keymap and check for common mistakes.
 """
+from pathlib import Path
+
 from milc import cli
 
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
-from qmk.keyboard import find_readme, keyboard_completer
+from qmk.keyboard import keyboard_completer, list_keyboards
 from qmk.keymap import locate_keymap
 from qmk.path import is_keyboard, keyboard
 
 
-@cli.argument('--strict', action='store_true', help='Treat warnings as errors.')
-@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='The keyboard to check.')
-@cli.argument('-km', '--keymap', help='The keymap to check.')
+def keymap_check(kb, km):
+    """Perform the keymap level checks.
+    """
+    ok = True
+    keymap_path = locate_keymap(kb, km)
+
+    if not keymap_path:
+        ok = False
+        cli.log.error("%s: Can't find %s keymap.", kb, km)
+
+    return ok
+
+
+def rules_mk_assignment_only(keyboard_path):
+    """Check the keyboard-level rules.mk to ensure it only has assignments.
+    """
+    current_path = Path()
+    errors = []
+
+    for path_part in keyboard_path.parts:
+        current_path = current_path / path_part
+        rules_mk = current_path / 'rules.mk'
+
+        if rules_mk.exists():
+            continuation = None
+
+            for i, line in enumerate(rules_mk.open()):
+                line = line.strip()
+
+                if '#' in line:
+                    line = line[:line.index('#')]
+
+                if continuation:
+                    line = continuation + line
+                    continuation = None
+
+                if line:
+                    if line[-1] == '\\':
+                        continuation = line[:-1]
+                        continue
+
+                    if line and '=' not in line:
+                        errors.append(f'Non-assignment code on line +{i} {rules_mk}: {line}')
+
+    return errors
+
+
+@cli.argument('--strict', action='store_true', help='Treat warnings as errors')
+@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='Comma separated list of keyboards to check')
+@cli.argument('-km', '--keymap', help='The keymap to check')
+@cli.argument('--all-kb', action='store_true', arg_only=True, help='Check all keyboards')
 @cli.subcommand('Check keyboard and keymap for common mistakes.')
 @automagic_keyboard
 @automagic_keymap
 def lint(cli):
     """Check keyboard and keymap for common mistakes.
     """
-    if not cli.config.lint.keyboard:
-        cli.log.error('Missing required argument: --keyboard')
+    failed = []
+
+    # Determine our keyboard list
+    if cli.args.all_kb:
+        if cli.args.keyboard:
+            cli.log.warning('Both --all-kb and --keyboard passed, --all-kb takes presidence.')
+
+        keyboard_list = list_keyboards()
+    elif not cli.config.lint.keyboard:
+        cli.log.error('Missing required arguments: --keyboard or --all-kb')
         cli.print_help()
         return False
+    else:
+        keyboard_list = cli.config.lint.keyboard.split(',')
 
-    if not is_keyboard(cli.config.lint.keyboard):
-        cli.log.error('No such keyboard: %s', cli.config.lint.keyboard)
-        return False
+    # Lint each keyboard
+    for kb in keyboard_list:
+        if not is_keyboard(kb):
+            cli.log.error('No such keyboard: %s', kb)
+            continue
 
-    # Gather data about the keyboard.
-    ok = True
-    keyboard_path = keyboard(cli.config.lint.keyboard)
-    keyboard_info = info_json(cli.config.lint.keyboard)
-    readme_path = find_readme(cli.config.lint.keyboard)
-    missing_readme_path = keyboard_path / 'readme.md'
+        # Gather data about the keyboard.
+        ok = True
+        keyboard_path = keyboard(kb)
+        keyboard_info = info_json(kb)
 
-    # Check for errors in the info.json
-    if keyboard_info['parse_errors']:
-        ok = False
-        cli.log.error('Errors found when generating info.json.')
-
-    if cli.config.lint.strict and keyboard_info['parse_warnings']:
-        ok = False
-        cli.log.error('Warnings found when generating info.json (Strict mode enabled.)')
-
-    # Check for a readme.md and warn if it doesn't exist
-    if not readme_path:
-        ok = False
-        cli.log.error('Missing %s', missing_readme_path)
-
-    # Keymap specific checks
-    if cli.config.lint.keymap:
-        keymap_path = locate_keymap(cli.config.lint.keyboard, cli.config.lint.keymap)
-
-        if not keymap_path:
+        # Check for errors in the info.json
+        if keyboard_info['parse_errors']:
             ok = False
-            cli.log.error("Can't find %s keymap for %s keyboard.", cli.config.lint.keymap, cli.config.lint.keyboard)
-        else:
-            keymap_readme = keymap_path.parent / 'readme.md'
-            if not keymap_readme.exists():
-                cli.log.warning('Missing %s', keymap_readme)
+            cli.log.error('%s: Errors found when generating info.json.', kb)
 
-                if cli.config.lint.strict:
-                    ok = False
+        if cli.config.lint.strict and keyboard_info['parse_warnings']:
+            ok = False
+            cli.log.error('%s: Warnings found when generating info.json (Strict mode enabled.)', kb)
+
+        # Check the rules.mk file(s)
+        rules_mk_assignment_errors = rules_mk_assignment_only(keyboard_path)
+        if rules_mk_assignment_errors:
+            ok = False
+            cli.log.error('%s: Non-assignment code found in rules.mk. Move it to post_rules.mk instead.', kb)
+            for assignment_error in rules_mk_assignment_errors:
+                cli.log.error(assignment_error)
+
+        # Keymap specific checks
+        if cli.config.lint.keymap:
+            if not keymap_check(kb, cli.config.lint.keymap):
+                ok = False
+
+        # Report status
+        if not ok:
+            failed.append(kb)
 
     # Check and report the overall status
-    if ok:
-        cli.log.info('Lint check passed!')
-        return True
+    if failed:
+        cli.log.error('Lint check failed for: %s', ', '.join(failed))
+        return False
 
-    cli.log.error('Lint check failed!')
-    return False
+    cli.log.info('Lint check passed!')
+    return True

lib/python/qmk/cli/pytest.py

@@ -12,6 +12,6 @@ def pytest(cli):
     """Run several linting/testing commands.
     """
     nose2 = cli.run(['nose2', '-v'], capture_output=False, stdin=DEVNULL)
-    flake8 = cli.run(['flake8', 'lib/python', 'bin/qmk'], capture_output=False, stdin=DEVNULL)
+    flake8 = cli.run(['flake8', 'lib/python'], capture_output=False, stdin=DEVNULL)
 
     return flake8.returncode | nose2.returncode

lib/python/qmk/commands.py

@@ -1,22 +1,28 @@
 """Helper functions for commands.
 """
+from functools import lru_cache
 import json
 import os
 import sys
 import shutil
+import threading
 from pathlib import Path
 from subprocess import DEVNULL
-from time import strftime
+from time import sleep, strftime
 
+from dotty_dict import dotty
 from milc import cli
 
 import qmk.keymap
 from qmk.constants import QMK_FIRMWARE, KEYBOARD_OUTPUT_PREFIX
+from qmk.info import info_json
 from qmk.json_schema import json_load
+from qmk.keyboard import list_keyboards
 
 time_fmt = '%Y-%m-%d-%H:%M:%S'
 
 
+@lru_cache(maxsize=0)
 def _find_make():
     """Returns the correct make command for this environment.
     """
@@ -55,7 +61,7 @@ def create_make_target(target, parallel=1, **env_vars):
     return [make_cmd, *get_make_parallel_args(parallel), *env, target]
 
 
-def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
+def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars):
     """Create a make compile command
 
     Args:
@@ -79,14 +85,29 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
 
         A command that can be run to make the specified keyboard and keymap
     """
-    make_args = [keyboard, keymap]
+    make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk']
+
+    env_vars['KEYBOARD'] = keyboard
+    env_vars['KEYMAP'] = keymap
+    env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk'
+    env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else ''
+    env_vars['SILENT'] = 'true' if silent else 'false'
+    env_vars['COLOR'] = 'true' if cli.config.general.color else ''
+
+    if parallel > 1:
+        make_cmd.append('-j')
+        make_cmd.append(str(parallel))
 
     if target:
-        make_args.append(target)
+        make_cmd.append(target)
 
-    return create_make_target(':'.join(make_args), parallel, **env_vars)
+    for key, value in env_vars.items():
+        make_cmd.append(f'{key}={value}')
+
+    return make_cmd
 
 
+@lru_cache(maxsize=0)
 def get_git_version(current_time, repo_dir='.', check_dir='.'):
     """Returns the current git version for a repo, or the current time.
     """
@@ -233,12 +254,13 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
         f'VERBOSE={verbose}',
         f'COLOR={color}',
         'SILENT=false',
-        f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}',
+        'QMK_BIN="qmk"',
     ])
 
-    return make_command
+    return user_keymap['keyboard'], user_keymap['keymap'], make_command
 
 
+@lru_cache(maxsize=0)
 def parse_configurator_json(configurator_file):
     """Open and parse a configurator json export
     """
@@ -332,3 +354,165 @@ def in_virtualenv():
     """
     active_prefix = getattr(sys, "base_prefix", None) or getattr(sys, "real_prefix", None) or sys.prefix
     return active_prefix != sys.prefix
+
+
+def do_compile(keyboard, keymap, parallel, target=None, filters=None, environment=None):
+    """Shared code between `qmk compile` and `qmk flash`.
+    """
+    if keyboard is None:
+        keyboard = ''
+
+    if environment is None:
+        environment = {}
+
+    all_keyboards = keyboard == 'all' or keyboard.startswith('all-')
+    all_keymaps = keymap == 'all'
+    multiple_compiles = all_keyboards or all_keymaps
+
+    # Setup the environment
+    envs = {'REQUIRE_PLATFORM_KEY': ''}
+
+    for env in environment:
+        if '=' in env:
+            key, value = env.split('=', 1)
+            if key in envs:
+                cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value)
+            envs[key] = value
+        else:
+            cli.log.warning('Invalid environment variable: %s', env)
+
+    if keyboard.startswith('all-'):
+        envs['REQUIRE_PLATFORM_KEY'] = keyboard[4:]
+
+    # Run clean if necessary
+    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+        for kb, km in keyboard_keymap_iter(keyboard, keymap, {}):
+            cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', kb, km)
+            make_cmd = create_make_command(kb, km, 'clean', 1, multiple_compiles, **envs)
+            cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
+
+    # Determine the compile command(s)
+    command = None
+
+    if cli.args.filename:
+        if cli.args.keyboard:
+            cli.log.warning('Ignoring --keyboard because a keymap.json was provided.')
+
+        if cli.args.keymap:
+            cli.log.warning('Ignoring --keymap because a keymap.json was provided.')
+
+        # If a configurator JSON was provided generate a keymap and compile it
+        user_keymap = parse_configurator_json(cli.args.filename)
+        command = compile_configurator_json(user_keymap, parallel=parallel, **envs)
+
+    elif keyboard and keymap:
+        if multiple_compiles:
+            command = 'multiple'
+        else:
+            command = create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=multiple_compiles, **envs)
+
+    elif not keyboard:
+        cli.log.error('Could not determine keyboard!')
+
+    elif not keymap:
+        cli.log.error('Could not determine keymap!')
+
+    # Compile the firmware, if we're able to
+    if command == 'multiple':
+        cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
+
+        returncodes = []
+        for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters):
+            command = create_make_command(keyboard, keymap, target=target, parallel=1, silent=multiple_compiles, **envs)
+            while threading.active_count() >= parallel + 1:
+                sleep(1)
+            threading.Thread(target=_execute_compile, args=(keyboard, keymap, command, target, returncodes)).start()
+
+        while threading.active_count() > 1:
+            sleep(1)
+
+        if any(returncodes):
+            print()
+            cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
+
+            for i, returncode in enumerate(returncodes):
+                if returncode != 0:
+                    keyboard, keymap, command = returncodes[i]
+                    cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
+
+    elif command:
+        if target:
+            cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target)
+        else:
+            cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
+
+        if _execute_compile(keyboard, keymap, command, target) != 0:
+            print()
+            cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
+            cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
+
+    elif filters:
+        cli.log.error('No keyboards found after applying filter(s)!')
+        return False
+
+    else:
+        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
+        cli.print_help()
+        return False
+
+
+def _execute_compile(keyboard, keymap, command, target, returncodes=None):
+    if not returncodes:
+        returncodes = []
+
+    if keymap not in qmk.keymap.list_keymaps(keyboard):
+        cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap)
+        return 0
+
+    cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command))
+
+    if not cli.args.dry_run:
+        compile = cli.run(command, combined_output=True)
+
+        cli.acquire_lock()
+        returncodes.append(compile.returncode)
+        cli.release_lock()
+
+        if compile.returncode != 0:
+            cli.log.info('Could not build firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
+            print(compile.stdout)
+
+
+@lru_cache()
+def _keyboard_list(keyboard):
+    """Returns a list of keyboards matching keyboard.
+    """
+    if keyboard == 'all' or keyboard.startswith('all-'):
+        return list_keyboards()
+
+    return [keyboard]
+
+
+def keyboard_keymap_iter(cli_keyboard, cli_keymap, filters):
+    """Iterates over the keyboard/keymap for this command and yields a pairing of each.
+    """
+    for keyboard in _keyboard_list(cli_keyboard):
+        continue_flag = False
+
+        if filters:
+            info_data = dotty(info_json(keyboard))
+
+            for key, value in filters.items():
+                if info_data.get(key) != value:
+                    continue_flag = True
+                    break
+
+            if continue_flag:
+                continue
+
+        if cli_keymap == 'all':
+            for keymap in qmk.keymap.list_keymaps(keyboard):
+                yield keyboard, keymap
+
+        else:
+            yield keyboard, cli_keymap

lib/python/qmk/comment_remover.py

@@ -3,6 +3,7 @@
 Gratefully adapted from https://stackoverflow.com/a/241506
 """
 import re
+from functools import lru_cache
 
 comment_pattern = re.compile(r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE)
 
@@ -14,6 +15,7 @@ def _comment_stripper(match):
     return ' ' if s.startswith('/') else s
 
 
+@lru_cache(maxsize=0)
 def comment_remover(text):
     """Remove C/C++ style comments from text.
     """

lib/python/qmk/converter.py

@@ -1,8 +1,10 @@
 """Functions to convert to and from QMK formats
 """
 from collections import OrderedDict
+from functools import lru_cache
 
 
+@lru_cache(maxsize=0)
 def kle2qmk(kle):
     """Convert a KLE layout to QMK's layout format.
     """

lib/python/qmk/info.py

@@ -1,5 +1,6 @@
 """Functions that help us generate and use info.json files.
 """
+from functools import lru_cache
 from glob import glob
 from pathlib import Path
 
@@ -12,37 +13,22 @@ from qmk.c_parse import find_layouts
 from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
 from qmk.keymap import list_keymaps
-from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
-
-true_values = ['1', 'on', 'yes']
-false_values = ['0', 'off', 'no']
+from qmk.metadata import basic_info_json, info_log_error, info_log_warning, true_values, false_values
 
 
+@lru_cache(maxsize=None)
 def _valid_community_layout(layout):
     """Validate that a declared community list exists
     """
     return (Path('layouts/default') / layout).exists()
 
 
+@lru_cache(maxsize=None)
 def info_json(keyboard):
     """Generate the info.json data for a specific keyboard.
     """
-    cur_dir = Path('keyboards')
-    rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
-    if 'DEFAULT_FOLDER' in rules:
-        keyboard = rules['DEFAULT_FOLDER']
-        rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
-
-    info_data = {
-        'keyboard_name': str(keyboard),
-        'keyboard_folder': str(keyboard),
-        'keymaps': {},
-        'layouts': {},
-        'parse_errors': [],
-        'parse_warnings': [],
-        'maintainer': 'qmk',
-    }
+    info_data = basic_info_json(keyboard)
 
     # Populate the list of JSON keymaps
     for keymap in list_keymaps(keyboard, c=False, fullpath=True):
@@ -81,20 +67,20 @@ def info_json(keyboard):
         _find_missing_layouts(info_data, keyboard)
 
     if not info_data.get('layouts'):
-        _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
+        info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
 
     # Filter out any non-existing community layouts
     for layout in info_data.get('community_layouts', []):
         if not _valid_community_layout(layout):
             # Ignore layout from future checks
             info_data['community_layouts'].remove(layout)
-            _log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
+            info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
 
     # Make sure we supply layout macros for the community layouts we claim to support
     for layout in info_data.get('community_layouts', []):
         layout_name = 'LAYOUT_' + layout
         if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
-            _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
+            info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
 
     # Check that the reported matrix size is consistent with the actual matrix size
     _check_matrix(info_data)
@@ -130,7 +116,7 @@ def _extract_features(info_data, rules):
                 info_data['features'] = {}
 
             if key in info_data['features']:
-                _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
+                info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
 
             info_data['features'][key] = value
             info_data['config_h_features'][key] = value
@@ -209,7 +195,7 @@ def _extract_split_main(info_data, config_c):
             info_data['split'] = {}
 
         if 'main' in info_data['split']:
-            _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
+            info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
 
         info_data['split']['main'] = 'pin'
 
@@ -218,7 +204,7 @@ def _extract_split_main(info_data, config_c):
             info_data['split'] = {}
 
         if 'main' in info_data['split']:
-            _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
+            info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
 
         info_data['split']['main'] = 'matrix_grid'
         info_data['split']['matrix_grid'] = _extract_pins(config_c['SPLIT_HAND_MATRIX_GRID'])
@@ -228,7 +214,7 @@ def _extract_split_main(info_data, config_c):
             info_data['split'] = {}
 
         if 'main' in info_data['split']:
-            _log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
+            info_log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
 
         info_data['split']['main'] = 'eeprom'
 
@@ -237,7 +223,7 @@ def _extract_split_main(info_data, config_c):
             info_data['split'] = {}
 
         if 'main' in info_data['split']:
-            _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
+            info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
 
         info_data['split']['main'] = 'right'
 
@@ -246,7 +232,7 @@ def _extract_split_main(info_data, config_c):
             info_data['split'] = {}
 
         if 'main' in info_data['split']:
-            _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
+            info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
 
         info_data['split']['main'] = 'left'
 
@@ -261,7 +247,7 @@ def _extract_split_transport(info_data, config_c):
             info_data['split']['transport'] = {}
 
         if 'protocol' in info_data['split']['transport']:
-            _log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport'])
+            info_log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport'])
 
         info_data['split']['transport']['protocol'] = 'i2c'
 
@@ -285,7 +271,7 @@ def _extract_split_right_pins(info_data, config_c):
 
     if row_pins and col_pins:
         if info_data.get('split', {}).get('matrix_pins', {}).get('right') in info_data:
-            _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
 
         if 'split' not in info_data:
             info_data['split'] = {}
@@ -303,7 +289,7 @@ def _extract_split_right_pins(info_data, config_c):
 
     if direct_pins:
         if info_data.get('split', {}).get('matrix_pins', {}).get('right', {}):
-            _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
 
         if 'split' not in info_data:
             info_data['split'] = {}
@@ -341,7 +327,7 @@ def _extract_matrix_info(info_data, config_c):
 
     if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
         if 'matrix_size' in info_data:
-            _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
 
         info_data['matrix_size'] = {
             'cols': compute(config_c.get('MATRIX_COLS', '0')),
@@ -350,14 +336,14 @@ def _extract_matrix_info(info_data, config_c):
 
     if row_pins and col_pins:
         if 'matrix_pins' in info_data and 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']:
-            _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
 
         info_snippet['cols'] = _extract_pins(col_pins)
         info_snippet['rows'] = _extract_pins(row_pins)
 
     if direct_pins:
         if 'matrix_pins' in info_data and 'direct' in info_data['matrix_pins']:
-            _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
 
         info_snippet['direct'] = _extract_direct_matrix(direct_pins)
 
@@ -369,7 +355,7 @@ def _extract_matrix_info(info_data, config_c):
 
     if config_c.get('CUSTOM_MATRIX', 'no') != 'no':
         if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']:
-            _log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.')
+            info_log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.')
 
         info_snippet['custom'] = True
 
@@ -398,7 +384,7 @@ def _extract_config_h(info_data):
         try:
             if config_key in config_c and info_dict.get('to_json', True):
                 if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
-                    _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
+                    info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
 
                 if key_type.startswith('array'):
                     if '.' in key_type:
@@ -429,7 +415,7 @@ def _extract_config_h(info_data):
                     dotty_info[info_key] = config_c[config_key]
 
         except Exception as e:
-            _log_warning(info_data, f'{config_key}->{info_key}: {e}')
+            info_log_warning(info_data, f'{config_key}->{info_key}: {e}')
 
     info_data.update(dotty_info)
 
@@ -470,7 +456,7 @@ def _extract_rules_mk(info_data):
         try:
             if rules_key in rules and info_dict.get('to_json', True):
                 if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
-                    _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
+                    info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
 
                 if key_type.startswith('array'):
                     if '.' in key_type:
@@ -501,7 +487,7 @@ def _extract_rules_mk(info_data):
                     dotty_info[info_key] = rules[rules_key]
 
         except Exception as e:
-            _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
+            info_log_warning(info_data, f'{rules_key}->{info_key}: {e}')
 
     info_data.update(dotty_info)
 
@@ -544,11 +530,11 @@ def _check_matrix(info_data):
 
         if col_count != actual_col_count and col_count != (actual_col_count / 2):
             # FIXME: once we can we should detect if split is enabled to do the actual_col_count/2 check.
-            _log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
+            info_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
 
         if row_count != actual_row_count and row_count != (actual_row_count / 2):
             # FIXME: once we can we should detect if split is enabled to do the actual_row_count/2 check.
-            _log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
+            info_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
 
 
 def _search_keyboard_h(keyboard):
@@ -577,7 +563,7 @@ def _find_missing_layouts(info_data, keyboard):
 
     If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
     """
-    _log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
+    info_log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
 
     for file in glob('keyboards/%s/*.h' % keyboard):
         these_layouts, these_aliases = find_layouts(file)
@@ -596,20 +582,6 @@ def _find_missing_layouts(info_data, keyboard):
                 info_data['layout_aliases'][alias] = alias_text
 
 
-def _log_error(info_data, message):
-    """Send an error message to both JSON and the log.
-    """
-    info_data['parse_errors'].append(message)
-    cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
-
-
-def _log_warning(info_data, message):
-    """Send a warning message to both JSON and the log.
-    """
-    info_data['parse_warnings'].append(message)
-    cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
-
-
 def arm_processor_rules(info_data, rules):
     """Setup the default info for an ARM board.
     """
@@ -668,7 +640,7 @@ def merge_info_jsons(keyboard, info_data):
         new_info_data = json_load(info_file)
 
         if not isinstance(new_info_data, dict):
-            _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
+            info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
             continue
 
         try:
@@ -686,13 +658,13 @@ def merge_info_jsons(keyboard, info_data):
 
         for layout_name, layout in new_info_data.get('layouts', {}).items():
             if layout_name in info_data.get('layout_aliases', {}):
-                _log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
+                info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
                 layout_name = info_data['layout_aliases'][layout_name]
 
             if layout_name in info_data['layouts']:
                 if len(info_data['layouts'][layout_name]['layout']) != len(layout['layout']):
                     msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
-                    _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
+                    info_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
                 else:
                     for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
                         existing_key.update(new_key)
@@ -720,15 +692,18 @@ def find_info_json(keyboard):
 
     # Add DEFAULT_FOLDER before parents, if present
     rules = rules_mk(keyboard)
+
     if 'DEFAULT_FOLDER' in rules:
         info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
 
     # Add in parent folders for least specific
     for _ in range(5):
-        info_jsons.append(keyboard_parent / 'info.json')
+        this_info_json = keyboard_parent / 'info.json'
+
+        if this_info_json.exists():
+            yield this_info_json
+
         if keyboard_parent.parent == base_path:
             break
-        keyboard_parent = keyboard_parent.parent
 
-    # Return a list of the info.json files that actually exist
-    return [info_json for info_json in info_jsons if info_json.exists()]
+        keyboard_parent = keyboard_parent.parent

lib/python/qmk/json_schema.py

@@ -10,6 +10,7 @@ import jsonschema
 from milc import cli
 
 
+@lru_cache(maxsize=0)
 def json_load(json_file):
     """Load a json file from disk.
 
@@ -23,6 +24,8 @@ def json_load(json_file):
         exit(1)
     except Exception as e:
         cli.log.error('Unknown error attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
+        if cli.args.verbose:
+            cli.log.exception(e)
         exit(1)
 
 

lib/python/qmk/keyboard.py

@@ -1,10 +1,11 @@
 """Functions that help us work with keyboards.
 """
+import os
 from array import array
+from functools import lru_cache
+from glob import glob
 from math import ceil
 from pathlib import Path
-import os
-from glob import glob
 
 import qmk.path
 from qmk.c_parse import parse_config_h_file
@@ -64,6 +65,17 @@ def find_readme(keyboard):
     return cur_dir / 'readme.md'
 
 
+def is_keyboard_target(keyboard_target):
+    """Checks to make sure the supplied keyboard_target is valid.
+
+    This is mainly used by commands that accept --keyboard.
+    """
+    if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']:
+        return keyboard_target
+
+    return keyboard_folder(keyboard_target)
+
+
 def keyboard_folder(keyboard):
     """Returns the actual keyboard folder.
 
@@ -206,6 +218,7 @@ def render_layout(layout_data, render_ascii, key_labels=None):
     return '\n'.join(lines)
 
 
+@lru_cache(maxsize=0)
 def render_layouts(info_json, render_ascii):
     """Renders all the layouts from an `info_json` structure.
     """

lib/python/qmk/keymap.py

@@ -2,6 +2,7 @@
 """
 import json
 import sys
+from functools import lru_cache
 from pathlib import Path
 from subprocess import DEVNULL
 
@@ -14,6 +15,7 @@ from pygments import lex
 import qmk.path
 from qmk.keyboard import find_keyboard_from_dir, rules_mk
 from qmk.errors import CppError
+from qmk.metadata import basic_info_json
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -30,6 +32,7 @@ __KEYMAP_GOES_HERE__
 """
 
 
+@lru_cache(maxsize=0)
 def template_json(keyboard):
     """Returns a `keymap.json` template for a keyboard.
 
@@ -47,6 +50,7 @@ def template_json(keyboard):
     return template
 
 
+@lru_cache(maxsize=0)
 def template_c(keyboard):
     """Returns a `keymap.c` template for a keyboard.
 
@@ -122,6 +126,7 @@ def keymap_completer(prefix, action, parser, parsed_args):
     return []
 
 
+@lru_cache(maxsize=0)
 def is_keymap_dir(keymap, c=True, json=True, additional_files=None):
     """Return True if Path object `keymap` has a keymap file inside.
 
@@ -180,6 +185,7 @@ def generate_json(keymap, keyboard, layout, layers):
     return new_keymap
 
 
+@lru_cache(maxsize=0)
 def generate_c(keyboard, layout, layers):
     """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
 
@@ -266,6 +272,7 @@ def write(keyboard, keymap, layout, layers):
     return write_file(keymap_file, keymap_content)
 
 
+@lru_cache(maxsize=0)
 def locate_keymap(keyboard, keymap):
     """Returns the path to a keymap for a specific keyboard.
     """
@@ -305,6 +312,7 @@ def locate_keymap(keyboard, keymap):
                     return community_layout / 'keymap.c'
 
 
+@lru_cache(maxsize=0)
 def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
     """List the available keymaps for a keyboard.
 
@@ -327,40 +335,38 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     Returns:
         a sorted list of valid keymap names.
     """
-    # parse all the rules.mk files for the keyboard
-    rules = rules_mk(keyboard)
+    info_data = basic_info_json(keyboard)
     names = set()
 
-    if rules:
-        keyboards_dir = Path('keyboards')
-        kb_path = keyboards_dir / keyboard
+    keyboards_dir = Path('keyboards')
+    kb_path = keyboards_dir / info_data['keyboard_folder']
 
-        # walk up the directory tree until keyboards_dir
-        # and collect all directories' name with keymap.c file in it
-        while kb_path != keyboards_dir:
-            keymaps_dir = kb_path / "keymaps"
+    # walk up the directory tree until keyboards_dir
+    # and collect all directories' name with keymap.c file in it
+    while kb_path != keyboards_dir:
+        keymaps_dir = kb_path / "keymaps"
 
-            if keymaps_dir.is_dir():
-                for keymap in keymaps_dir.iterdir():
-                    if is_keymap_dir(keymap, c, json, additional_files):
-                        keymap = keymap if fullpath else keymap.name
-                        names.add(keymap)
+        if keymaps_dir.is_dir():
+            for keymap in keymaps_dir.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
 
-            kb_path = kb_path.parent
+        kb_path = kb_path.parent
 
-        # if community layouts are supported, get them
-        if "LAYOUTS" in rules:
-            for layout in rules["LAYOUTS"].split():
-                cl_path = Path('layouts/community') / layout
-                if cl_path.is_dir():
-                    for keymap in cl_path.iterdir():
-                        if is_keymap_dir(keymap, c, json, additional_files):
-                            keymap = keymap if fullpath else keymap.name
-                            names.add(keymap)
+    # if community layouts are supported, get them
+    for layout in info_data.get('community_layouts', []):
+        cl_path = Path('layouts/community') / layout
+        if cl_path.is_dir():
+            for keymap in cl_path.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
 
     return sorted(names)
 
 
+@lru_cache(maxsize=0)
 def _c_preprocess(path, stdin=DEVNULL):
     """ Run a file through the C pre-processor
 
@@ -380,6 +386,7 @@ def _c_preprocess(path, stdin=DEVNULL):
     return pre_processed_keymap.stdout
 
 
+@lru_cache(maxsize=0)
 def _get_layers(keymap):  # noqa C901 : until someone has a good idea how to simplify/split up this code
     """ Find the layers in a keymap.c file.
 
@@ -500,6 +507,7 @@ def _get_layers(keymap):  # noqa C901 : until someone has a good idea how to sim
     return layers
 
 
+@lru_cache(maxsize=0)
 def parse_keymap_c(keymap_file, use_cpp=True):
     """ Parse a keymap.c file.
 
@@ -529,6 +537,7 @@ def parse_keymap_c(keymap_file, use_cpp=True):
     return keymap
 
 
+@lru_cache(maxsize=0)
 def c2json(keyboard, keymap, keymap_file, use_cpp=True):
     """ Convert keymap.c to keymap.json
 

lib/python/qmk/makefile.py

@@ -1,8 +1,10 @@
 """ Functions for working with Makefiles
 """
+from functools import lru_cache
 from pathlib import Path
 
 
+@lru_cache(maxsize=0)
 def parse_rules_mk_file(file, rules_mk=None):
     """Turn a rules.mk file into a dictionary.
 

lib/python/qmk/math.py

@@ -3,12 +3,22 @@
 Gratefully copied from https://stackoverflow.com/a/9558001
 """
 import ast
-import operator as op
+import operator
+from functools import lru_cache
 
 # supported operators
-operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv, ast.Pow: op.pow, ast.BitXor: op.xor, ast.USub: op.neg}
+operators = {
+    ast.Add: operator.add,
+    ast.Sub: operator.sub,
+    ast.Mult: operator.mul,
+    ast.Div: operator.truediv,
+    ast.Pow: operator.pow,
+    ast.BitXor: operator.xor,
+    ast.USub: operator.neg,
+}
 
 
+@lru_cache(maxsize=0)
 def compute(expr):
     """Parse a mathematical expression and return the answer.
 
@@ -22,6 +32,7 @@ def compute(expr):
     return _eval(ast.parse(expr, mode='eval').body)
 
 
+@lru_cache(maxsize=0)
 def _eval(node):
     if isinstance(node, ast.Num):  # <number>
         return node.n