Rework qmk compile to bypass Makefile. Add new --filter option.

Makefile

@@ -393,7 +393,7 @@ define PARSE_KEYMAP
     # Specify the variables that we are passing forward to submake
     MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN)
     # And the first part of the make command
-    MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
+    MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
     # The message to display
     MAKE_MSG := $$(MSG_MAKE_KB)
     # We run the command differently, depending on if we want more output or not

lib/python/qmk/cli/clean.py

@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import _find_make
 from milc import cli
 
 
@@ -11,4 +11,6 @@ from milc import cli
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean']
+
+    cli.run(make_cmd, capture_output=False, stdin=DEVNULL)

lib/python/qmk/cli/compile.py

@@ -5,20 +5,70 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
 from subprocess import DEVNULL
 
 from argcomplete.completers import FilesCompleter
+from dotty_dict import dotty
 from milc import cli
 
 import qmk.path
-from qmk.decorators import automagic_keyboard, automagic_keymap
+from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.keymap import keymap_completer
+from qmk.info import info_json
+from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards
+from qmk.keymap import keymap_completer, list_keymaps
+from qmk.metadata import true_values, false_values
+
+
+@lru_cache()
+def _keyboard_list():
+    """Returns a list of keyboards matching cli.config.compile.keyboard.
+    """
+    if cli.config.compile.keyboard == 'all':
+        return list_keyboards()
+
+    elif cli.config.compile.keyboard.startswith('all-'):
+        return list_keyboards()
+
+    return [cli.config.compile.keyboard]
+
+
+def keyboard_keymap_iter():
+    """Iterates over the keyboard/keymap for this command and yields a pairing of each.
+    """
+    for keyboard in _keyboard_list():
+        continue_flag = False
+        if cli.args.filter:
+            info_data = dotty(info_json(keyboard))
+            for filter in cli.args.filter:
+                if '=' in filter:
+                    key, value = filter.split('=', 1)
+                    if value in true_values:
+                        value = True
+                    elif value in false_values:
+                        value = False
+                    elif value.isdigit():
+                        value = int(value)
+                    elif '.' in value and value.replace('.').isdigit():
+                        value = float(value)
+
+                    if info_data.get(key) != value:
+                        continue_flag = True
+                        break
+
+            if continue_flag:
+                continue
+
+        if cli.config.compile.keymap == 'all':
+            for keymap in list_keymaps(keyboard):
+                yield keyboard, keymap
+        else:
+            yield keyboard, cli.config.compile.keymap
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.subcommand('Compile a QMK Firmware.')
@@ -31,47 +81,79 @@ def compile(cli):
 
     If a keyboard and keymap are provided this command will build a firmware based on that.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
-        cli.run(command, capture_output=False, stdin=DEVNULL)
+    envs = {'REQUIRE_PLATFORM_KEY': ''}
+    silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all'
 
-    # Build the environment vars
-    envs = {}
+    # Setup the environment
     for env in cli.args.env:
         if '=' in env:
             key, value = env.split('=', 1)
+            if key in envs:
+                cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value)
             envs[key] = value
         else:
             cli.log.warning('Invalid environment variable: %s', env)
 
-    # Determine the compile command
-    command = None
+    if cli.config.compile.keyboard.startswith('all-'):
+        envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:]
+
+    # Run clean if necessary
+    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+        for keyboard, keymap in keyboard_keymap_iter():
+            cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap)
+            _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs)
+            cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
+
+    # Determine the compile command(s)
+    commands = None
 
     if cli.args.filename:
         # If a configurator JSON was provided generate a keymap and compile it
         user_keymap = parse_configurator_json(cli.args.filename)
-        command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
+        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)]
 
-    else:
-        if cli.config.compile.keyboard and cli.config.compile.keymap:
-            # Generate the make command for a specific keyboard/keymap.
-            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
+    elif cli.config.compile.keyboard and cli.config.compile.keymap:
+        if cli.args.filter:
+            cli.log.info('Generating the list of keyboards to compile, this may take some time.')
 
-        elif not cli.config.compile.keyboard:
-            cli.log.error('Could not determine keyboard!')
-        elif not cli.config.compile.keymap:
-            cli.log.error('Could not determine keymap!')
+        commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()]
+
+    elif not cli.config.compile.keyboard:
+        cli.log.error('Could not determine keyboard!')
+
+    elif not cli.config.compile.keymap:
+        cli.log.error('Could not determine keymap!')
 
     # Compile the firmware, if we're able to
-    if command:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-        if not cli.args.dry_run:
-            cli.echo('\n')
-            # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
-            compile = cli.run(command, capture_output=False, text=False)
-            return compile.returncode
+    if commands:
+        returncodes = []
+        for keyboard, keymap, command in commands:
+            print()
+            cli.log.info('Compiling QMK Firmware for keyboard {fg_cyan}%s{fg_reset} and keymap {fg_cyan}%s', keyboard, keymap)
+            cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command))
+
+            if not cli.args.dry_run:
+                compile = cli.run(command, capture_output=False)
+                returncodes.append(compile.returncode)
+                if compile.returncode == 0:
+                    cli.log.info('Success!')
+                else:
+                    cli.log.error('Failed!')
+
+        if any(returncodes):
+            print()
+            cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
+
+            for i, returncode in enumerate(returncodes):
+                if returncode != 0:
+                    keyboard, keymap, command = commands[i]
+                    cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
+
+    elif cli.args.filter:
+        cli.log.error('No keyboards found after applying filter(s)!')
+        return False
 
     else:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
+        cli.print_help()
         return False

lib/python/qmk/cli/flash.py

@@ -11,7 +11,7 @@ from milc import cli
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keyboard import keyboard_completer, is_keyboard_target
 
 
 def print_bootloader_help():
@@ -36,7 +36,7 @@ def print_bootloader_help():
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")

lib/python/qmk/commands.py

@@ -27,34 +27,7 @@ def _find_make():
     return make_cmd
 
 
-def create_make_target(target, parallel=1, **env_vars):
-    """Create a make command
-
-    Args:
-
-        target
-            Usually a make rule, such as 'clean' or 'all'.
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    env = []
-    make_cmd = _find_make()
-
-    for key, value in env_vars.items():
-        env.append(f'{key}={value}')
-
-    return [make_cmd, '-j', str(parallel), *env, target]
-
-
-def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
+def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars):
     """Create a make compile command
 
     Args:
@@ -78,12 +51,26 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
 
         A command that can be run to make the specified keyboard and keymap
     """
-    make_args = [keyboard, keymap]
+    make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk']
+
+    env_vars['KEYBOARD'] = keyboard
+    env_vars['KEYMAP'] = keymap
+    env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk'
+    env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else ''
+    env_vars['SILENT'] = 'true' if silent else 'false'
+    env_vars['COLOR'] = 'true' if cli.config.general.color else ''
+
+    if parallel > 1:
+        make_cmd.append('-j')
+        make_cmd.append(parallel)
 
     if target:
-        make_args.append(target)
+        make_cmd.append(target)
 
-    return create_make_target(':'.join(make_args), parallel, **env_vars)
+    for key, value in env_vars.items():
+        make_cmd.append(f'{key}={value}')
+
+    return keyboard, keymap, make_cmd
 
 
 def get_git_version(repo_dir='.', check_dir='.'):
@@ -204,7 +191,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
         f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}',
     ])
 
-    return make_command
+    return user_keymap['keyboard'], user_keymap['keymap'], make_command
 
 
 def parse_configurator_json(configurator_file):

lib/python/qmk/keyboard.py

@@ -64,6 +64,17 @@ def find_readme(keyboard):
     return cur_dir / 'readme.md'
 
 
+def is_keyboard_target(keyboard_target):
+    """Checks to make sure the supplied keyboard_target is valid.
+
+    This is mainly used by commands that accept --keyboard.
+    """
+    if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']:
+        return keyboard_target
+
+    return keyboard_folder(keyboard_target)
+
+
 def keyboard_folder(keyboard):
     """Returns the actual keyboard folder.
 

lib/python/qmk/keymap.py

@@ -14,6 +14,7 @@ from pygments import lex
 import qmk.path
 from qmk.keyboard import find_keyboard_from_dir, rules_mk
 from qmk.errors import CppError
+from qmk.metadata import basic_info_json
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -327,36 +328,33 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     Returns:
         a sorted list of valid keymap names.
     """
-    # parse all the rules.mk files for the keyboard
-    rules = rules_mk(keyboard)
+    info_data = basic_info_json(keyboard)
     names = set()
 
-    if rules:
-        keyboards_dir = Path('keyboards')
-        kb_path = keyboards_dir / keyboard
+    keyboards_dir = Path('keyboards')
+    kb_path = keyboards_dir / info_data['keyboard_folder']
 
-        # walk up the directory tree until keyboards_dir
-        # and collect all directories' name with keymap.c file in it
-        while kb_path != keyboards_dir:
-            keymaps_dir = kb_path / "keymaps"
+    # walk up the directory tree until keyboards_dir
+    # and collect all directories' name with keymap.c file in it
+    while kb_path != keyboards_dir:
+        keymaps_dir = kb_path / "keymaps"
 
-            if keymaps_dir.is_dir():
-                for keymap in keymaps_dir.iterdir():
-                    if is_keymap_dir(keymap, c, json, additional_files):
-                        keymap = keymap if fullpath else keymap.name
-                        names.add(keymap)
+        if keymaps_dir.is_dir():
+            for keymap in keymaps_dir.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
 
-            kb_path = kb_path.parent
+        kb_path = kb_path.parent
 
-        # if community layouts are supported, get them
-        if "LAYOUTS" in rules:
-            for layout in rules["LAYOUTS"].split():
-                cl_path = Path('layouts/community') / layout
-                if cl_path.is_dir():
-                    for keymap in cl_path.iterdir():
-                        if is_keymap_dir(keymap, c, json, additional_files):
-                            keymap = keymap if fullpath else keymap.name
-                            names.add(keymap)
+    # if community layouts are supported, get them
+    for layout in info_data.get('community_layouts', []):
+        cl_path = Path('layouts/community') / layout
+        if cl_path.is_dir():
+            for keymap in cl_path.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
 
     return sorted(names)