Compare commits

...

14 Commits

Author SHA1 Message Date
81b17125eb fix after rebase 2021-09-09 08:33:41 -07:00
6f4742bde6 log output tweaks 2021-08-29 17:10:26 -07:00
4fadb98a02 cleanup output 2021-08-29 17:10:26 -07:00
335dd3c5c3 ensure parallel is string 2021-08-29 17:10:26 -07:00
dcbfdb5cfc lru_cache everywhere 2021-08-29 17:10:26 -07:00
823a74ebae add support for building multiple keyboards in parallel 2021-08-29 17:09:52 -07:00
08b0ecb175 compile matching boards as we find them, not after building the whole list 2021-08-29 17:09:51 -07:00
b4e18c9019 Track mtimes for info.json files
This allows us to skip validation when the file has not been changed
since the last time it was validated.
2021-08-29 17:09:51 -07:00
07b8035ba9 do some optimizing 2021-08-29 17:09:51 -07:00
4f20c94b97 unify the compile and flash commands 2021-08-29 17:09:51 -07:00
ea862e24f6 refactor the compile code into commands.py 2021-08-29 17:09:08 -07:00
7fe506006e fix Makefile 2021-08-29 17:07:33 -07:00
d3ed6fa8a4 eliminate the need for -kb all 2021-08-29 17:06:11 -07:00
50fdb2a52c Rework qmk compile to bypass Makefile. Add new --filter option. 2021-08-29 17:06:11 -07:00
16 changed files with 884 additions and 195 deletions

View File

@ -1,7 +1,8 @@
"""Functions for working with config.h files.
"""
from pathlib import Path
import re
from functools import lru_cache
from pathlib import Path
from milc import cli
@ -12,18 +13,21 @@ single_comment_regex = re.compile(r'\s+/[/*].*$')
multi_comment_regex = re.compile(r'/\*(.|\n)*?\*/', re.MULTILINE)
@lru_cache(maxsize=0)
def strip_line_comment(string):
"""Removes comments from a single line string.
"""
return single_comment_regex.sub('', string)
@lru_cache(maxsize=0)
def strip_multiline_comment(string):
"""Removes comments from a single line string.
"""
return multi_comment_regex.sub('', string)
@lru_cache(maxsize=0)
def c_source_files(dir_names):
"""Returns a list of all *.c, *.h, and *.cpp files for a given list of directories
@ -38,6 +42,7 @@ def c_source_files(dir_names):
return files
@lru_cache(maxsize=0)
def find_layouts(file):
"""Returns list of parsed LAYOUT preprocessor macros found in the supplied include file.
"""
@ -144,6 +149,7 @@ def _default_key(label=None):
return new_key
@lru_cache(maxsize=0)
def _parse_layout_macro(layout_macro):
"""Split the LAYOUT macro into its constituent parts
"""
@ -154,6 +160,7 @@ def _parse_layout_macro(layout_macro):
return macro_name, layout, matrix
@lru_cache(maxsize=0)
def _parse_matrix_locations(matrix, file, macro_name):
"""Parse raw matrix data into a dictionary keyed by the LAYOUT identifier.
"""

View File

@ -2,7 +2,7 @@
"""
from subprocess import DEVNULL
from qmk.commands import create_make_target
from qmk.commands import _find_make
from milc import cli
@ -11,4 +11,6 @@ from milc import cli
def clean(cli):
"""Runs `make clean` (or `make distclean` if --all is passed)
"""
cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean']
cli.run(make_cmd, capture_output=False, stdin=DEVNULL)

View File

@ -2,23 +2,24 @@
You can compile a keymap already in the repo or using a QMK Configurator export.
"""
from subprocess import DEVNULL
from argcomplete.completers import FilesCompleter
from milc import cli
import qmk.path
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.commands import do_compile
from qmk.keyboard import keyboard_completer, is_keyboard_target
from qmk.keymap import keymap_completer
from qmk.metadata import true_values, false_values
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-t', '--target', help="The make target to run. By default it compiles the keyboard only.")
@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.subcommand('Compile a QMK Firmware.')
@ -31,47 +32,31 @@ def compile(cli):
If a keyboard and keymap are provided this command will build a firmware based on that.
"""
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
cli.run(command, capture_output=False, stdin=DEVNULL)
# If -f has been specified without a keyboard target, assume -kb all
keyboard = cli.config.compile.keyboard or ''
# Build the environment vars
envs = {}
for env in cli.args.env:
if '=' in env:
key, value = env.split('=', 1)
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
if cli.args.filter and not cli.args.keyboard:
cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.')
keyboard = 'all'
# Determine the compile command
command = None
if cli.args.filename and cli.args.filter:
cli.log.warning('Ignoring --filter because a keymap.json was provided.')
if cli.args.filename:
# If a configurator JSON was provided generate a keymap and compile it
user_keymap = parse_configurator_json(cli.args.filename)
command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
filters = {}
else:
if cli.config.compile.keyboard and cli.config.compile.keymap:
# Generate the make command for a specific keyboard/keymap.
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
for filter in cli.args.filter:
if '=' in filter:
key, value = filter.split('=', 1)
elif not cli.config.compile.keyboard:
cli.log.error('Could not determine keyboard!')
elif not cli.config.compile.keymap:
cli.log.error('Could not determine keymap!')
if value in true_values:
value = True
elif value in false_values:
value = False
elif value.isdigit():
value = int(value)
elif '.' in value and value.replace('.').isdigit():
value = float(value)
# Compile the firmware, if we're able to
if command:
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run:
cli.echo('\n')
# FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
compile = cli.run(command, capture_output=False, text=False)
return compile.returncode
filters[key] = value
else:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
return False
return do_compile(keyboard, cli.config.compile.keymap, cli.config.compile.parallel, cli.config.compile.target, filters)

View File

@ -3,15 +3,13 @@
You can compile a keymap already in the repo or using a QMK Configurator export.
A bootloader must be specified.
"""
from subprocess import DEVNULL
from argcomplete.completers import FilesCompleter
from milc import cli
import qmk.path
from qmk.decorators import automagic_keyboard, automagic_keymap
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.commands import do_compile
from qmk.keyboard import keyboard_completer, is_keyboard_target
def print_bootloader_help():
@ -36,7 +34,7 @@ def print_bootloader_help():
@cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
@cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@ -54,55 +52,10 @@ def flash(cli):
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
"""
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
cli.run(command, capture_output=False, stdin=DEVNULL)
# Build the environment vars
envs = {}
for env in cli.args.env:
if '=' in env:
key, value = env.split('=', 1)
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
# Determine the compile command
command = ''
if cli.args.bootloaders:
# Provide usage and list bootloaders
cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
cli.print_usage()
print_bootloader_help()
return False
if cli.args.filename:
# Handle compiling a configurator JSON
user_keymap = parse_configurator_json(cli.args.filename)
keymap_path = qmk.path.keymap(user_keymap['keyboard'])
command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
else:
if cli.config.flash.keyboard and cli.config.flash.keymap:
# Generate the make command for a specific keyboard/keymap.
command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
elif not cli.config.flash.keyboard:
cli.log.error('Could not determine keyboard!')
elif not cli.config.flash.keymap:
cli.log.error('Could not determine keymap!')
# Compile the firmware, if we're able to
if command:
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run:
cli.echo('\n')
compile = cli.run(command, capture_output=False, stdin=DEVNULL)
return compile.returncode
else:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
return False
return do_compile(cli.config.flash.keyboard, cli.config.flash.keymap, cli.config.flash.parallel, cli.config.flash.bootloader)

View File

@ -1,22 +1,28 @@
"""Helper functions for commands.
"""
from functools import lru_cache
import json
import os
import sys
import shutil
import threading
from pathlib import Path
from subprocess import DEVNULL
from time import strftime
from time import sleep, strftime
from dotty_dict import dotty
from milc import cli
import qmk.keymap
from qmk.constants import QMK_FIRMWARE, KEYBOARD_OUTPUT_PREFIX
from qmk.info import info_json
from qmk.json_schema import json_load
from qmk.keyboard import list_keyboards
time_fmt = '%Y-%m-%d-%H:%M:%S'
@lru_cache(maxsize=0)
def _find_make():
"""Returns the correct make command for this environment.
"""
@ -55,7 +61,7 @@ def create_make_target(target, parallel=1, **env_vars):
return [make_cmd, *get_make_parallel_args(parallel), *env, target]
def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars):
"""Create a make compile command
Args:
@ -79,14 +85,29 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
A command that can be run to make the specified keyboard and keymap
"""
make_args = [keyboard, keymap]
make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk']
env_vars['KEYBOARD'] = keyboard
env_vars['KEYMAP'] = keymap
env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk'
env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else ''
env_vars['SILENT'] = 'true' if silent else 'false'
env_vars['COLOR'] = 'true' if cli.config.general.color else ''
if parallel > 1:
make_cmd.append('-j')
make_cmd.append(str(parallel))
if target:
make_args.append(target)
make_cmd.append(target)
return create_make_target(':'.join(make_args), parallel, **env_vars)
for key, value in env_vars.items():
make_cmd.append(f'{key}={value}')
return make_cmd
@lru_cache(maxsize=0)
def get_git_version(current_time, repo_dir='.', check_dir='.'):
"""Returns the current git version for a repo, or the current time.
"""
@ -236,9 +257,10 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
'QMK_BIN="qmk"',
])
return make_command
return user_keymap['keyboard'], user_keymap['keymap'], make_command
@lru_cache(maxsize=0)
def parse_configurator_json(configurator_file):
"""Open and parse a configurator json export
"""
@ -332,3 +354,165 @@ def in_virtualenv():
"""
active_prefix = getattr(sys, "base_prefix", None) or getattr(sys, "real_prefix", None) or sys.prefix
return active_prefix != sys.prefix
def do_compile(keyboard, keymap, parallel, target=None, filters=None, environment=None):
"""Shared code between `qmk compile` and `qmk flash`.
"""
if keyboard is None:
keyboard = ''
if environment is None:
environment = {}
all_keyboards = keyboard == 'all' or keyboard.startswith('all-')
all_keymaps = keymap == 'all'
multiple_compiles = all_keyboards or all_keymaps
# Setup the environment
envs = {'REQUIRE_PLATFORM_KEY': ''}
for env in environment:
if '=' in env:
key, value = env.split('=', 1)
if key in envs:
cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value)
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
if keyboard.startswith('all-'):
envs['REQUIRE_PLATFORM_KEY'] = keyboard[4:]
# Run clean if necessary
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
for kb, km in keyboard_keymap_iter(keyboard, keymap, {}):
cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', kb, km)
make_cmd = create_make_command(kb, km, 'clean', 1, multiple_compiles, **envs)
cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
# Determine the compile command(s)
command = None
if cli.args.filename:
if cli.args.keyboard:
cli.log.warning('Ignoring --keyboard because a keymap.json was provided.')
if cli.args.keymap:
cli.log.warning('Ignoring --keymap because a keymap.json was provided.')
# If a configurator JSON was provided generate a keymap and compile it
user_keymap = parse_configurator_json(cli.args.filename)
command = compile_configurator_json(user_keymap, parallel=parallel, **envs)
elif keyboard and keymap:
if multiple_compiles:
command = 'multiple'
else:
command = create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=multiple_compiles, **envs)
elif not keyboard:
cli.log.error('Could not determine keyboard!')
elif not keymap:
cli.log.error('Could not determine keymap!')
# Compile the firmware, if we're able to
if command == 'multiple':
cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
returncodes = []
for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters):
command = create_make_command(keyboard, keymap, target=target, parallel=1, silent=multiple_compiles, **envs)
while threading.active_count() >= parallel + 1:
sleep(1)
threading.Thread(target=_execute_compile, args=(keyboard, keymap, command, target, returncodes)).start()
while threading.active_count() > 1:
sleep(1)
if any(returncodes):
print()
cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
for i, returncode in enumerate(returncodes):
if returncode != 0:
keyboard, keymap, command = returncodes[i]
cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
elif command:
if target:
cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target)
else:
cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
if _execute_compile(keyboard, keymap, command, target) != 0:
print()
cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
elif filters:
cli.log.error('No keyboards found after applying filter(s)!')
return False
else:
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
cli.print_help()
return False
def _execute_compile(keyboard, keymap, command, target, returncodes=None):
if not returncodes:
returncodes = []
if keymap not in qmk.keymap.list_keymaps(keyboard):
cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap)
return 0
cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command))
if not cli.args.dry_run:
compile = cli.run(command, combined_output=True)
cli.acquire_lock()
returncodes.append(compile.returncode)
cli.release_lock()
if compile.returncode != 0:
cli.log.info('Could not build firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
print(compile.stdout)
@lru_cache()
def _keyboard_list(keyboard):
"""Returns a list of keyboards matching keyboard.
"""
if keyboard == 'all' or keyboard.startswith('all-'):
return list_keyboards()
return [keyboard]
def keyboard_keymap_iter(cli_keyboard, cli_keymap, filters):
"""Iterates over the keyboard/keymap for this command and yields a pairing of each.
"""
for keyboard in _keyboard_list(cli_keyboard):
continue_flag = False
if filters:
info_data = dotty(info_json(keyboard))
for key, value in filters.items():
if info_data.get(key) != value:
continue_flag = True
break
if continue_flag:
continue
if cli_keymap == 'all':
for keymap in qmk.keymap.list_keymaps(keyboard):
yield keyboard, keymap
else:
yield keyboard, cli_keymap

View File

@ -3,6 +3,7 @@
Gratefully adapted from https://stackoverflow.com/a/241506
"""
import re
from functools import lru_cache
comment_pattern = re.compile(r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE)
@ -14,6 +15,7 @@ def _comment_stripper(match):
return ' ' if s.startswith('/') else s
@lru_cache(maxsize=0)
def comment_remover(text):
"""Remove C/C++ style comments from text.
"""

View File

@ -1,8 +1,10 @@
"""Functions to convert to and from QMK formats
"""
from collections import OrderedDict
from functools import lru_cache
@lru_cache(maxsize=0)
def kle2qmk(kle):
"""Convert a KLE layout to QMK's layout format.
"""

View File

@ -1,5 +1,6 @@
"""Functions that help us generate and use info.json files.
"""
from functools import lru_cache
from glob import glob
from pathlib import Path
@ -12,37 +13,22 @@ from qmk.c_parse import find_layouts
from qmk.json_schema import deep_update, json_load, validate
from qmk.keyboard import config_h, rules_mk
from qmk.keymap import list_keymaps
from qmk.makefile import parse_rules_mk_file
from qmk.math import compute
true_values = ['1', 'on', 'yes']
false_values = ['0', 'off', 'no']
from qmk.metadata import basic_info_json, info_log_error, info_log_warning, true_values, false_values
@lru_cache(maxsize=None)
def _valid_community_layout(layout):
"""Validate that a declared community list exists
"""
return (Path('layouts/default') / layout).exists()
@lru_cache(maxsize=None)
def info_json(keyboard):
"""Generate the info.json data for a specific keyboard.
"""
cur_dir = Path('keyboards')
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
if 'DEFAULT_FOLDER' in rules:
keyboard = rules['DEFAULT_FOLDER']
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
info_data = {
'keyboard_name': str(keyboard),
'keyboard_folder': str(keyboard),
'keymaps': {},
'layouts': {},
'parse_errors': [],
'parse_warnings': [],
'maintainer': 'qmk',
}
info_data = basic_info_json(keyboard)
# Populate the list of JSON keymaps
for keymap in list_keymaps(keyboard, c=False, fullpath=True):
@ -81,20 +67,20 @@ def info_json(keyboard):
_find_missing_layouts(info_data, keyboard)
if not info_data.get('layouts'):
_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
# Filter out any non-existing community layouts
for layout in info_data.get('community_layouts', []):
if not _valid_community_layout(layout):
# Ignore layout from future checks
info_data['community_layouts'].remove(layout)
_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
# Make sure we supply layout macros for the community layouts we claim to support
for layout in info_data.get('community_layouts', []):
layout_name = 'LAYOUT_' + layout
if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
# Check that the reported matrix size is consistent with the actual matrix size
_check_matrix(info_data)
@ -130,7 +116,7 @@ def _extract_features(info_data, rules):
info_data['features'] = {}
if key in info_data['features']:
_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
info_data['features'][key] = value
info_data['config_h_features'][key] = value
@ -209,7 +195,7 @@ def _extract_split_main(info_data, config_c):
info_data['split'] = {}
if 'main' in info_data['split']:
_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_data['split']['main'] = 'pin'
@ -218,7 +204,7 @@ def _extract_split_main(info_data, config_c):
info_data['split'] = {}
if 'main' in info_data['split']:
_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_data['split']['main'] = 'matrix_grid'
info_data['split']['matrix_grid'] = _extract_pins(config_c['SPLIT_HAND_MATRIX_GRID'])
@ -228,7 +214,7 @@ def _extract_split_main(info_data, config_c):
info_data['split'] = {}
if 'main' in info_data['split']:
_log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_data['split']['main'] = 'eeprom'
@ -237,7 +223,7 @@ def _extract_split_main(info_data, config_c):
info_data['split'] = {}
if 'main' in info_data['split']:
_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_data['split']['main'] = 'right'
@ -246,7 +232,7 @@ def _extract_split_main(info_data, config_c):
info_data['split'] = {}
if 'main' in info_data['split']:
_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main'])
info_data['split']['main'] = 'left'
@ -261,7 +247,7 @@ def _extract_split_transport(info_data, config_c):
info_data['split']['transport'] = {}
if 'protocol' in info_data['split']['transport']:
_log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport'])
info_log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport'])
info_data['split']['transport']['protocol'] = 'i2c'
@ -285,7 +271,7 @@ def _extract_split_right_pins(info_data, config_c):
if row_pins and col_pins:
if info_data.get('split', {}).get('matrix_pins', {}).get('right') in info_data:
_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
if 'split' not in info_data:
info_data['split'] = {}
@ -303,7 +289,7 @@ def _extract_split_right_pins(info_data, config_c):
if direct_pins:
if info_data.get('split', {}).get('matrix_pins', {}).get('right', {}):
_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
if 'split' not in info_data:
info_data['split'] = {}
@ -341,7 +327,7 @@ def _extract_matrix_info(info_data, config_c):
if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
if 'matrix_size' in info_data:
_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
info_data['matrix_size'] = {
'cols': compute(config_c.get('MATRIX_COLS', '0')),
@ -350,14 +336,14 @@ def _extract_matrix_info(info_data, config_c):
if row_pins and col_pins:
if 'matrix_pins' in info_data and 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']:
_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
info_snippet['cols'] = _extract_pins(col_pins)
info_snippet['rows'] = _extract_pins(row_pins)
if direct_pins:
if 'matrix_pins' in info_data and 'direct' in info_data['matrix_pins']:
_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
info_snippet['direct'] = _extract_direct_matrix(direct_pins)
@ -369,7 +355,7 @@ def _extract_matrix_info(info_data, config_c):
if config_c.get('CUSTOM_MATRIX', 'no') != 'no':
if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']:
_log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.')
info_log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.')
info_snippet['custom'] = True
@ -398,7 +384,7 @@ def _extract_config_h(info_data):
try:
if config_key in config_c and info_dict.get('to_json', True):
if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
if key_type.startswith('array'):
if '.' in key_type:
@ -429,7 +415,7 @@ def _extract_config_h(info_data):
dotty_info[info_key] = config_c[config_key]
except Exception as e:
_log_warning(info_data, f'{config_key}->{info_key}: {e}')
info_log_warning(info_data, f'{config_key}->{info_key}: {e}')
info_data.update(dotty_info)
@ -470,7 +456,7 @@ def _extract_rules_mk(info_data):
try:
if rules_key in rules and info_dict.get('to_json', True):
if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
if key_type.startswith('array'):
if '.' in key_type:
@ -501,7 +487,7 @@ def _extract_rules_mk(info_data):
dotty_info[info_key] = rules[rules_key]
except Exception as e:
_log_warning(info_data, f'{rules_key}->{info_key}: {e}')
info_log_warning(info_data, f'{rules_key}->{info_key}: {e}')
info_data.update(dotty_info)
@ -544,11 +530,11 @@ def _check_matrix(info_data):
if col_count != actual_col_count and col_count != (actual_col_count / 2):
# FIXME: once we can we should detect if split is enabled to do the actual_col_count/2 check.
_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
info_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
if row_count != actual_row_count and row_count != (actual_row_count / 2):
# FIXME: once we can we should detect if split is enabled to do the actual_row_count/2 check.
_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
info_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
def _search_keyboard_h(keyboard):
@ -577,7 +563,7 @@ def _find_missing_layouts(info_data, keyboard):
If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
"""
_log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
info_log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
for file in glob('keyboards/%s/*.h' % keyboard):
these_layouts, these_aliases = find_layouts(file)
@ -596,20 +582,6 @@ def _find_missing_layouts(info_data, keyboard):
info_data['layout_aliases'][alias] = alias_text
def _log_error(info_data, message):
"""Send an error message to both JSON and the log.
"""
info_data['parse_errors'].append(message)
cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
def _log_warning(info_data, message):
"""Send a warning message to both JSON and the log.
"""
info_data['parse_warnings'].append(message)
cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
def arm_processor_rules(info_data, rules):
"""Setup the default info for an ARM board.
"""
@ -668,7 +640,7 @@ def merge_info_jsons(keyboard, info_data):
new_info_data = json_load(info_file)
if not isinstance(new_info_data, dict):
_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
continue
try:
@ -686,13 +658,13 @@ def merge_info_jsons(keyboard, info_data):
for layout_name, layout in new_info_data.get('layouts', {}).items():
if layout_name in info_data.get('layout_aliases', {}):
_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
layout_name = info_data['layout_aliases'][layout_name]
if layout_name in info_data['layouts']:
if len(info_data['layouts'][layout_name]['layout']) != len(layout['layout']):
msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
info_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
else:
for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
existing_key.update(new_key)
@ -720,15 +692,18 @@ def find_info_json(keyboard):
# Add DEFAULT_FOLDER before parents, if present
rules = rules_mk(keyboard)
if 'DEFAULT_FOLDER' in rules:
info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
# Add in parent folders for least specific
for _ in range(5):
info_jsons.append(keyboard_parent / 'info.json')
this_info_json = keyboard_parent / 'info.json'
if this_info_json.exists():
yield this_info_json
if keyboard_parent.parent == base_path:
break
keyboard_parent = keyboard_parent.parent
# Return a list of the info.json files that actually exist
return [info_json for info_json in info_jsons if info_json.exists()]
keyboard_parent = keyboard_parent.parent

View File

@ -10,6 +10,7 @@ import jsonschema
from milc import cli
@lru_cache(maxsize=0)
def json_load(json_file):
"""Load a json file from disk.
@ -23,6 +24,8 @@ def json_load(json_file):
exit(1)
except Exception as e:
cli.log.error('Unknown error attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
if cli.args.verbose:
cli.log.exception(e)
exit(1)

View File

@ -1,10 +1,11 @@
"""Functions that help us work with keyboards.
"""
import os
from array import array
from functools import lru_cache
from glob import glob
from math import ceil
from pathlib import Path
import os
from glob import glob
import qmk.path
from qmk.c_parse import parse_config_h_file
@ -64,6 +65,17 @@ def find_readme(keyboard):
return cur_dir / 'readme.md'
def is_keyboard_target(keyboard_target):
"""Checks to make sure the supplied keyboard_target is valid.
This is mainly used by commands that accept --keyboard.
"""
if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']:
return keyboard_target
return keyboard_folder(keyboard_target)
def keyboard_folder(keyboard):
"""Returns the actual keyboard folder.
@ -206,6 +218,7 @@ def render_layout(layout_data, render_ascii, key_labels=None):
return '\n'.join(lines)
@lru_cache(maxsize=0)
def render_layouts(info_json, render_ascii):
"""Renders all the layouts from an `info_json` structure.
"""

View File

@ -2,6 +2,7 @@
"""
import json
import sys
from functools import lru_cache
from pathlib import Path
from subprocess import DEVNULL
@ -14,6 +15,7 @@ from pygments import lex
import qmk.path
from qmk.keyboard import find_keyboard_from_dir, rules_mk
from qmk.errors import CppError
from qmk.metadata import basic_info_json
# The `keymap.c` template to use when a keyboard doesn't have its own
DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@ -30,6 +32,7 @@ __KEYMAP_GOES_HERE__
"""
@lru_cache(maxsize=0)
def template_json(keyboard):
"""Returns a `keymap.json` template for a keyboard.
@ -47,6 +50,7 @@ def template_json(keyboard):
return template
@lru_cache(maxsize=0)
def template_c(keyboard):
"""Returns a `keymap.c` template for a keyboard.
@ -122,6 +126,7 @@ def keymap_completer(prefix, action, parser, parsed_args):
return []
@lru_cache(maxsize=0)
def is_keymap_dir(keymap, c=True, json=True, additional_files=None):
"""Return True if Path object `keymap` has a keymap file inside.
@ -180,6 +185,7 @@ def generate_json(keymap, keyboard, layout, layers):
return new_keymap
@lru_cache(maxsize=0)
def generate_c(keyboard, layout, layers):
"""Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
@ -266,6 +272,7 @@ def write(keyboard, keymap, layout, layers):
return write_file(keymap_file, keymap_content)
@lru_cache(maxsize=0)
def locate_keymap(keyboard, keymap):
"""Returns the path to a keymap for a specific keyboard.
"""
@ -305,6 +312,7 @@ def locate_keymap(keyboard, keymap):
return community_layout / 'keymap.c'
@lru_cache(maxsize=0)
def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
"""List the available keymaps for a keyboard.
@ -327,40 +335,38 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
Returns:
a sorted list of valid keymap names.
"""
# parse all the rules.mk files for the keyboard
rules = rules_mk(keyboard)
info_data = basic_info_json(keyboard)
names = set()
if rules:
keyboards_dir = Path('keyboards')
kb_path = keyboards_dir / keyboard
keyboards_dir = Path('keyboards')
kb_path = keyboards_dir / info_data['keyboard_folder']
# walk up the directory tree until keyboards_dir
# and collect all directories' name with keymap.c file in it
while kb_path != keyboards_dir:
keymaps_dir = kb_path / "keymaps"
# walk up the directory tree until keyboards_dir
# and collect all directories' name with keymap.c file in it
while kb_path != keyboards_dir:
keymaps_dir = kb_path / "keymaps"
if keymaps_dir.is_dir():
for keymap in keymaps_dir.iterdir():
if is_keymap_dir(keymap, c, json, additional_files):
keymap = keymap if fullpath else keymap.name
names.add(keymap)
if keymaps_dir.is_dir():
for keymap in keymaps_dir.iterdir():
if is_keymap_dir(keymap, c, json, additional_files):
keymap = keymap if fullpath else keymap.name
names.add(keymap)
kb_path = kb_path.parent
kb_path = kb_path.parent
# if community layouts are supported, get them
if "LAYOUTS" in rules:
for layout in rules["LAYOUTS"].split():
cl_path = Path('layouts/community') / layout
if cl_path.is_dir():
for keymap in cl_path.iterdir():
if is_keymap_dir(keymap, c, json, additional_files):
keymap = keymap if fullpath else keymap.name
names.add(keymap)
# if community layouts are supported, get them
for layout in info_data.get('community_layouts', []):
cl_path = Path('layouts/community') / layout
if cl_path.is_dir():
for keymap in cl_path.iterdir():
if is_keymap_dir(keymap, c, json, additional_files):
keymap = keymap if fullpath else keymap.name
names.add(keymap)
return sorted(names)
@lru_cache(maxsize=0)
def _c_preprocess(path, stdin=DEVNULL):
""" Run a file through the C pre-processor
@ -380,6 +386,7 @@ def _c_preprocess(path, stdin=DEVNULL):
return pre_processed_keymap.stdout
@lru_cache(maxsize=0)
def _get_layers(keymap): # noqa C901 : until someone has a good idea how to simplify/split up this code
""" Find the layers in a keymap.c file.
@ -500,6 +507,7 @@ def _get_layers(keymap): # noqa C901 : until someone has a good idea how to sim
return layers
@lru_cache(maxsize=0)
def parse_keymap_c(keymap_file, use_cpp=True):
""" Parse a keymap.c file.
@ -529,6 +537,7 @@ def parse_keymap_c(keymap_file, use_cpp=True):
return keymap
@lru_cache(maxsize=0)
def c2json(keyboard, keymap, keymap_file, use_cpp=True):
""" Convert keymap.c to keymap.json

View File

@ -1,8 +1,10 @@
""" Functions for working with Makefiles
"""
from functools import lru_cache
from pathlib import Path
@lru_cache(maxsize=0)
def parse_rules_mk_file(file, rules_mk=None):
"""Turn a rules.mk file into a dictionary.

View File

@ -3,12 +3,22 @@
Gratefully copied from https://stackoverflow.com/a/9558001
"""
import ast
import operator as op
import operator
from functools import lru_cache
# supported operators
operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv, ast.Pow: op.pow, ast.BitXor: op.xor, ast.USub: op.neg}
operators = {
ast.Add: operator.add,
ast.Sub: operator.sub,
ast.Mult: operator.mul,
ast.Div: operator.truediv,
ast.Pow: operator.pow,
ast.BitXor: operator.xor,
ast.USub: operator.neg,
}
@lru_cache(maxsize=0)
def compute(expr):
"""Parse a mathematical expression and return the answer.
@ -22,6 +32,7 @@ def compute(expr):
return _eval(ast.parse(expr, mode='eval').body)
@lru_cache(maxsize=0)
def _eval(node):
if isinstance(node, ast.Num): # <number>
return node.n

533
lib/python/qmk/metadata.py Normal file

File diff suppressed because it is too large Load Diff

View File

@ -3,12 +3,14 @@
import logging
import os
import argparse
from functools import lru_cache
from pathlib import Path
from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE
from qmk.errors import NoSuchKeyboardError
@lru_cache(maxsize=0)
def is_keyboard(keyboard_name):
"""Returns True if `keyboard_name` is a keyboard we can compile.
"""
@ -19,6 +21,7 @@ def is_keyboard(keyboard_name):
return rules_mk.exists()
@lru_cache(maxsize=0)
def under_qmk_firmware():
"""Returns a Path object representing the relative path under qmk_firmware, or None.
"""
@ -30,12 +33,14 @@ def under_qmk_firmware():
return None
@lru_cache(maxsize=0)
def keyboard(keyboard_name):
"""Returns the path to a keyboard's directory relative to the qmk root.
"""
return Path('keyboards') / keyboard_name
@lru_cache(maxsize=0)
def keymap(keyboard_name):
"""Locate the correct directory for storing a keymap.
@ -56,6 +61,7 @@ def keymap(keyboard_name):
raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard_name)
@lru_cache(maxsize=0)
def normpath(path):
"""Returns a `pathlib.Path()` object for a given path.

View File

@ -1,8 +1,10 @@
"""Functions for working with QMK's submodules.
"""
from functools import lru_cache
from milc import cli
@lru_cache(maxsize=0)
def status():
"""Returns a dictionary of submodules.