Add qmk mass-compile, which intends to supercede qmk multibuild whilst providing support for filtering as per data-driven definitions. (#18971)

This commit is contained in:
Nick Brassel 2023-01-04 15:10:18 +11:00 committed by GitHub
parent 46ed46e384
commit 691668340c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 167 additions and 1 deletions

View File

@ -51,7 +51,7 @@ ChibiOS and ChibiOS-Contrib need to be updated in tandem -- the latter has a bra
* `./util/chibios_conf_updater.sh`
* Build everything
* `cd $QMK_FIRMWARE`
* `qmk multibuild -j4`
* `qmk mass-compile -j 4`
* Make sure there are no errors
* Push to the repo
* `git commit -am 'Update ChibiOS to 99.9.9'`

View File

@ -71,6 +71,7 @@ subcommands = [
'qmk.cli.list.keymaps',
'qmk.cli.list.layouts',
'qmk.cli.kle2json',
'qmk.cli.mass_compile',
'qmk.cli.multibuild',
'qmk.cli.new.keyboard',
'qmk.cli.new.keymap',

View File

@ -0,0 +1,165 @@
"""Compile all keyboards.
This will compile everything in parallel, for testing purposes.
"""
import logging
import multiprocessing
import os
import re
from pathlib import Path
from subprocess import DEVNULL
from dotty_dict import dotty
from milc import cli
from qmk.constants import QMK_FIRMWARE
from qmk.commands import _find_make, get_make_parallel_args
from qmk.info import keymap_json
import qmk.keyboard
import qmk.keymap
def _set_log_level(level):
cli.acquire_lock()
old = cli.log_level
cli.log_level = level
cli.log.setLevel(level)
logging.root.setLevel(level)
cli.release_lock()
return old
def _all_keymaps(keyboard):
old = _set_log_level(logging.CRITICAL)
keymaps = qmk.keymap.list_keymaps(keyboard)
_set_log_level(old)
return (keyboard, keymaps)
def _keymap_exists(keyboard, keymap):
old = _set_log_level(logging.CRITICAL)
ret = keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
_set_log_level(old)
return ret
def _load_keymap_info(keyboard, keymap):
old = _set_log_level(logging.CRITICAL)
ret = (keyboard, keymap, keymap_json(keyboard, keymap))
_set_log_level(old)
return ret
@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.argument(
'-f',
'--filter',
arg_only=True,
action='append',
default=[],
help="Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the format 'features.rgblight=true'. May be passed multiple times, all filters need to match."
)
@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
def mass_compile(cli):
"""Compile QMK Firmware against all keyboards.
"""
make_cmd = _find_make()
if cli.args.clean:
cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
builddir = Path(QMK_FIRMWARE) / '.build'
makefile = builddir / 'parallel_kb_builds.mk'
targets = []
with multiprocessing.Pool() as pool:
cli.log.info(f'Retrieving list of keyboards with keymap "{cli.args.keymap}"...')
target_list = []
if cli.args.keymap == 'all':
kb_to_kms = pool.map(_all_keymaps, qmk.keyboard.list_keyboards())
for targets in kb_to_kms:
keyboard = targets[0]
keymaps = targets[1]
target_list.extend([(keyboard, keymap) for keymap in keymaps])
else:
target_list = [(kb, cli.args.keymap) for kb in filter(lambda kb: kb is not None, pool.starmap(_keymap_exists, [(kb, cli.args.keymap) for kb in qmk.keyboard.list_keyboards()]))]
if len(cli.args.filter) == 0:
targets = target_list
else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
filter_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
for filter_txt in cli.args.filter:
f = filter_re.match(filter_txt)
if f is not None:
key = f.group('key')
value = f.group('value')
cli.log.info(f'Filtering on condition ("{key}" == "{value}")...')
def _make_filter(k, v):
def f(e):
lhs = e[2].get(k)
lhs = str(False if lhs is None else lhs).lower()
rhs = str(v).lower()
return lhs == rhs
return f
valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
targets = [(e[0], e[1]) for e in valid_keymaps]
if len(targets) == 0:
return
builddir.mkdir(parents=True, exist_ok=True)
with open(makefile, "w") as f:
for target in sorted(targets):
keyboard_name = target[0]
keymap_name = target[1]
keyboard_safe = keyboard_name.replace('/', '_')
# yapf: disable
f.write(
f"""\
all: {keyboard_safe}_{keymap_name}_binary
{keyboard_safe}_{keymap_name}_binary:
@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}.{keymap_name}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 2>&1 \\
|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{keymap_name}"
@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" || true
"""# noqa
)
# yapf: enable
if cli.args.no_temp:
# yapf: disable
f.write(
f"""\
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.hex" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.bin" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.uf2" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true
"""# noqa
)
# yapf: enable
f.write('\n')
cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
# Check for failures
failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
if len(failures) > 0:
return False