CLI refactoring for common build target APIs (#22221)
This commit is contained in:
Nick Brassel
GitHub
parent
c4d3521ba6
commit
4938210711
211
lib/python/qmk/build_targets.py
Normal file
211
lib/python/qmk/build_targets.py
Normal file
@ -0,0 +1,211 @@
|
||||
# Copyright 2023 Nick Brassel (@tzarc)
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
import json
|
||||
import shutil
|
||||
from typing import List, Union
|
||||
from pathlib import Path
|
||||
from dotty_dict import dotty, Dotty
|
||||
from milc import cli
|
||||
from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
|
||||
from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
|
||||
from qmk.keyboard import keyboard_folder
|
||||
from qmk.info import keymap_json
|
||||
from qmk.cli.generate.compilation_database import write_compilation_database
|
||||
|
||||
|
||||
class BuildTarget:
|
||||
def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
|
||||
self._keyboard = keyboard_folder(keyboard)
|
||||
self._keyboard_safe = self._keyboard.replace('/', '_')
|
||||
self._keymap = keymap
|
||||
self._parallel = 1
|
||||
self._clean = False
|
||||
self._compiledb = False
|
||||
self._target = f'{self._keyboard_safe}_{self.keymap}'
|
||||
self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
|
||||
self._generated_files_path = self._intermediate_output / 'src'
|
||||
self._json = json.to_dict() if isinstance(json, Dotty) else json
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.keyboard}:{self.keymap}'
|
||||
|
||||
def __repr__(self):
|
||||
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
|
||||
|
||||
def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
|
||||
if parallel is not None:
|
||||
self._parallel = parallel
|
||||
if clean is not None:
|
||||
self._clean = clean
|
||||
if compiledb is not None:
|
||||
self._compiledb = compiledb
|
||||
|
||||
@property
|
||||
def keyboard(self) -> str:
|
||||
return self._keyboard
|
||||
|
||||
@property
|
||||
def keymap(self) -> str:
|
||||
return self._keymap
|
||||
|
||||
@property
|
||||
def json(self) -> dict:
|
||||
if not self._json:
|
||||
self._load_json()
|
||||
if not self._json:
|
||||
return {}
|
||||
return self._json
|
||||
|
||||
@property
|
||||
def dotty(self) -> Dotty:
|
||||
return dotty(self.json)
|
||||
|
||||
def _common_make_args(self, dry_run: bool = False, build_target: str = None):
|
||||
compile_args = [
|
||||
find_make(),
|
||||
*get_make_parallel_args(self._parallel),
|
||||
'-r',
|
||||
'-R',
|
||||
'-f',
|
||||
'builddefs/build_keyboard.mk',
|
||||
]
|
||||
|
||||
if not cli.config.general.verbose:
|
||||
compile_args.append('-s')
|
||||
|
||||
verbose = 'true' if cli.config.general.verbose else 'false'
|
||||
color = 'true' if cli.config.general.color else 'false'
|
||||
|
||||
if dry_run:
|
||||
compile_args.append('-n')
|
||||
|
||||
if build_target:
|
||||
compile_args.append(build_target)
|
||||
|
||||
compile_args.extend([
|
||||
f'KEYBOARD={self.keyboard}',
|
||||
f'KEYMAP={self.keymap}',
|
||||
f'KEYBOARD_FILESAFE={self._keyboard_safe}',
|
||||
f'TARGET={self._target}',
|
||||
f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
|
||||
f'VERBOSE={verbose}',
|
||||
f'COLOR={color}',
|
||||
'SILENT=false',
|
||||
'QMK_BIN="qmk"',
|
||||
])
|
||||
|
||||
return compile_args
|
||||
|
||||
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
|
||||
raise NotImplementedError("prepare_build() not implemented in base class")
|
||||
|
||||
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
|
||||
raise NotImplementedError("compile_command() not implemented in base class")
|
||||
|
||||
def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
|
||||
self.prepare_build(build_target=build_target, **env_vars)
|
||||
command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
|
||||
write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
|
||||
|
||||
def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
|
||||
if self._clean or self._compiledb:
|
||||
command = [find_make(), "clean"]
|
||||
if dry_run:
|
||||
command.append('-n')
|
||||
cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
|
||||
cli.run(command, capture_output=False)
|
||||
|
||||
if self._compiledb and not dry_run:
|
||||
self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
|
||||
|
||||
self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
|
||||
command = self.compile_command(build_target=build_target, **env_vars)
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
|
||||
if not dry_run:
|
||||
cli.echo('\n')
|
||||
ret = cli.run(command, capture_output=False)
|
||||
if ret.returncode:
|
||||
return ret.returncode
|
||||
|
||||
|
||||
class KeyboardKeymapBuildTarget(BuildTarget):
|
||||
def __init__(self, keyboard: str, keymap: str, json: dict = None):
|
||||
super().__init__(keyboard=keyboard, keymap=keymap, json=json)
|
||||
|
||||
def __repr__(self):
|
||||
return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
|
||||
|
||||
def _load_json(self):
|
||||
self._json = keymap_json(self.keyboard, self.keymap)
|
||||
|
||||
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
|
||||
pass
|
||||
|
||||
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
|
||||
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
compile_args.append(f'{key}={value}')
|
||||
|
||||
return compile_args
|
||||
|
||||
|
||||
class JsonKeymapBuildTarget(BuildTarget):
|
||||
def __init__(self, json_path):
|
||||
if isinstance(json_path, Path):
|
||||
self.json_path = json_path
|
||||
else:
|
||||
self.json_path = None
|
||||
|
||||
json = parse_configurator_json(json_path) # Will load from stdin if provided
|
||||
|
||||
# In case the user passes a keymap.json from a keymap directory directly to the CLI.
|
||||
# e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
|
||||
json["keymap"] = json.get("keymap", "default_json")
|
||||
|
||||
super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
|
||||
|
||||
self._keymap_json = self._generated_files_path / 'keymap.json'
|
||||
|
||||
def __repr__(self):
|
||||
return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
|
||||
|
||||
def _load_json(self):
|
||||
pass # Already loaded in constructor
|
||||
|
||||
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
|
||||
if self._clean:
|
||||
if self._intermediate_output.exists():
|
||||
shutil.rmtree(self._intermediate_output)
|
||||
|
||||
# begin with making the deepest folder in the tree
|
||||
self._generated_files_path.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
# Compare minified to ensure consistent comparison
|
||||
new_content = json.dumps(self.json, separators=(',', ':'))
|
||||
if self._keymap_json.exists():
|
||||
old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
|
||||
if old_content == new_content:
|
||||
new_content = None
|
||||
|
||||
# Write the keymap.json file if different so timestamps are only updated
|
||||
# if the content changes -- running `make` won't treat it as modified.
|
||||
if new_content:
|
||||
self._keymap_json.write_text(new_content, encoding='utf-8')
|
||||
|
||||
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
|
||||
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
|
||||
compile_args.extend([
|
||||
f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
|
||||
f'KEYMAP_JSON={self._keymap_json}',
|
||||
f'KEYMAP_PATH={self._generated_files_path}',
|
||||
])
|
||||
|
||||
for key, value in env_vars.items():
|
||||
compile_args.append(f'{key}={value}')
|
||||
|
||||
return compile_args
|
||||
@ -2,7 +2,7 @@
|
||||
"""
|
||||
from subprocess import DEVNULL
|
||||
|
||||
from qmk.commands import create_make_target
|
||||
from qmk.commands import find_make
|
||||
from milc import cli
|
||||
|
||||
|
||||
@ -11,4 +11,4 @@ from milc import cli
|
||||
def clean(cli):
|
||||
"""Runs `make clean` (or `make distclean` if --all is passed)
|
||||
"""
|
||||
cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
|
||||
cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)
|
||||
|
||||
@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter
|
||||
from milc import cli
|
||||
|
||||
import qmk.path
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
|
||||
from qmk.commands import build_environment
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
|
||||
from qmk.keymap import keymap_completer, locate_keymap
|
||||
from qmk.cli.generate.compilation_database import write_compilation_database
|
||||
|
||||
|
||||
def _is_keymap_target(keyboard, keymap):
|
||||
if keymap == 'all':
|
||||
return True
|
||||
|
||||
if locate_keymap(keyboard, keymap):
|
||||
return True
|
||||
|
||||
return False
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
|
||||
@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
|
||||
@cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
|
||||
@cli.subcommand('Compile a QMK Firmware.')
|
||||
@automagic_keyboard
|
||||
@ -53,47 +43,27 @@ def compile(cli):
|
||||
# Build the environment vars
|
||||
envs = build_environment(cli.args.env)
|
||||
|
||||
# Determine the compile command
|
||||
commands = []
|
||||
|
||||
current_keyboard = None
|
||||
current_keymap = None
|
||||
# Handler for the build target
|
||||
target = None
|
||||
|
||||
if cli.args.filename:
|
||||
# If a configurator JSON was provided generate a keymap and compile it
|
||||
user_keymap = parse_configurator_json(cli.args.filename)
|
||||
commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
|
||||
# if we were given a filename, assume we have a json build target
|
||||
target = JsonKeymapBuildTarget(cli.args.filename)
|
||||
|
||||
elif cli.config.compile.keyboard and cli.config.compile.keymap:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
|
||||
# if we got a keyboard and keymap, attempt to find it
|
||||
if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
|
||||
cli.log.error('Invalid keymap argument.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.clean:
|
||||
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
|
||||
commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
|
||||
# If we got here, then we have a valid keyboard and keymap for a build target
|
||||
target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
|
||||
|
||||
current_keyboard = cli.config.compile.keyboard
|
||||
current_keymap = cli.config.compile.keymap
|
||||
|
||||
if not commands:
|
||||
if not target:
|
||||
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.compiledb:
|
||||
if current_keyboard is None or current_keymap is None:
|
||||
cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
|
||||
cli.print_help()
|
||||
return False
|
||||
write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
|
||||
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
|
||||
if not cli.args.dry_run:
|
||||
cli.echo('\n')
|
||||
for command in commands:
|
||||
ret = cli.run(command, capture_output=False)
|
||||
if ret.returncode:
|
||||
return ret.returncode
|
||||
target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
|
||||
target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)
|
||||
|
||||
@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets
|
||||
def find(cli):
|
||||
"""Search through all keyboards and keymaps for a given search criteria.
|
||||
"""
|
||||
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
print(f'{target}')
|
||||
|
||||
if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
|
||||
cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
|
||||
|
||||
targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
|
||||
for keyboard, keymap, print_vals in targets:
|
||||
print(f'{keyboard}:{keymap}')
|
||||
|
||||
for key, val in print_vals:
|
||||
print(f' {key}={val}')
|
||||
for key in cli.args.print:
|
||||
print(f' {key}={target.dotty.get(key, None)}')
|
||||
|
||||
@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
|
||||
A bootloader must be specified.
|
||||
"""
|
||||
from argcomplete.completers import FilesCompleter
|
||||
from pathlib import Path
|
||||
|
||||
from milc import cli
|
||||
|
||||
import qmk.path
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
|
||||
from qmk.commands import build_environment
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
from qmk.keymap import keymap_completer, locate_keymap
|
||||
from qmk.flashers import flasher
|
||||
|
||||
|
||||
def _is_keymap_target(keyboard, keymap):
|
||||
if keymap == 'all':
|
||||
return True
|
||||
|
||||
if locate_keymap(keyboard, keymap):
|
||||
return True
|
||||
|
||||
return False
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
def _list_bootloaders():
|
||||
@ -89,7 +81,7 @@ def flash(cli):
|
||||
|
||||
If bootloader is omitted the make system will use the configured bootloader for that keyboard.
|
||||
"""
|
||||
if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
|
||||
if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
|
||||
return _flash_binary(cli.args.filename, cli.args.mcu)
|
||||
|
||||
if cli.args.bootloaders:
|
||||
@ -98,34 +90,27 @@ def flash(cli):
|
||||
# Build the environment vars
|
||||
envs = build_environment(cli.args.env)
|
||||
|
||||
# Determine the compile command
|
||||
commands = []
|
||||
# Handler for the build target
|
||||
target = None
|
||||
|
||||
if cli.args.filename:
|
||||
# If a configurator JSON was provided generate a keymap and compile it
|
||||
user_keymap = parse_configurator_json(cli.args.filename)
|
||||
commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
|
||||
# if we were given a filename, assume we have a json build target
|
||||
target = JsonKeymapBuildTarget(cli.args.filename)
|
||||
|
||||
elif cli.config.flash.keyboard and cli.config.flash.keymap:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
|
||||
# if we got a keyboard and keymap, attempt to find it
|
||||
if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
|
||||
cli.log.error('Invalid keymap argument.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
if cli.args.clean:
|
||||
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
|
||||
commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
|
||||
# If we got here, then we have a valid keyboard and keymap for a build target
|
||||
target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
|
||||
|
||||
if not commands:
|
||||
if not target:
|
||||
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
|
||||
if not cli.args.dry_run:
|
||||
cli.echo('\n')
|
||||
for command in commands:
|
||||
ret = cli.run(command, capture_output=False)
|
||||
if ret.returncode:
|
||||
return ret.returncode
|
||||
target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
|
||||
target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)
|
||||
|
||||
@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union
|
||||
|
||||
from milc import cli, MILC
|
||||
|
||||
from qmk.commands import create_make_command
|
||||
from qmk.commands import find_make
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
|
||||
return records
|
||||
|
||||
|
||||
def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
|
||||
def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
command = create_make_command(keyboard, keymap, dry_run=True)
|
||||
if not command:
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget # Lazy load due to circular references
|
||||
target = KeyboardKeymapBuildTarget(keyboard, keymap)
|
||||
command = target.compile_command(dry_run=True, **env_vars)
|
||||
|
||||
if not command:
|
||||
cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
|
||||
env.pop("MAKEFLAGS", None)
|
||||
|
||||
# re-use same executable as the main make invocation (might be gmake)
|
||||
clean_command = [command[0], 'clean']
|
||||
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
|
||||
cli.run(clean_command, capture_output=False, check=True, env=env)
|
||||
if not skip_clean:
|
||||
clean_command = [find_make(), "clean"]
|
||||
cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
|
||||
cli.run(clean_command, capture_output=False, check=True, env=env)
|
||||
|
||||
cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
|
||||
|
||||
|
||||
@ -3,26 +3,28 @@
|
||||
This will compile everything in parallel, for testing purposes.
|
||||
"""
|
||||
import os
|
||||
from typing import List
|
||||
from pathlib import Path
|
||||
from subprocess import DEVNULL
|
||||
from milc import cli
|
||||
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.commands import _find_make, get_make_parallel_args
|
||||
from qmk.commands import find_make, get_make_parallel_args, build_environment
|
||||
from qmk.search import search_keymap_targets, search_make_targets
|
||||
from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
|
||||
|
||||
|
||||
def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
|
||||
def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
|
||||
if len(targets) == 0:
|
||||
return
|
||||
|
||||
make_cmd = _find_make()
|
||||
make_cmd = find_make()
|
||||
builddir = Path(QMK_FIRMWARE) / '.build'
|
||||
makefile = builddir / 'parallel_kb_builds.mk'
|
||||
|
||||
if dry_run:
|
||||
cli.log.info('Compilation targets:')
|
||||
for target in sorted(targets):
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
|
||||
else:
|
||||
if clean:
|
||||
@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
|
||||
|
||||
builddir.mkdir(parents=True, exist_ok=True)
|
||||
with open(makefile, "w") as f:
|
||||
for target in sorted(targets):
|
||||
keyboard_name = target[0]
|
||||
keymap_name = target[1]
|
||||
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
|
||||
keyboard_name = target.keyboard
|
||||
keymap_name = target.keymap
|
||||
target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
|
||||
target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
|
||||
command = target.compile_command(**env)
|
||||
command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
|
||||
keyboard_safe = keyboard_name.replace('/', '_')
|
||||
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
|
||||
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
|
||||
@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary
|
||||
{keyboard_safe}_{keymap_name}_binary:
|
||||
@rm -f "{build_log}" || true
|
||||
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
|
||||
+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
|
||||
{' '.join(command)} \\
|
||||
>>"{build_log}" 2>&1 \\
|
||||
|| cp "{build_log}" "{failed_log}"
|
||||
@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|
||||
@ -95,8 +101,11 @@ def mass_compile(cli):
|
||||
"""Compile QMK Firmware against all keyboards.
|
||||
"""
|
||||
if len(cli.args.builds) > 0:
|
||||
targets = search_make_targets(cli.args.builds, cli.args.filter)
|
||||
json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
|
||||
make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
|
||||
targets = search_make_targets(make_like_targets)
|
||||
targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
|
||||
else:
|
||||
targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
|
||||
|
||||
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
|
||||
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))
|
||||
|
||||
@ -2,18 +2,16 @@
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from milc import cli
|
||||
import jsonschema
|
||||
|
||||
from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
|
||||
from qmk.json_schema import json_load, validate
|
||||
|
||||
|
||||
def _find_make():
|
||||
def find_make():
|
||||
"""Returns the correct make command for this environment.
|
||||
"""
|
||||
make_cmd = os.environ.get('MAKE')
|
||||
@ -24,74 +22,6 @@ def _find_make():
|
||||
return make_cmd
|
||||
|
||||
|
||||
def create_make_target(target, dry_run=False, parallel=1, **env_vars):
|
||||
"""Create a make command
|
||||
|
||||
Args:
|
||||
|
||||
target
|
||||
Usually a make rule, such as 'clean' or 'all'.
|
||||
|
||||
dry_run
|
||||
make -n -- don't actually build
|
||||
|
||||
parallel
|
||||
The number of make jobs to run in parallel
|
||||
|
||||
**env_vars
|
||||
Environment variables to be passed to make.
|
||||
|
||||
Returns:
|
||||
|
||||
A command that can be run to make the specified keyboard and keymap
|
||||
"""
|
||||
env = []
|
||||
make_cmd = _find_make()
|
||||
|
||||
for key, value in env_vars.items():
|
||||
env.append(f'{key}={value}')
|
||||
|
||||
if cli.config.general.verbose:
|
||||
env.append('VERBOSE=true')
|
||||
|
||||
return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
|
||||
|
||||
|
||||
def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
|
||||
"""Create a make compile command
|
||||
|
||||
Args:
|
||||
|
||||
keyboard
|
||||
The path of the keyboard, for example 'plank'
|
||||
|
||||
keymap
|
||||
The name of the keymap, for example 'algernon'
|
||||
|
||||
target
|
||||
Usually a bootloader.
|
||||
|
||||
dry_run
|
||||
make -n -- don't actually build
|
||||
|
||||
parallel
|
||||
The number of make jobs to run in parallel
|
||||
|
||||
**env_vars
|
||||
Environment variables to be passed to make.
|
||||
|
||||
Returns:
|
||||
|
||||
A command that can be run to make the specified keyboard and keymap
|
||||
"""
|
||||
make_args = [keyboard, keymap]
|
||||
|
||||
if target:
|
||||
make_args.append(target)
|
||||
|
||||
return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
|
||||
|
||||
|
||||
def get_make_parallel_args(parallel=1):
|
||||
"""Returns the arguments for running the specified number of parallel jobs.
|
||||
"""
|
||||
@ -100,7 +30,7 @@ def get_make_parallel_args(parallel=1):
|
||||
if int(parallel) <= 0:
|
||||
# 0 or -1 means -j without argument (unlimited jobs)
|
||||
parallel_args.append('--jobs')
|
||||
else:
|
||||
elif int(parallel) > 1:
|
||||
parallel_args.append('--jobs=' + str(parallel))
|
||||
|
||||
if int(parallel) != 1:
|
||||
@ -110,96 +40,6 @@ def get_make_parallel_args(parallel=1):
|
||||
return parallel_args
|
||||
|
||||
|
||||
def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
|
||||
"""Convert a configurator export JSON file into a C file and then compile it.
|
||||
|
||||
Args:
|
||||
|
||||
user_keymap
|
||||
A deserialized keymap export
|
||||
|
||||
bootloader
|
||||
A bootloader to flash
|
||||
|
||||
parallel
|
||||
The number of make jobs to run in parallel
|
||||
|
||||
Returns:
|
||||
|
||||
A command to run to compile and flash the C file.
|
||||
"""
|
||||
# In case the user passes a keymap.json from a keymap directory directly to the CLI.
|
||||
# e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
|
||||
user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
|
||||
|
||||
keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
|
||||
target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
|
||||
intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
|
||||
keymap_dir = intermediate_output / 'src'
|
||||
keymap_json = keymap_dir / 'keymap.json'
|
||||
|
||||
if clean:
|
||||
if intermediate_output.exists():
|
||||
shutil.rmtree(intermediate_output)
|
||||
|
||||
# begin with making the deepest folder in the tree
|
||||
keymap_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
# Compare minified to ensure consistent comparison
|
||||
new_content = json.dumps(user_keymap, separators=(',', ':'))
|
||||
if keymap_json.exists():
|
||||
old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
|
||||
if old_content == new_content:
|
||||
new_content = None
|
||||
|
||||
# Write the keymap.json file if different
|
||||
if new_content:
|
||||
keymap_json.write_text(new_content, encoding='utf-8')
|
||||
|
||||
# Return a command that can be run to make the keymap and flash if given
|
||||
verbose = 'true' if cli.config.general.verbose else 'false'
|
||||
color = 'true' if cli.config.general.color else 'false'
|
||||
make_command = [_find_make()]
|
||||
|
||||
if not cli.config.general.verbose:
|
||||
make_command.append('-s')
|
||||
|
||||
make_command.extend([
|
||||
*get_make_parallel_args(parallel),
|
||||
'-r',
|
||||
'-R',
|
||||
'-f',
|
||||
'builddefs/build_keyboard.mk',
|
||||
])
|
||||
|
||||
if bootloader:
|
||||
make_command.append(bootloader)
|
||||
|
||||
make_command.extend([
|
||||
f'KEYBOARD={user_keymap["keyboard"]}',
|
||||
f'KEYMAP={user_keymap["keymap"]}',
|
||||
f'KEYBOARD_FILESAFE={keyboard_filesafe}',
|
||||
f'TARGET={target}',
|
||||
f'INTERMEDIATE_OUTPUT={intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_1={intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_2={intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_3={intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_4={intermediate_output}',
|
||||
f'MAIN_KEYMAP_PATH_5={intermediate_output}',
|
||||
f'KEYMAP_JSON={keymap_json}',
|
||||
f'KEYMAP_PATH={keymap_dir}',
|
||||
f'VERBOSE={verbose}',
|
||||
f'COLOR={color}',
|
||||
'SILENT=false',
|
||||
'QMK_BIN="qmk"',
|
||||
])
|
||||
|
||||
for key, value in env_vars.items():
|
||||
make_command.append(f'{key}={value}')
|
||||
|
||||
return make_command
|
||||
|
||||
|
||||
def parse_configurator_json(configurator_file):
|
||||
"""Open and parse a configurator json export
|
||||
"""
|
||||
|
||||
@ -98,11 +98,7 @@ def keyboard_folder(keyboard):
|
||||
if keyboard == last_keyboard:
|
||||
break
|
||||
|
||||
rules_mk_file = Path(base_path, keyboard, 'rules.mk')
|
||||
|
||||
if rules_mk_file.exists():
|
||||
rules_mk = parse_rules_mk_file(rules_mk_file)
|
||||
keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard)
|
||||
keyboard = resolve_keyboard(keyboard)
|
||||
|
||||
if not qmk.path.is_keyboard(keyboard):
|
||||
raise ValueError(f'Invalid keyboard: {keyboard}')
|
||||
|
||||
@ -11,8 +11,9 @@ from milc import cli
|
||||
|
||||
from qmk.util import parallel_map
|
||||
from qmk.info import keymap_json
|
||||
import qmk.keyboard
|
||||
import qmk.keymap
|
||||
from qmk.keyboard import list_keyboards, keyboard_folder
|
||||
from qmk.keymap import list_keymaps, locate_keymap
|
||||
from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
|
||||
|
||||
|
||||
def _set_log_level(level):
|
||||
@ -36,15 +37,15 @@ def _all_keymaps(keyboard):
|
||||
"""Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
|
||||
"""
|
||||
with ignore_logging():
|
||||
keyboard = qmk.keyboard.resolve_keyboard(keyboard)
|
||||
return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)]
|
||||
keyboard = keyboard_folder(keyboard)
|
||||
return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
|
||||
|
||||
|
||||
def _keymap_exists(keyboard, keymap):
|
||||
"""Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.
|
||||
"""
|
||||
with ignore_logging():
|
||||
return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
|
||||
return keyboard if locate_keymap(keyboard, keymap) is not None else None
|
||||
|
||||
|
||||
def _load_keymap_info(kb_km):
|
||||
@ -75,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
|
||||
Caters for 'all' in either keyboard or keymap, or both.
|
||||
"""
|
||||
if all_keyboards is None:
|
||||
all_keyboards = qmk.keyboard.list_keyboards()
|
||||
all_keyboards = list_keyboards()
|
||||
|
||||
if keyboard == 'all':
|
||||
if keymap == 'all':
|
||||
@ -90,30 +91,29 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
|
||||
return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
|
||||
else:
|
||||
if keymap == 'all':
|
||||
keyboard = qmk.keyboard.resolve_keyboard(keyboard)
|
||||
cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
|
||||
return _all_keymaps(keyboard)
|
||||
else:
|
||||
return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
|
||||
return [(keyboard, keymap)]
|
||||
|
||||
|
||||
def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
"""Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
|
||||
"""
|
||||
overall_targets = []
|
||||
all_keyboards = qmk.keyboard.list_keyboards()
|
||||
all_keyboards = list_keyboards()
|
||||
for target in targets:
|
||||
overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
|
||||
return list(sorted(set(overall_targets)))
|
||||
|
||||
|
||||
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
|
||||
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
|
||||
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
|
||||
|
||||
Optionally includes the values of the queried info.json keys.
|
||||
"""
|
||||
if len(filters) == 0 and len(print_vals) == 0:
|
||||
targets = [(kb, km, {}) for kb, km in target_list]
|
||||
if len(filters) == 0:
|
||||
targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
|
||||
else:
|
||||
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
|
||||
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
|
||||
@ -172,18 +172,18 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
|
||||
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
|
||||
continue
|
||||
|
||||
targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
|
||||
targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
|
||||
|
||||
return targets
|
||||
|
||||
|
||||
def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
|
||||
def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
|
||||
"""Search for build targets matching the supplied criteria.
|
||||
"""
|
||||
return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
|
||||
return _filter_keymap_targets(expand_keymap_targets(targets), filters)
|
||||
|
||||
|
||||
def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
|
||||
def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
|
||||
"""Search for build targets matching the supplied criteria.
|
||||
"""
|
||||
return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
|
||||
return _filter_keymap_targets(expand_make_targets(targets), filters)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user