1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
"""Compile all keyboards.
This will compile everything in parallel, for testing purposes.
"""
import fnmatch
import logging
import multiprocessing
import os
import re
from pathlib import Path
from subprocess import DEVNULL
from dotty_dict import dotty
from milc import cli
from qmk.constants import QMK_FIRMWARE
from qmk.commands import _find_make, get_make_parallel_args
from qmk.info import keymap_json
import qmk.keyboard
import qmk.keymap
def _set_log_level(level):
cli.acquire_lock()
old = cli.log_level
cli.log_level = level
cli.log.setLevel(level)
logging.root.setLevel(level)
cli.release_lock()
return old
def _all_keymaps(keyboard):
old = _set_log_level(logging.CRITICAL)
keymaps = qmk.keymap.list_keymaps(keyboard)
_set_log_level(old)
return (keyboard, keymaps)
def _keymap_exists(keyboard, keymap):
old = _set_log_level(logging.CRITICAL)
ret = keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
_set_log_level(old)
return ret
def _load_keymap_info(keyboard, keymap):
old = _set_log_level(logging.CRITICAL)
ret = (keyboard, keymap, keymap_json(keyboard, keymap))
_set_log_level(old)
return ret
@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.argument(
'-f',
'--filter',
arg_only=True,
action='append',
default=[],
help= # noqa: `format-python` and `pytest` don't agree here.
"Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'." # noqa: `format-python` and `pytest` don't agree here.
)
@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
def mass_compile(cli):
"""Compile QMK Firmware against all keyboards.
"""
make_cmd = _find_make()
if cli.args.clean:
cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
builddir = Path(QMK_FIRMWARE) / '.build'
makefile = builddir / 'parallel_kb_builds.mk'
targets = []
with multiprocessing.Pool() as pool:
cli.log.info(f'Retrieving list of keyboards with keymap "{cli.args.keymap}"...')
target_list = []
if cli.args.keymap == 'all':
kb_to_kms = pool.map(_all_keymaps, qmk.keyboard.list_keyboards())
for targets in kb_to_kms:
keyboard = targets[0]
keymaps = targets[1]
target_list.extend([(keyboard, keymap) for keymap in keymaps])
else:
target_list = [(kb, cli.args.keymap) for kb in filter(lambda kb: kb is not None, pool.starmap(_keymap_exists, [(kb, cli.args.keymap) for kb in qmk.keyboard.list_keyboards()]))]
if len(cli.args.filter) == 0:
targets = target_list
else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in pool.starmap(_load_keymap_info, target_list)]
equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
exists_re = re.compile(r'^exists\((?P<key>[a-zA-Z0-9_\.]+)\)$')
for filter_txt in cli.args.filter:
f = equals_re.match(filter_txt)
if f is not None:
key = f.group('key')
value = f.group('value')
cli.log.info(f'Filtering on condition ("{key}" == "{value}")...')
def _make_filter(k, v):
expr = fnmatch.translate(v)
rule = re.compile(f'^{expr}$', re.IGNORECASE)
def f(e):
lhs = e[2].get(k)
lhs = str(False if lhs is None else lhs)
return rule.search(lhs) is not None
return f
valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
f = exists_re.match(filter_txt)
if f is not None:
key = f.group('key')
cli.log.info(f'Filtering on condition (exists: "{key}")...')
valid_keymaps = filter(lambda e: e[2].get(key) is not None, valid_keymaps)
targets = [(e[0], e[1]) for e in valid_keymaps]
if len(targets) == 0:
return
builddir.mkdir(parents=True, exist_ok=True)
with open(makefile, "w") as f:
for target in sorted(targets):
keyboard_name = target[0]
keymap_name = target[1]
keyboard_safe = keyboard_name.replace('/', '_')
# yapf: disable
f.write(
f"""\
all: {keyboard_safe}_{keymap_name}_binary
{keyboard_safe}_{keymap_name}_binary:
@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}.{keymap_name}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\
>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 2>&1 \\
|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{keymap_name}"
@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" || true
"""# noqa
)
# yapf: enable
if cli.args.no_temp:
# yapf: disable
f.write(
f"""\
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.hex" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.bin" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.uf2" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true
"""# noqa
)
# yapf: enable
f.write('\n')
cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
# Check for failures
failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
if len(failures) > 0:
return False
|
