From 3e59bbd7316aaa61a4f90bfa1b50f95cc967d0dc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 30 Nov 2022 20:08:54 +0000
Subject: Automate "Data Driven" migrations (#17820)

---
 lib/python/qmk/cli/__init__.py |  1 +
 lib/python/qmk/cli/migrate.py  | 81 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 lib/python/qmk/cli/migrate.py

(limited to 'lib')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 9190af4e50..4e3ce63da3 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -71,6 +71,7 @@ subcommands = [
     'qmk.cli.list.keymaps',
     'qmk.cli.list.layouts',
     'qmk.cli.kle2json',
+    'qmk.cli.migrate',
     'qmk.cli.multibuild',
     'qmk.cli.new.keyboard',
     'qmk.cli.new.keymap',
diff --git a/lib/python/qmk/cli/migrate.py b/lib/python/qmk/cli/migrate.py
new file mode 100644
index 0000000000..4164f9c8ad
--- /dev/null
+++ b/lib/python/qmk/cli/migrate.py
@@ -0,0 +1,81 @@
+"""Migrate keyboard configuration to "Data Driven"
+"""
+import json
+from pathlib import Path
+from dotty_dict import dotty
+
+from milc import cli
+
+from qmk.keyboard import keyboard_completer, keyboard_folder, resolve_keyboard
+from qmk.info import info_json, find_info_json
+from qmk.json_encoders import InfoJSONEncoder
+from qmk.json_schema import json_load
+
+
+def _candidate_files(keyboard):
+    kb_dir = Path(resolve_keyboard(keyboard))
+
+    cur_dir = Path('keyboards')
+    files = []
+    for dir in kb_dir.parts:
+        cur_dir = cur_dir / dir
+        files.append(cur_dir / 'config.h')
+        files.append(cur_dir / 'rules.mk')
+
+    return [file for file in files if file.exists()]
+
+
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the performed migrations based on the supplied value. Supported format is 'KEY' located from 'data/mappings'. May be passed multiple times.")
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name')
+@cli.subcommand('Migrate keyboard config to "Data Driven".', hidden=True)
+def migrate(cli):
+    """Migrate keyboard configuration to "Data Driven"
+    """
+    # Merge mappings as we do not care to where "KEY" is found just that its removed
+    info_config_map = json_load(Path('data/mappings/info_config.hjson'))
+    info_rules_map = json_load(Path('data/mappings/info_rules.hjson'))
+    info_map = {**info_config_map, **info_rules_map}
+
+    # Parse target info.json which will receive updates
+    target_info = Path(find_info_json(cli.args.keyboard)[0])
+    info_data = dotty(json_load(target_info))
+
+    # Already parsed used for updates
+    kb_info_json = dotty(info_json(cli.args.keyboard))
+
+    # List of candidate files
+    files = _candidate_files(cli.args.keyboard)
+
+    # Filter down keys if requested
+    keys = info_map.keys()
+    if cli.args.filter:
+        keys = list(set(keys) & set(cli.args.filter))
+
+    cli.log.info(f'{{fg_green}}Migrating keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}}.{{fg_reset}}')
+
+    # Start migration
+    for file in files:
+        cli.log.info(f'  Migrating file {file}')
+        file_contents = file.read_text(encoding='utf-8').split('\n')
+        for key in keys:
+            for num, line in enumerate(file_contents):
+                if line.startswith(f'{key} =') or line.startswith(f'#define {key} '):
+                    cli.log.info(f'    Migrating {key}...')
+
+                    while line.rstrip().endswith('\\'):
+                        file_contents.pop(num)
+                        line = file_contents[num]
+                    file_contents.pop(num)
+
+                    update_key = info_map[key]["info_key"]
+                    if update_key in kb_info_json:
+                        info_data[update_key] = kb_info_json[update_key]
+
+        file.write_text('\n'.join(file_contents), encoding='utf-8')
+
+    # Finally write out updated info.json
+    cli.log.info(f'  Updating {target_info}')
+    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder))
+
+    cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}')
+    cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.")
-- 
cgit v1.2.3


From 82760bcea65d1f9e8f39aaca06f1d3b4e22a1f64 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 3 Dec 2022 00:42:54 +0000
Subject: Apply suggested workaround for #18371 (#19226)

Fixes undefined
---
 lib/python/qmk/submodules.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py
index 52efa602a0..d37ca5e644 100644
--- a/lib/python/qmk/submodules.py
+++ b/lib/python/qmk/submodules.py
@@ -40,7 +40,7 @@ def status():
         else:
             raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status)
 
-    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
+    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --no-show-signature --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
     for log_line in submodule_logs.stdout.split('\n'):
         if not log_line:
             continue
-- 
cgit v1.2.3


From 32dabd5320b2914bb5631bd6f95b8cd7ca4419e7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 3 Dec 2022 12:04:06 +0000
Subject: Align new-keymap with new-keyboard (#19229)

---
 lib/python/qmk/cli/new/keymap.py | 56 ++++++++++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 60cb743cb6..e7823bc46d 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -1,12 +1,32 @@
 """This script automates the copying of the default keymap into your own keymap.
 """
 import shutil
-from pathlib import Path
 
-import qmk.path
+from milc import cli
+from milc.questions import question
+
+from qmk.path import is_keyboard, keymap
+from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from milc import cli
+
+
+def prompt_keyboard():
+    prompt = """{fg_yellow}Select Keyboard{style_reset_all}
+If you`re unsure you can view a full list of supported keyboards with {fg_yellow}qmk list-keyboards{style_reset_all}.
+
+Keyboard Name? """
+
+    return question(prompt)
+
+
+def prompt_user():
+    prompt = """
+{fg_yellow}Name Your Keymap{style_reset_all}
+Used for maintainer, copyright, etc
+
+Your GitHub Username? """
+    return question(prompt, default=git_get_username())
 
 
 @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
@@ -17,32 +37,34 @@ from milc import cli
 def new_keymap(cli):
     """Creates a new keymap for the keyboard of your choosing.
     """
-    # ask for user input if keyboard or keymap was not provided in the command line
-    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
-    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
+    cli.log.info('{style_bright}Generating a new keymap{style_normal}')
+    cli.echo('')
 
-    # generate keymap paths
-    kb_path = Path('keyboards') / keyboard
-    keymap_path = qmk.path.keymap(keyboard)
-    keymap_path_default = keymap_path / 'default'
-    keymap_path_new = keymap_path / keymap
+    # ask for user input if keyboard or keymap was not provided in the command line
+    kb_name = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else prompt_keyboard()
+    user_name = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else prompt_user()
 
     # check directories
-    if not kb_path.exists():
-        cli.log.error('Keyboard %s does not exist!', kb_path)
+    if not is_keyboard(kb_name):
+        cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} does not exist! Please choose a valid name.')
         return False
 
+    # generate keymap paths
+    km_path = keymap(kb_name)
+    keymap_path_default = km_path / 'default'
+    keymap_path_new = km_path / user_name
+
     if not keymap_path_default.exists():
-        cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
+        cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')
         return False
 
     if keymap_path_new.exists():
-        cli.log.error('Keymap %s already exists!', keymap_path_new)
+        cli.log.error(f'Keymap {{fg_cyan}}{user_name}{{fg_reset}} already exists! Please choose a different name.')
         return False
 
     # create user directory with default keymap files
     shutil.copytree(keymap_path_default, keymap_path_new, symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path_new)
-    cli.log.info("Compile a firmware with your new keymap by typing: \n\n\tqmk compile -kb %s -km %s\n", keyboard, keymap)
+    cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
+    cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
-- 
cgit v1.2.3


From 9bc7e9afbd32e3ed34580f1be9a79e6040158de5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 9 Dec 2022 00:54:52 +0000
Subject: Initial uk+us DD keymap_extras migration (#19031)

---
 lib/python/qmk/cli/generate/api.py      |  9 +++-
 lib/python/qmk/cli/generate/keycodes.py | 54 ++++++++++++++++++++++++
 lib/python/qmk/keycodes.py              | 73 +++++++++++++++++++++++++--------
 3 files changed, 117 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index 8650a36b84..dd4830f543 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -11,7 +11,7 @@ from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
-from qmk.keycodes import load_spec, list_versions
+from qmk.keycodes import load_spec, list_versions, list_languages
 
 DATA_PATH = Path('data')
 TEMPLATE_PATH = DATA_PATH / 'templates/api/'
@@ -44,6 +44,13 @@ def _resolve_keycode_specs(output_folder):
         output_file = output_folder / f'constants/keycodes_{version}.json'
         output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
 
+    for lang in list_languages():
+        for version in list_versions(lang):
+            overall = load_spec(version, lang)
+
+            output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
+            output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+
     # Purge files consumed by 'load_spec'
     shutil.rmtree(output_folder / 'constants/keycodes/')
 
diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index 29b7db3c80..cf80689708 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -8,6 +8,24 @@ from qmk.path import normpath
 from qmk.keycodes import load_spec
 
 
+def _render_key(key):
+    width = 7
+    if 'S(' in key:
+        width += len('S()')
+    if 'A(' in key:
+        width += len('A()')
+    if 'RCTL(' in key:
+        width += len('RCTL()')
+    if 'ALGR(' in key:
+        width += len('ALGR()')
+    return key.ljust(width)
+
+
+def _render_label(label):
+    label = label.replace("\\", "(backslash)")
+    return label
+
+
 def _generate_ranges(lines, keycodes):
     lines.append('')
     lines.append('enum qk_keycode_ranges {')
@@ -67,6 +85,22 @@ def _generate_helpers(lines, keycodes):
         lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
 
 
+def _generate_aliases(lines, keycodes):
+    lines.append('')
+    lines.append('// Aliases')
+    for key, value in keycodes["aliases"].items():
+        define = _render_key(value.get("key"))
+        val = _render_key(key)
+        label = _render_label(value.get("label"))
+
+        lines.append(f'#define {define} {val} // {label}')
+
+    lines.append('')
+    for key, value in keycodes["aliases"].items():
+        for alias in value.get("aliases", []):
+            lines.append(f'#define {alias} {value.get("key")}')
+
+
 @cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@@ -86,3 +120,23 @@ def generate_keycodes(cli):
 
     # Show the results
     dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-l', '--lang', arg_only=True, required=True, help='Language of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate keymap_{lang}.h from keycodes_{lang}_{version}.json', hidden=True)
+def generate_keycode_extras(cli):
+    """Generates the header file.
+    """
+
+    # Build the header file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keymap.h"', '// clang-format off']
+
+    keycodes = load_spec(cli.args.version, cli.args.lang)
+
+    _generate_aliases(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
index cf1ee0767a..600163bab9 100644
--- a/lib/python/qmk/keycodes.py
+++ b/lib/python/qmk/keycodes.py
@@ -2,7 +2,42 @@ from pathlib import Path
 
 from qmk.json_schema import deep_update, json_load, validate
 
-CONSTANTS_PATH = Path('data/constants/keycodes/')
+CONSTANTS_PATH = Path('data/constants/')
+KEYCODES_PATH = CONSTANTS_PATH / 'keycodes'
+EXTRAS_PATH = KEYCODES_PATH / 'extras'
+
+
+def _find_versions(path, prefix):
+    ret = []
+    for file in path.glob(f'{prefix}_[0-9].[0-9].[0-9].hjson'):
+        ret.append(file.stem.split('_')[-1])
+
+    ret.sort(reverse=True)
+    return ret
+
+
+def _load_fragments(path, prefix, version):
+    file = path / f'{prefix}_{version}.hjson'
+    if not file.exists():
+        raise ValueError(f'Requested keycode spec ({prefix}:{version}) is invalid!')
+
+    # Load base
+    spec = json_load(file)
+
+    # Merge in fragments
+    fragments = path.glob(f'{prefix}_{version}_*.hjson')
+    for file in fragments:
+        deep_update(spec, json_load(file))
+
+    return spec
+
+
+def _search_path(lang=None):
+    return EXTRAS_PATH if lang else KEYCODES_PATH
+
+
+def _search_prefix(lang=None):
+    return f'keycodes_{lang}' if lang else 'keycodes'
 
 
 def _validate(spec):
@@ -19,26 +54,20 @@ def _validate(spec):
         raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})')
 
 
-def load_spec(version):
+def load_spec(version, lang=None):
     """Build keycode data from the requested spec file
     """
     if version == 'latest':
-        version = list_versions()[0]
+        version = list_versions(lang)[0]
 
-    file = CONSTANTS_PATH / f'keycodes_{version}.hjson'
-    if not file.exists():
-        raise ValueError(f'Requested keycode spec ({version}) is invalid!')
+    path = _search_path(lang)
+    prefix = _search_prefix(lang)
 
     # Load base
-    spec = json_load(file)
-
-    # Merge in fragments
-    fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson')
-    for file in fragments:
-        deep_update(spec, json_load(file))
+    spec = _load_fragments(path, prefix, version)
 
     # Sort?
-    spec['keycodes'] = dict(sorted(spec['keycodes'].items()))
+    spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))
 
     # Validate?
     _validate(spec)
@@ -46,12 +75,20 @@ def load_spec(version):
     return spec
 
 
-def list_versions():
+def list_versions(lang=None):
     """Return available versions - sorted newest first
     """
-    ret = []
-    for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'):
-        ret.append(file.stem.split('_')[1])
+    path = _search_path(lang)
+    prefix = _search_prefix(lang)
+
+    return _find_versions(path, prefix)
+
+
+def list_languages():
+    """Return available languages
+    """
+    ret = set()
+    for file in EXTRAS_PATH.glob('keycodes_*_[0-9].[0-9].[0-9].hjson'):
+        ret.add(file.stem.split('_')[1])
 
-    ret.sort(reverse=True)
     return ret
-- 
cgit v1.2.3


From 9dc3f791965e4c54bc969dcc2f2c7597f59e2262 Mon Sep 17 00:00:00 2001
From: Stefan Kerkmann <karlk90@pm.me>
Date: Sun, 11 Dec 2022 14:04:29 +0100
Subject: [RP2040] update i2c drivers to reflect peripheral number (#19277)

---
 lib/chibios-contrib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/chibios-contrib b/lib/chibios-contrib
index bb8356fb5a..1130173eae 160000
--- a/lib/chibios-contrib
+++ b/lib/chibios-contrib
@@ -1 +1 @@
-Subproject commit bb8356fb5a3a9bbc1561826f174a9a631c614546
+Subproject commit 1130173eae6b7402443aff18ad68228acbe25cc4
-- 
cgit v1.2.3


From 102f22f7e99d87989cd95e10370863c3f96ba7e2 Mon Sep 17 00:00:00 2001
From: jpe230 <pablin.123.ra@gmail.com>
Date: Mon, 12 Dec 2022 14:51:14 -0600
Subject: [Core] Quantum Painter - LVGL Integration (#18499)

Co-authored-by: Nick Brassel <nick@tzarc.org>
---
 lib/lvgl | 1 +
 1 file changed, 1 insertion(+)
 create mode 160000 lib/lvgl

(limited to 'lib')

diff --git a/lib/lvgl b/lib/lvgl
new file mode 160000
index 0000000000..e19410f8f8
--- /dev/null
+++ b/lib/lvgl
@@ -0,0 +1 @@
+Subproject commit e19410f8f8a256609da72cff549598e0df6fa4cf
-- 
cgit v1.2.3


From 962e4c0e1854b10612bab547c3d842c5f967dd23 Mon Sep 17 00:00:00 2001
From: Stefan Kerkmann <karlk90@pm.me>
Date: Wed, 14 Dec 2022 16:31:08 +0100
Subject: [Test] Reset timer for every unit test and provide timestamps for log
 messages (#17028)

---
 lib/python/qmk/cli/__init__.py                |  1 +
 lib/python/qmk/cli/generate/keycodes_tests.py | 39 +++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 lib/python/qmk/cli/generate/keycodes_tests.py

(limited to 'lib')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 4e3ce63da3..a6d53c1cb6 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -57,6 +57,7 @@ subcommands = [
     'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
     'qmk.cli.generate.keycodes',
+    'qmk.cli.generate.keycodes_tests',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
diff --git a/lib/python/qmk/cli/generate/keycodes_tests.py b/lib/python/qmk/cli/generate/keycodes_tests.py
new file mode 100644
index 0000000000..453b4693a7
--- /dev/null
+++ b/lib/python/qmk/cli/generate/keycodes_tests.py
@@ -0,0 +1,39 @@
+"""Used by the make system to generate a keycode lookup table from keycodes_{version}.json
+"""
+from milc import cli
+
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.commands import dump_lines
+from qmk.path import normpath
+from qmk.keycodes import load_spec
+
+
+def _generate_defines(lines, keycodes):
+    lines.append('')
+    lines.append('std::map<uint16_t, std::string> KEYCODE_ID_TABLE = {')
+    for key, value in keycodes["keycodes"].items():
+        lines.append(f'    {{{value.get("key")}, "{value.get("key")}"}},')
+    lines.append('};')
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate a keycode lookup table from keycodes_{version}.json', hidden=True)
+def generate_keycodes_tests(cli):
+    """Generates a keycode to identifier lookup table for unit test output.
+    """
+
+    # Build the keycodes.h file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '// clang-format off']
+    keycodes_h_lines.append('extern "C" {\n#include <keycode.h>\n}')
+    keycodes_h_lines.append('#include <map>')
+    keycodes_h_lines.append('#include <string>')
+    keycodes_h_lines.append('#include <cstdint>')
+
+    keycodes = load_spec(cli.args.version)
+
+    _generate_defines(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
-- 
cgit v1.2.3


From e5721bbd37fcb0372fd007caae5a3f2aed060479 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 21 Dec 2022 23:35:23 +0000
Subject: Remaining DD keymap_extras migration  (#19110)

* Parse headers to data

* Regen headers from data
---
 lib/python/qmk/cli/generate/keycodes.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
index cf80689708..2ed84cd589 100644
--- a/lib/python/qmk/cli/generate/keycodes.py
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -91,9 +91,10 @@ def _generate_aliases(lines, keycodes):
     for key, value in keycodes["aliases"].items():
         define = _render_key(value.get("key"))
         val = _render_key(key)
-        label = _render_label(value.get("label"))
-
-        lines.append(f'#define {define} {val} // {label}')
+        if 'label' in value:
+            lines.append(f'#define {define} {val} // {_render_label(value.get("label"))}')
+        else:
+            lines.append(f'#define {define} {val}')
 
     lines.append('')
     for key, value in keycodes["aliases"].items():
-- 
cgit v1.2.3


From 003cee00980172c1305bf97b3c5a5801de336866 Mon Sep 17 00:00:00 2001
From: jack <0x6A73@pm.me>
Date: Fri, 23 Dec 2022 11:18:57 -0700
Subject: Validate keyboard name before accepting further input (#19394)

---
 lib/python/qmk/cli/new/keyboard.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py
index 251ad919dd..cdd3919168 100644
--- a/lib/python/qmk/cli/new/keyboard.py
+++ b/lib/python/qmk/cli/new/keyboard.py
@@ -195,11 +195,6 @@ def new_keyboard(cli):
     cli.echo('')
 
     kb_name = cli.args.keyboard if cli.args.keyboard else prompt_keyboard()
-    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
-    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
-    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
-    mcu = cli.args.type if cli.args.type else prompt_mcu()
-
     if not validate_keyboard_name(kb_name):
         cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.')
         return 1
@@ -208,6 +203,11 @@ def new_keyboard(cli):
         cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')
         return 1
 
+    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user()
+    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name)
+    default_layout = cli.args.layout if cli.args.layout else prompt_layout()
+    mcu = cli.args.type if cli.args.type else prompt_mcu()
+
     # Preprocess any development_board presets
     if mcu in dev_boards:
         defaults_map = json_load(Path('data/mappings/defaults.hjson'))
-- 
cgit v1.2.3


From e4cfbd2532aa173fe8389de5f2935989c0be804d Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sun, 1 Jan 2023 04:51:29 +0000
Subject: Allow CLI to flash .uf2 files (#19462)

---
 lib/python/qmk/cli/flash.py | 46 ++++++++++++++++++++++++---------------------
 lib/python/qmk/flashers.py  | 14 ++++++++++++++
 2 files changed, 39 insertions(+), 21 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py
index 40bfbdab56..52defb5f0d 100644
--- a/lib/python/qmk/cli/flash.py
+++ b/lib/python/qmk/cli/flash.py
@@ -11,12 +11,14 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keymap import keymap_completer
 from qmk.flashers import flasher
 
 
-def print_bootloader_help():
+def _list_bootloaders():
     """Prints the available bootloaders listed in docs.qmk.fm.
     """
+    cli.print_help()
     cli.log.info('Here are the available bootloaders:')
     cli.echo('\tavrdude')
     cli.echo('\tbootloadhid')
@@ -36,14 +38,29 @@ def print_bootloader_help():
     cli.echo('\tuf2-split-left')
     cli.echo('\tuf2-split-right')
     cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')
+    return False
+
+
+def _flash_binary(filename, mcu):
+    """Try to flash binary firmware
+    """
+    cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
+    try:
+        err, msg = flasher(mcu, filename)
+        if err:
+            cli.log.error(msg)
+            return False
+    except KeyboardInterrupt:
+        cli.log.info('Ctrl-C was pressed, exiting...')
+    return True
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.')
-@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@@ -56,30 +73,17 @@ def flash(cli):
 
     If a binary firmware is supplied, try to flash that.
 
-    If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments
-    will be ignored.
+    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
 
-    If no file is supplied, keymap and keyboard are expected.
+    If a keyboard and keymap are provided this command will build a firmware based on that.
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']:
-        # Try to flash binary firmware
-        cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
-        try:
-            err, msg = flasher(cli.args.mcu, cli.args.filename)
-            if err:
-                cli.log.error(msg)
-                return False
-        except KeyboardInterrupt:
-            cli.log.info('Ctrl-C was pressed, exiting...')
-        return True
+    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+        return _flash_binary(cli.args.filename, cli.args.mcu)
 
     if cli.args.bootloaders:
-        # Provide usage and list bootloaders
-        cli.print_help()
-        print_bootloader_help()
-        return False
+        return _list_bootloaders()
 
     # Build the environment vars
     envs = build_environment(cli.args.env)
diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py
index e902e5072f..f83665d9ac 100644
--- a/lib/python/qmk/flashers.py
+++ b/lib/python/qmk/flashers.py
@@ -71,6 +71,12 @@ def _find_usb_device(vid_hex, pid_hex):
             return usb.core.find(idVendor=vid_hex, idProduct=pid_hex)
 
 
+def _find_uf2_devices():
+    """Delegate to uf2conv.py as VID:PID pairs can potentially fluctuate more than other bootloaders
+    """
+    return cli.run(['util/uf2conv.py', '--list']).stdout.splitlines()
+
+
 def _find_bootloader():
     # To avoid running forever in the background, only look for bootloaders for 10min
     start_time = time.time()
@@ -95,6 +101,8 @@ def _find_bootloader():
                     else:
                         details = None
                     return (bl, details)
+        if _find_uf2_devices():
+            return ('_uf2_compatible_', None)
         time.sleep(0.1)
     return (None, None)
 
@@ -184,6 +192,10 @@ def _flash_mdloader(file):
     cli.run(['mdloader', '--first', '--download', file, '--restart'], capture_output=False)
 
 
+def _flash_uf2(file):
+    cli.run(['util/uf2conv.py', '--deploy', file], capture_output=False)
+
+
 def flasher(mcu, file):
     bl, details = _find_bootloader()
     # Add a small sleep to avoid race conditions
@@ -208,6 +220,8 @@ def flasher(mcu, file):
             return (True, "Specifying the MCU with '-m' is necessary for ISP flashing!")
     elif bl == 'md-boot':
         _flash_mdloader(file)
+    elif bl == '_uf2_compatible_':
+        _flash_uf2(file)
     else:
         return (True, "Known bootloader found but flashing not currently supported!")
 
-- 
cgit v1.2.3


From 24adecd9227931ebaec16115e05055e0b580d351 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sun, 1 Jan 2023 19:16:38 +0000
Subject: Implement XAP style merge semantics for DD keycodes (#19397)

---
 lib/python/qmk/json_schema.py | 36 ++++++++++++++++++++++++++++--
 lib/python/qmk/keycodes.py    | 52 ++++++++++++++++++++++++++++++-------------
 2 files changed, 71 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py
index 934e2f841f..c886a0d868 100644
--- a/lib/python/qmk/json_schema.py
+++ b/lib/python/qmk/json_schema.py
@@ -1,12 +1,13 @@
 """Functions that help us generate and use info.json files.
 """
 import json
+import hjson
+import jsonschema
 from collections.abc import Mapping
 from functools import lru_cache
+from typing import OrderedDict
 from pathlib import Path
 
-import hjson
-import jsonschema
 from milc import cli
 
 
@@ -101,3 +102,34 @@ def deep_update(origdict, newdict):
             origdict[key] = value
 
     return origdict
+
+
+def merge_ordered_dicts(dicts):
+    """Merges nested OrderedDict objects resulting from reading a hjson file.
+    Later input dicts overrides earlier dicts for plain values.
+    Arrays will be appended. If the first entry of an array is "!reset!", the contents of the array will be cleared and replaced with RHS.
+    Dictionaries will be recursively merged. If any entry is "!reset!", the contents of the dictionary will be cleared and replaced with RHS.
+    """
+    result = OrderedDict()
+
+    def add_entry(target, k, v):
+        if k in target and isinstance(v, (OrderedDict, dict)):
+            if "!reset!" in v:
+                target[k] = v
+            else:
+                target[k] = merge_ordered_dicts([target[k], v])
+            if "!reset!" in target[k]:
+                del target[k]["!reset!"]
+        elif k in target and isinstance(v, list):
+            if v[0] == '!reset!':
+                target[k] = v[1:]
+            else:
+                target[k] = target[k] + v
+        else:
+            target[k] = v
+
+    for d in dicts:
+        for (k, v) in d.items():
+            add_entry(result, k, v)
+
+    return result
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
index 600163bab9..d2f2492829 100644
--- a/lib/python/qmk/keycodes.py
+++ b/lib/python/qmk/keycodes.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from qmk.json_schema import deep_update, json_load, validate
+from qmk.json_schema import merge_ordered_dicts, deep_update, json_load, validate
 
 CONSTANTS_PATH = Path('data/constants/')
 KEYCODES_PATH = CONSTANTS_PATH / 'keycodes'
@@ -16,20 +16,13 @@ def _find_versions(path, prefix):
     return ret
 
 
-def _load_fragments(path, prefix, version):
-    file = path / f'{prefix}_{version}.hjson'
-    if not file.exists():
-        raise ValueError(f'Requested keycode spec ({prefix}:{version}) is invalid!')
+def _potential_search_versions(version, lang=None):
+    versions = list_versions(lang)
+    versions.reverse()
 
-    # Load base
-    spec = json_load(file)
+    loc = versions.index(version) + 1
 
-    # Merge in fragments
-    fragments = path.glob(f'{prefix}_{version}_*.hjson')
-    for file in fragments:
-        deep_update(spec, json_load(file))
-
-    return spec
+    return versions[:loc]
 
 
 def _search_path(lang=None):
@@ -40,6 +33,34 @@ def _search_prefix(lang=None):
     return f'keycodes_{lang}' if lang else 'keycodes'
 
 
+def _locate_files(path, prefix, versions):
+    # collate files by fragment "type"
+    files = {'_': []}
+    for version in versions:
+        files['_'].append(path / f'{prefix}_{version}.hjson')
+
+        for file in path.glob(f'{prefix}_{version}_*.hjson'):
+            fragment = file.stem.replace(f'{prefix}_{version}_', '')
+            if fragment not in files:
+                files[fragment] = []
+            files[fragment].append(file)
+
+    return files
+
+
+def _process_files(files):
+    # allow override within types of fragments - but not globally
+    spec = {}
+    for category in files.values():
+        specs = []
+        for file in category:
+            specs.append(json_load(file))
+
+        deep_update(spec, merge_ordered_dicts(specs))
+
+    return spec
+
+
 def _validate(spec):
     # first throw it to the jsonschema
     validate(spec, 'qmk.keycodes.v1')
@@ -62,9 +83,10 @@ def load_spec(version, lang=None):
 
     path = _search_path(lang)
     prefix = _search_prefix(lang)
+    versions = _potential_search_versions(version, lang)
 
-    # Load base
-    spec = _load_fragments(path, prefix, version)
+    # Load bases + any fragments
+    spec = _process_files(_locate_files(path, prefix, versions))
 
     # Sort?
     spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))
-- 
cgit v1.2.3


From 3a5a4c708f4817919d9fe7396ceb2b8e57d2e68e Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 2 Jan 2023 22:00:29 +0000
Subject: Report submodule status when not valid work-tree (#19474)

---
 lib/python/qmk/cli/doctor/check.py |  2 --
 lib/python/qmk/cli/doctor/main.py  |  2 +-
 lib/python/qmk/submodules.py       | 24 ++++++++++--------------
 3 files changed, 11 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index 8a0422ba72..a368242fdb 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -119,10 +119,8 @@ def check_submodules():
     """
     for submodule in submodules.status().values():
         if submodule['status'] is None:
-            cli.log.error('Submodule %s has not yet been cloned!', submodule['name'])
             return CheckStatus.ERROR
         elif not submodule['status']:
-            cli.log.warning('Submodule %s is not up to date!', submodule['name'])
             return CheckStatus.WARNING
 
     return CheckStatus.OK
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 1600ab8dd4..d55a11e5fd 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -142,7 +142,7 @@ def doctor(cli):
     if sub_ok == CheckStatus.OK:
         cli.log.info('Submodules are up to date.')
     else:
-        if yesno('Would you like to clone the submodules?', default=True):
+        if git_check_repo() and yesno('Would you like to clone the submodules?', default=True):
             submodules.update()
             sub_ok = check_submodules()
 
diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py
index d37ca5e644..d0050b371d 100644
--- a/lib/python/qmk/submodules.py
+++ b/lib/python/qmk/submodules.py
@@ -21,15 +21,17 @@ def status():
     status is None when the submodule doesn't exist, False when it's out of date, and True when it's current
     """
     submodules = {}
-    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30)
-
-    for line in git_cmd.stdout.split('\n'):
-        if not line:
-            continue
+    gitmodule_config = cli.run(['git', 'config', '-f', '.gitmodules', '-l'], timeout=30)
+    for line in gitmodule_config.stdout.splitlines():
+        key, value = line.split('=', maxsplit=2)
+        if key.endswith('.path'):
+            submodules[value] = {'name': value, 'status': None}
 
+    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30)
+    for line in git_cmd.stdout.splitlines():
         status = line[0]
         githash, submodule = line[1:].split()[:2]
-        submodules[submodule] = {'name': submodule, 'githash': githash}
+        submodules[submodule]['githash'] = githash
 
         if status == '-':
             submodules[submodule]['status'] = None
@@ -41,10 +43,7 @@ def status():
             raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status)
 
     submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --no-show-signature --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
-    for log_line in submodule_logs.stdout.split('\n'):
-        if not log_line:
-            continue
-
+    for log_line in submodule_logs.stdout.splitlines():
         r = log_line.split('\x01')
         submodule = r[0]
         submodules[submodule]['shorthash'] = r[1] if len(r) > 1 else ''
@@ -52,10 +51,7 @@ def status():
         submodules[submodule]['last_log_message'] = r[3] if len(r) > 3 else ''
 
     submodule_tags = cli.run(['git', 'submodule', '-q', 'foreach', '\'echo $sm_path `git describe --tags`\''])
-    for log_line in submodule_tags.stdout.split('\n'):
-        if not log_line:
-            continue
-
+    for log_line in submodule_tags.stdout.splitlines():
         r = log_line.split()
         submodule = r[0]
         submodules[submodule]['describe'] = r[1] if len(r) > 1 else ''
-- 
cgit v1.2.3


From b297531dbf645a28fd5a67e546a9ff27d49e1b1e Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 2 Jan 2023 22:11:57 +0000
Subject: Migrate 'make git-submodule' to CLI command (#19479)

---
 lib/python/qmk/cli/__init__.py      |  1 +
 lib/python/qmk/cli/git/__init__.py  |  0
 lib/python/qmk/cli/git/submodule.py | 22 ++++++++++++++++++++++
 3 files changed, 23 insertions(+)
 create mode 100644 lib/python/qmk/cli/git/__init__.py
 create mode 100644 lib/python/qmk/cli/git/submodule.py

(limited to 'lib')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index a6d53c1cb6..e0ab67da7b 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -61,6 +61,7 @@ subcommands = [
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
+    'qmk.cli.git.submodule',
     'qmk.cli.hello',
     'qmk.cli.import.kbfirmware',
     'qmk.cli.import.keyboard',
diff --git a/lib/python/qmk/cli/git/__init__.py b/lib/python/qmk/cli/git/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/git/submodule.py b/lib/python/qmk/cli/git/submodule.py
new file mode 100644
index 0000000000..bcc3868a52
--- /dev/null
+++ b/lib/python/qmk/cli/git/submodule.py
@@ -0,0 +1,22 @@
+import shutil
+from qmk.path import normpath
+
+from milc import cli
+
+REMOVE_DIRS = [
+    'lib/ugfx',
+    'lib/pico-sdk',
+    'lib/chibios-contrib/ext/mcux-sdk',
+    'lib/lvgl',
+]
+
+
+@cli.subcommand('Git Submodule actions.')
+def git_submodule(cli):
+    for folder in REMOVE_DIRS:
+        if normpath(folder).is_dir():
+            print(f"Removing '{folder}'")
+            shutil.rmtree(folder)
+
+    cli.run(['git', 'submodule', 'sync', '--recursive'], capture_output=False)
+    cli.run(['git', 'submodule', 'update', '--init', '--recursive', '--progress'], capture_output=False)
-- 
cgit v1.2.3


From c345278101b3882a2f33b078021ab31a6129120a Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Tue, 3 Jan 2023 03:15:29 +0000
Subject: Replace list_keyboards.sh with CLI calls (#19485)

---
 lib/python/qmk/cli/list/keyboards.py |  3 ++-
 lib/python/qmk/keyboard.py           | 10 +++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
index 8b6c451673..405b9210e4 100644
--- a/lib/python/qmk/cli/list/keyboards.py
+++ b/lib/python/qmk/cli/list/keyboards.py
@@ -5,9 +5,10 @@ from milc import cli
 import qmk.keyboard
 
 
+@cli.argument('--no-resolve-defaults', arg_only=True, action='store_false', help='Ignore any "DEFAULT_FOLDER" within keyboards rules.mk')
 @cli.subcommand("List the keyboards currently defined within QMK")
 def list_keyboards(cli):
     """List the keyboards currently defined within QMK
     """
-    for keyboard_name in qmk.keyboard.list_keyboards():
+    for keyboard_name in qmk.keyboard.list_keyboards(cli.args.no_resolve_defaults):
         print(keyboard_name)
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 6ddbba8fa5..0c980faf2b 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -98,14 +98,18 @@ def keyboard_completer(prefix, action, parser, parsed_args):
     return list_keyboards()
 
 
-def list_keyboards():
-    """Returns a list of all keyboards.
+def list_keyboards(resolve_defaults=True):
+    """Returns a list of all keyboards - optionally processing any DEFAULT_FOLDER.
     """
     # We avoid pathlib here because this is performance critical code.
     kb_wildcard = os.path.join(base_path, "**", "rules.mk")
     paths = [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path]
 
-    return sorted(set(map(resolve_keyboard, map(_find_name, paths))))
+    found = map(_find_name, paths)
+    if resolve_defaults:
+        found = map(resolve_keyboard, found)
+
+    return sorted(set(found))
 
 
 def resolve_keyboard(keyboard):
-- 
cgit v1.2.3


From 5c730d971ec78344e1304f746436ac0e1d61aea5 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Fri, 6 Jan 2023 04:16:52 +0000
Subject: Migrate submodule dirty check to CLI (#19488)

---
 lib/python/qmk/cli/git/submodule.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/git/submodule.py b/lib/python/qmk/cli/git/submodule.py
index bcc3868a52..9f354c021e 100644
--- a/lib/python/qmk/cli/git/submodule.py
+++ b/lib/python/qmk/cli/git/submodule.py
@@ -1,8 +1,10 @@
 import shutil
-from qmk.path import normpath
 
 from milc import cli
 
+from qmk.path import normpath
+from qmk import submodules
+
 REMOVE_DIRS = [
     'lib/ugfx',
     'lib/pico-sdk',
@@ -11,8 +13,22 @@ REMOVE_DIRS = [
 ]
 
 
+@cli.argument('--check', arg_only=True, action='store_true', help='Check if the submodules are dirty, and display a warning if they are.')
+@cli.argument('--sync', arg_only=True, action='store_true', help='Shallow clone any missing submodules.')
 @cli.subcommand('Git Submodule actions.')
 def git_submodule(cli):
+    """Git Submodule actions
+    """
+    if cli.args.check:
+        return all(item['status'] for item in submodules.status().values())
+
+    if cli.args.sync:
+        cli.run(['git', 'submodule', 'sync', '--recursive'])
+        for name, item in submodules.status().items():
+            if item['status'] is None:
+                cli.run(['git', 'submodule', 'update', '--depth=50', '--init', name], capture_output=False)
+        return True
+
     for folder in REMOVE_DIRS:
         if normpath(folder).is_dir():
             print(f"Removing '{folder}'")
-- 
cgit v1.2.3


From 974a1eaf2a1076de0cc06deeefe12e15b7e209bc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Sat, 7 Jan 2023 17:05:53 +0000
Subject: Ignore defaults.hjson values if already set (#19511)

* Ignore defaults.hjson values if already set

* Add warning when nothing is merged
---
 lib/python/qmk/info.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 7e588b5182..86b1e2c063 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -561,8 +561,16 @@ def _process_defaults(info_data):
     for default_type in defaults_map.keys():
         thing_map = defaults_map[default_type]
         if default_type in info_data:
-            for key, value in thing_map.get(info_data[default_type], {}).items():
-                info_data[key] = value
+            merged_count = 0
+            thing_items = thing_map.get(info_data[default_type], {}).items()
+            for key, value in thing_items:
+                if key not in info_data:
+                    info_data[key] = value
+                    merged_count += 1
+
+            if merged_count == 0 and len(thing_items) > 0:
+                _log_warning(info_data, 'All defaults for \'%s\' were skipped, potential redundant config or misconfiguration detected' % (default_type))
+
     return info_data
 
 
-- 
cgit v1.2.3


From 1b045b1e60c2c10178828b20036a3590784bbbfc Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 9 Jan 2023 08:21:21 +0000
Subject: Handle doctor permission issues while checking udev (#19548)

---
 lib/python/qmk/cli/doctor/linux.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py
index a803305c0d..95bafe8c64 100644
--- a/lib/python/qmk/cli/doctor/linux.py
+++ b/lib/python/qmk/cli/doctor/linux.py
@@ -78,10 +78,13 @@ def check_udev_rules():
 
         # Collect all rules from the config files
         for rule_file in udev_rules:
-            for line in rule_file.read_text(encoding='utf-8').split('\n'):
-                line = line.strip()
-                if not line.startswith("#") and len(line):
-                    current_rules.add(line)
+            try:
+                for line in rule_file.read_text(encoding='utf-8').split('\n'):
+                    line = line.strip()
+                    if not line.startswith("#") and len(line):
+                        current_rules.add(line)
+            except PermissionError:
+                cli.log.debug("Failed to read: %s", rule_file)
 
         # Check if the desired rules are among the currently present rules
         for bootloader, rules in desired_rules.items():
-- 
cgit v1.2.3


From b57714f793058cfef3e10c0bbdc9ffb02b20c5a7 Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Mon, 9 Jan 2023 09:27:41 +0000
Subject: `qmk doctor` - Handle timeouts while checking binaries (#19549)

---
 lib/python/qmk/cli/doctor/check.py | 73 ++++++++++++++++++++++----------------
 lib/python/qmk/cli/doctor/main.py  |  6 ++--
 2 files changed, 46 insertions(+), 33 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index a368242fdb..426876e98a 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -3,7 +3,7 @@
 from enum import Enum
 import re
 import shutil
-from subprocess import DEVNULL
+from subprocess import DEVNULL, TimeoutExpired
 
 from milc import cli
 from qmk import submodules
@@ -41,9 +41,8 @@ def _parse_gcc_version(version):
 def _check_arm_gcc_version():
     """Returns True if the arm-none-eabi-gcc version is not known to cause problems.
     """
-    if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']:
-        version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
-        cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
+    version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
+    cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
 
     return CheckStatus.OK  # Right now all known arm versions are ok
 
@@ -51,44 +50,37 @@ def _check_arm_gcc_version():
 def _check_avr_gcc_version():
     """Returns True if the avr-gcc version is not known to cause problems.
     """
-    rc = CheckStatus.ERROR
-    if 'output' in ESSENTIAL_BINARIES['avr-gcc']:
-        version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
+    cli.log.info('Found avr-gcc version %s', version_number)
 
-        cli.log.info('Found avr-gcc version %s', version_number)
-        rc = CheckStatus.OK
+    parsed_version = _parse_gcc_version(version_number)
+    if parsed_version['major'] > 8:
+        cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
+        return CheckStatus.WARNING
 
-        parsed_version = _parse_gcc_version(version_number)
-        if parsed_version['major'] > 8:
-            cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
-            rc = CheckStatus.WARNING
-
-    return rc
+    return CheckStatus.OK
 
 
 def _check_avrdude_version():
-    if 'output' in ESSENTIAL_BINARIES['avrdude']:
-        last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
-        version_number = last_line.split()[2][:-1]
-        cli.log.info('Found avrdude version %s', version_number)
+    last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
+    version_number = last_line.split()[2][:-1]
+    cli.log.info('Found avrdude version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_util_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-util']:
-        first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-util version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-util version %s', version_number)
 
     return CheckStatus.OK
 
 
 def _check_dfu_programmer_version():
-    if 'output' in ESSENTIAL_BINARIES['dfu-programmer']:
-        first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
-        version_number = first_line.split()[1]
-        cli.log.info('Found dfu-programmer version %s', version_number)
+    first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
+    version_number = first_line.split()[1]
+    cli.log.info('Found dfu-programmer version %s', version_number)
 
     return CheckStatus.OK
 
@@ -96,11 +88,16 @@ def _check_dfu_programmer_version():
 def check_binaries():
     """Iterates through ESSENTIAL_BINARIES and tests them.
     """
-    ok = True
+    ok = CheckStatus.OK
 
     for binary in sorted(ESSENTIAL_BINARIES):
-        if not is_executable(binary):
-            ok = False
+        try:
+            if not is_executable(binary):
+                ok = CheckStatus.ERROR
+        except TimeoutExpired:
+            cli.log.debug('Timeout checking %s', binary)
+            if ok != CheckStatus.ERROR:
+                ok = CheckStatus.WARNING
 
     return ok
 
@@ -108,8 +105,22 @@ def check_binaries():
 def check_binary_versions():
     """Check the versions of ESSENTIAL_BINARIES
     """
+    checks = {
+        'arm-none-eabi-gcc': _check_arm_gcc_version,
+        'avr-gcc': _check_avr_gcc_version,
+        'avrdude': _check_avrdude_version,
+        'dfu-util': _check_dfu_util_version,
+        'dfu-programmer': _check_dfu_programmer_version,
+    }
+
     versions = []
-    for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version):
+    for binary in sorted(ESSENTIAL_BINARIES):
+        if 'output' not in ESSENTIAL_BINARIES[binary]:
+            cli.log.warning('Unknown version for %s', binary)
+            versions.append(CheckStatus.WARNING)
+            continue
+
+        check = checks[binary]
         versions.append(check())
     return versions
 
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index d55a11e5fd..6a6feb87d1 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -119,13 +119,15 @@ def doctor(cli):
     # Make sure the basic CLI tools we need are available and can be executed.
     bin_ok = check_binaries()
 
-    if not bin_ok:
+    if bin_ok == CheckStatus.ERROR:
         if yesno('Would you like to install dependencies?', default=True):
             cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False)
             bin_ok = check_binaries()
 
-    if bin_ok:
+    if bin_ok == CheckStatus.OK:
         cli.log.info('All dependencies are installed.')
+    elif bin_ok == CheckStatus.WARNING:
+        cli.log.warning('Issues encountered while checking dependencies.')
     else:
         status = CheckStatus.ERROR
 
-- 
cgit v1.2.3


From 20474ae2321f0fb456731646b5b2e3090990b8df Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 01:38:35 +0000
Subject: Fix CLI community detection (#19562)

---
 lib/python/qmk/cli/generate/api.py |  6 ++++
 lib/python/qmk/info.py             | 11 ++++----
 lib/python/qmk/keymap.py           | 56 +++++++++++++++++++-------------------
 3 files changed, 40 insertions(+), 33 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index dd4830f543..d662e14e00 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -10,6 +10,7 @@ from qmk.datetime import current_datetime
 from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
+from qmk.keymap import list_keymaps
 from qmk.keyboard import find_readme, list_keyboards
 from qmk.keycodes import load_spec, list_versions, list_languages
 
@@ -111,6 +112,11 @@ def generate_api(cli):
     # Generate and write keyboard specific JSON files
     for keyboard_name in keyboard_list:
         kb_all[keyboard_name] = info_json(keyboard_name)
+
+        # Populate the list of JSON keymaps
+        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
+            kb_all[keyboard_name]['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
+
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
         keyboard_readme = keyboard_dir / 'readme.md'
diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 86b1e2c063..af3b0f3091 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -10,7 +10,6 @@ from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts, parse_config_h_file, find_led_config
 from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
-from qmk.keymap import list_keymaps, locate_keymap
 from qmk.commands import parse_configurator_json
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
@@ -99,10 +98,6 @@ def info_json(keyboard):
         'maintainer': 'qmk',
     }
 
-    # Populate the list of JSON keymaps
-    for keymap in list_keymaps(keyboard, c=False, fullpath=True):
-        info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
-
     # Populate layout data
     layouts, aliases = _search_keyboard_h(keyboard)
 
@@ -872,6 +867,9 @@ def find_info_json(keyboard):
 def keymap_json_config(keyboard, keymap):
     """Extract keymap level config
     """
+    # TODO: resolve keymap.py and info.py circular dependencies
+    from qmk.keymap import locate_keymap
+
     keymap_folder = locate_keymap(keyboard, keymap).parent
 
     km_info_json = parse_configurator_json(keymap_folder / 'keymap.json')
@@ -881,6 +879,9 @@ def keymap_json_config(keyboard, keymap):
 def keymap_json(keyboard, keymap):
     """Generate the info.json data for a specific keymap.
     """
+    # TODO: resolve keymap.py and info.py circular dependencies
+    from qmk.keymap import locate_keymap
+
     keymap_folder = locate_keymap(keyboard, keymap).parent
 
     # Files to scan
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 315af35b73..ce518e379d 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -12,8 +12,9 @@ from pygments.token import Token
 from pygments import lex
 
 import qmk.path
-from qmk.keyboard import find_keyboard_from_dir, rules_mk, keyboard_folder
+from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
 from qmk.errors import CppError
+from qmk.info import info_json
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -374,11 +375,11 @@ def locate_keymap(keyboard, keymap):
         return keymap_path
 
     # Check community layouts as a fallback
-    rules = rules_mk(keyboard)
+    info = info_json(keyboard)
 
-    if "LAYOUTS" in rules:
-        for layout in rules["LAYOUTS"].split():
-            community_layout = Path('layouts/community') / layout / keymap
+    for community_parent in Path('layouts').glob('*/'):
+        for layout in info.get("community_layouts", []):
+            community_layout = community_parent / layout / keymap
             if community_layout.exists():
                 if (community_layout / 'keymap.json').exists():
                     return community_layout / 'keymap.json'
@@ -408,37 +409,36 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     Returns:
         a sorted list of valid keymap names.
     """
-    # parse all the rules.mk files for the keyboard
-    rules = rules_mk(keyboard)
     names = set()
 
-    if rules is not None:
-        keyboards_dir = Path('keyboards')
-        kb_path = keyboards_dir / keyboard
+    keyboards_dir = Path('keyboards')
+    kb_path = keyboards_dir / keyboard
 
-        # walk up the directory tree until keyboards_dir
-        # and collect all directories' name with keymap.c file in it
-        while kb_path != keyboards_dir:
-            keymaps_dir = kb_path / "keymaps"
+    # walk up the directory tree until keyboards_dir
+    # and collect all directories' name with keymap.c file in it
+    while kb_path != keyboards_dir:
+        keymaps_dir = kb_path / "keymaps"
 
-            if keymaps_dir.is_dir():
-                for keymap in keymaps_dir.iterdir():
+        if keymaps_dir.is_dir():
+            for keymap in keymaps_dir.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
+
+        kb_path = kb_path.parent
+
+    # Check community layouts as a fallback
+    info = info_json(keyboard)
+
+    for community_parent in Path('layouts').glob('*/'):
+        for layout in info.get("community_layouts", []):
+            cl_path = community_parent / layout
+            if cl_path.is_dir():
+                for keymap in cl_path.iterdir():
                     if is_keymap_dir(keymap, c, json, additional_files):
                         keymap = keymap if fullpath else keymap.name
                         names.add(keymap)
 
-            kb_path = kb_path.parent
-
-        # if community layouts are supported, get them
-        if "LAYOUTS" in rules:
-            for layout in rules["LAYOUTS"].split():
-                cl_path = Path('layouts/community') / layout
-                if cl_path.is_dir():
-                    for keymap in cl_path.iterdir():
-                        if is_keymap_dir(keymap, c, json, additional_files):
-                            keymap = keymap if fullpath else keymap.name
-                            names.add(keymap)
-
     return sorted(names)
 
 
-- 
cgit v1.2.3


From e11235ee14f9cd3fc45b836eec99ed312cb137dd Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 02:13:32 +0000
Subject: De-duplicate platform detection (#19545)

---
 lib/python/qmk/info.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index af3b0f3091..789d05850c 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -751,6 +751,7 @@ def arm_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
+    info_data['platform_key'] = 'chibios'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -758,6 +759,7 @@ def arm_processor_rules(info_data, rules):
         info_data['platform'] = rules['MCU_SERIES']
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
+        info_data['platform_key'] = 'arm_atsam'
 
     return info_data
 
@@ -767,6 +769,7 @@ def avr_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'avr'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
+    info_data['platform_key'] = 'avr'
     info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'
 
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
-- 
cgit v1.2.3


From 46c85c93f05003ecc9d5b9266bc78e98cc7a843b Mon Sep 17 00:00:00 2001
From: Joel Challis <git@zvecr.com>
Date: Wed, 11 Jan 2023 19:58:27 +0000
Subject: Revert "De-duplicate platform detection (#19545)" (#19564)

This reverts commit e11235ee14f9cd3fc45b836eec99ed312cb137dd.
---
 lib/python/qmk/info.py | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'lib')

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index 789d05850c..af3b0f3091 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -751,7 +751,6 @@ def arm_processor_rules(info_data, rules):
     """
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
-    info_data['platform_key'] = 'chibios'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -759,7 +758,6 @@ def arm_processor_rules(info_data, rules):
         info_data['platform'] = rules['MCU_SERIES']
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
-        info_data['platform_key'] = 'arm_atsam'
 
     return info_data
 
@@ -769,7 +767,6 @@ def avr_processor_