Initial DD keycode migration (#18643)

* Initial DD keycode migration

* Sort magic keycodes

4 files changed, 160 insertions, 0 deletions

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index cf5b5ad87e..9190af4e50 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -56,6 +56,7 @@ subcommands = [
     'qmk.cli.generate.info_json',
     'qmk.cli.generate.keyboard_c',
     'qmk.cli.generate.keyboard_h',
+    'qmk.cli.generate.keycodes',
     'qmk.cli.generate.rgb_breathe_table',
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
index 8d8ca3cd41..0f29cd2327 100755
--- a/lib/python/qmk/cli/generate/api.py
+++ b/lib/python/qmk/cli/generate/api.py
@@ -11,12 +11,23 @@ from qmk.info import info_json
 from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
+from qmk.keycodes import load_spec, list_versions
 
 DATA_PATH = Path('data')
 TEMPLATE_PATH = DATA_PATH / 'templates/api/'
 BUILD_API_PATH = Path('.build/api_data/')
 
 
+def _resolve_keycode_specs(output_folder):
+    """To make it easier for consumers, publish pre-merged spec files
+    """
+    for version in list_versions():
+        overall = load_spec(version)
+
+        output_file = output_folder / f'constants/keycodes_{version}.json'
+        output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8')
+
+
 def _filtered_keyboard_list():
     """Perform basic filtering of list_keyboards
     """
@@ -95,6 +106,9 @@ def generate_api(cli):
         'usb': usb_list,
     }
 
+    # Feature specific handling
+    _resolve_keycode_specs(v1_dir)
+
     # Write the global JSON files
     keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, cls=InfoJSONEncoder)
     usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, cls=InfoJSONEncoder)
diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py
new file mode 100644
index 0000000000..29b7db3c80
--- /dev/null
+++ b/lib/python/qmk/cli/generate/keycodes.py
@@ -0,0 +1,88 @@
+"""Used by the make system to generate keycodes.h from keycodes_{version}.json
+"""
+from milc import cli
+
+from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
+from qmk.commands import dump_lines
+from qmk.path import normpath
+from qmk.keycodes import load_spec
+
+
+def _generate_ranges(lines, keycodes):
+    lines.append('')
+    lines.append('enum qk_keycode_ranges {')
+    lines.append('// Ranges')
+    for key, value in keycodes["ranges"].items():
+        lo, mask = map(lambda x: int(x, 16), key.split("/"))
+        hi = lo + mask
+        define = value.get("define")
+        lines.append(f'    {define.ljust(30)} = 0x{lo:04X},')
+        lines.append(f'    {(define + "_MAX").ljust(30)} = 0x{hi:04X},')
+    lines.append('};')
+
+
+def _generate_defines(lines, keycodes):
+    lines.append('')
+    lines.append('enum qk_keycode_defines {')
+    lines.append('// Keycodes')
+    for key, value in keycodes["keycodes"].items():
+        lines.append(f'    {value.get("key")} = {key},')
+
+    lines.append('')
+    lines.append('// Alias')
+    for key, value in keycodes["keycodes"].items():
+        temp = value.get("key")
+        for alias in value.get("aliases", []):
+            lines.append(f'    {alias.ljust(10)} = {temp},')
+
+    lines.append('};')
+
+
+def _generate_helpers(lines, keycodes):
+    lines.append('')
+    lines.append('// Range Helpers')
+    for value in keycodes["ranges"].values():
+        define = value.get("define")
+        lines.append(f'#define IS_{define}(code) ((code) >= {define} && (code) <= {define + "_MAX"})')
+
+    # extract min/max
+    temp = {}
+    for key, value in keycodes["keycodes"].items():
+        group = value.get('group', None)
+        if not group:
+            continue
+        if group not in temp:
+            temp[group] = [0xFFFF, 0]
+        key = int(key, 16)
+        if key < temp[group][0]:
+            temp[group][0] = key
+        if key > temp[group][1]:
+            temp[group][1] = key
+
+    lines.append('')
+    lines.append('// Group Helpers')
+    for group, codes in temp.items():
+        lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key']
+        hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key']
+        lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})')
+
+
+@cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')
+@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.subcommand('Used by the make system to generate keycodes.h from keycodes_{version}.json', hidden=True)
+def generate_keycodes(cli):
+    """Generates the keycodes.h file.
+    """
+
+    # Build the keycodes.h file.
+    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '// clang-format off']
+
+    keycodes = load_spec(cli.args.version)
+
+    _generate_ranges(keycodes_h_lines, keycodes)
+    _generate_defines(keycodes_h_lines, keycodes)
+    _generate_helpers(keycodes_h_lines, keycodes)
+
+    # Show the results
+    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet)
diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py
new file mode 100644
index 0000000000..cf1ee0767a
--- /dev/null
+++ b/lib/python/qmk/keycodes.py
@@ -0,0 +1,57 @@
+from pathlib import Path
+
+from qmk.json_schema import deep_update, json_load, validate
+
+CONSTANTS_PATH = Path('data/constants/keycodes/')
+
+
+def _validate(spec):
+    # first throw it to the jsonschema
+    validate(spec, 'qmk.keycodes.v1')
+
+    # no duplicate keycodes
+    keycodes = []
+    for value in spec['keycodes'].values():
+        keycodes.append(value['key'])
+        keycodes.extend(value.get('aliases', []))
+    duplicates = set([x for x in keycodes if keycodes.count(x) > 1])
+    if duplicates:
+        raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})')
+
+
+def load_spec(version):
+    """Build keycode data from the requested spec file
+    """
+    if version == 'latest':
+        version = list_versions()[0]
+
+    file = CONSTANTS_PATH / f'keycodes_{version}.hjson'
+    if not file.exists():
+        raise ValueError(f'Requested keycode spec ({version}) is invalid!')
+
+    # Load base
+    spec = json_load(file)
+
+    # Merge in fragments
+    fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson')
+    for file in fragments:
+        deep_update(spec, json_load(file))
+
+    # Sort?
+    spec['keycodes'] = dict(sorted(spec['keycodes'].items()))
+
+    # Validate?
+    _validate(spec)
+
+    return spec
+
+
+def list_versions():
+    """Return available versions - sorted newest first
+    """
+    ret = []
+    for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'):
+        ret.append(file.stem.split('_')[1])
+
+    ret.sort(reverse=True)
+    return ret