summaryrefslogtreecommitdiffstats
path: root/lib/python/qmk/cli/generate/api.py
blob: 61a2f9f7329c26238d254f634526d8b3c0ea5202 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
"""This script automates the generation of the QMK API data.
"""
from pathlib import Path
import shutil
import json

from milc import cli

from qmk.datetime import current_datetime
from qmk.info import info_json
from qmk.json_schema import json_load
from qmk.keymap import list_keymaps
from qmk.keyboard import find_readme, list_keyboards
from qmk.keycodes import load_spec, list_versions, list_languages

DATA_PATH = Path('data')
TEMPLATE_PATH = DATA_PATH / 'templates/api/'
BUILD_API_PATH = Path('.build/api_data/')


def _list_constants(output_folder):
    """Produce a map of available constants
    """
    ret = {}
    for file in (output_folder / 'constants').glob('**/*_[0-9].[0-9].[0-9].json'):
        name, version = file.stem.rsplit('_', 1)
        if name not in ret:
            ret[name] = []
        ret[name].append(version)

    # Ensure content is sorted
    for name in ret:
        ret[name] = sorted(ret[name])

    return ret


def _resolve_keycode_specs(output_folder):
    """To make it easier for consumers, publish pre-merged spec files
    """
    for version in list_versions():
        overall = load_spec(version)

        output_file = output_folder / f'constants/keycodes_{version}.json'
        output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')

    for lang in list_languages():
        for version in list_versions(lang):
            overall = load_spec(version, lang)

            output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
            output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')

    # Purge files consumed by 'load_spec'
    shutil.rmtree(output_folder / 'constants/keycodes/')


def _filtered_copy(src, dst):
    src = Path(src)
    dst = Path(dst)

    if dst.suffix == '.hjson':
        data = json_load(src)

        dst = dst.with_suffix('.json')
        dst.write_text(json.dumps(data, separators=(',', ':')), encoding='utf-8')
        return dst

    if dst.suffix == '.jsonschema':
        data = json_load(src)

        dst.write_text(json.dumps(data), encoding='utf-8')
        return dst

    return shutil.copy2(src, dst)


def _filtered_keyboard_list():
    """Perform basic filtering of list_keyboards
    """
    keyboard_list = list_keyboards()
    if cli.args.filter:
        kb_list = []
        for keyboard_name in keyboard_list:
            if any(i in keyboard_name for i in cli.args.filter):
                kb_list.append(keyboard_name)
        keyboard_list = kb_list
    return keyboard_list


@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
@cli.subcommand('Generate QMK API data', hidden=False if cli.config.user.developer else True)
def generate_api(cli):
    """Generates the QMK API data.
    """
    v1_dir = BUILD_API_PATH / 'v1'
    keyboard_all_file = v1_dir / 'keyboards.json'  # A massive JSON containing everything
    keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
    keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
    keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
    constants_metadata_file = v1_dir / 'constants_metadata.json'  # Metadata for available constants
    usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target

    if BUILD_API_PATH.exists():
        shutil.rmtree(BUILD_API_PATH)

    shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
    shutil.copytree(DATA_PATH, v1_dir, copy_function=_filtered_copy)

    # Filter down when required
    keyboard_list = _filtered_keyboard_list()

    kb_all = {}
    usb_list = {}

    # Generate and write keyboard specific JSON files
    for keyboard_name in keyboard_list:
        kb_json = info_json(keyboard_name)
        kb_all[keyboard_name] = kb_json

        keyboard_dir = v1_dir / 'keyboards' / keyboard_name
        keyboard_info = keyboard_dir / 'info.json'
        keyboard_readme = keyboard_dir / 'readme.md'
        keyboard_readme_src = find_readme(keyboard_name)

        # Populate the list of JSON keymaps
        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
            kb_json['keymaps'][keymap.name] = {
                # TODO: deprecate 'url' as consumer needs to know its potentially hjson
                'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json',

                # Instead consumer should grab from API and not repo directly
                'path': (keymap / 'keymap.json').as_posix(),
            }

        keyboard_dir.mkdir(parents=True, exist_ok=True)
        keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}}, separators=(',', ':'))
        if not cli.args.dry_run:
            keyboard_info.write_text(keyboard_json, encoding='utf-8')
            cli.log.debug('Wrote file %s', keyboard_info)

            if keyboard_readme_src:
                shutil.copyfile(keyboard_readme_src, keyboard_readme)
                cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)

            # resolve keymaps as json
            for keymap in kb_json['keymaps']:
                keymap_hjson = kb_json['keymaps'][keymap]['path']
                keymap_json = v1_dir / keymap_hjson
                keymap_json.parent.mkdir(parents=True, exist_ok=True)
                keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson)), separators=(',', ':')), encoding='utf-8')
                cli.log.debug('Wrote keymap %s', keymap_json)

        if 'usb' in kb_json:
            usb = kb_json['usb']

            if 'vid' in usb and usb['vid'] not in usb_list:
                usb_list[usb['vid']] = {}

            if 'pid' in usb and usb['pid'] not in usb_list[usb['vid']]:
                usb_list[usb['vid']][usb['pid']] = {}

            if 'vid' in usb and 'pid' in usb:
                usb_list[usb['vid']][usb['pid']][keyboard_name] = usb

    # Generate data for the global files
    keyboard_list = sorted(kb_all)
    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
    keyboard_metadata = {
        'last_updated': current_datetime(),
        'keyboards': keyboard_list,
        'keyboard_aliases': keyboard_aliases,
        'usb': usb_list,
    }

    # Feature specific handling
    _resolve_keycode_specs(v1_dir)

    # Write the global JSON files
    keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, separators=(',', ':'))
    usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, separators=(',', ':'))
    keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, separators=(',', ':'))
    keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, separators=(',', ':'))
    keyboard_metadata_json = json.dumps(keyboard_metadata, separators=(',', ':'))
    constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)}, separators=(',', ':'))

    if not cli.args.dry_run:
        keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8')
        usb_file.write_text(usb_json, encoding='utf-8')
        keyboard_list_file.write_text(keyboard_list_json, encoding='utf-8')
        keyboard_aliases_file.write_text(keyboard_aliases_json, encoding='utf-8')
        keyboard_metadata_file.write_text(keyboard_metadata_json, encoding='utf-8')
        constants_metadata_file.write_text(constants_metadata_json, encoding='utf-8')