From a25dd58bc56b0c4010673723ac44eaff914979bb Mon Sep 17 00:00:00 2001
From: skullydazed <skullydazed@users.noreply.github.com>
Date: Mon, 15 Jul 2019 12:14:27 -0700
Subject: QMK CLI and JSON keymap support (#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function
---
 lib/python/qmk/cli/compile/__init__.py |  0
 lib/python/qmk/cli/compile/json.py     | 44 +++++++++++++++++++++++++++
 lib/python/qmk/cli/doctor.py           | 47 +++++++++++++++++++++++++++++
 lib/python/qmk/cli/hello.py            | 13 ++++++++
 lib/python/qmk/cli/json/__init__.py    |  0
 lib/python/qmk/cli/json/keymap.py      | 54 ++++++++++++++++++++++++++++++++++
 6 files changed, 158 insertions(+)
 create mode 100644 lib/python/qmk/cli/compile/__init__.py
 create mode 100755 lib/python/qmk/cli/compile/json.py
 create mode 100755 lib/python/qmk/cli/doctor.py
 create mode 100755 lib/python/qmk/cli/hello.py
 create mode 100644 lib/python/qmk/cli/json/__init__.py
 create mode 100755 lib/python/qmk/cli/json/keymap.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/compile/__init__.py b/lib/python/qmk/cli/compile/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/compile/json.py b/lib/python/qmk/cli/compile/json.py
new file mode 100755
index 0000000000..89c16b2063
--- /dev/null
+++ b/lib/python/qmk/cli/compile/json.py
@@ -0,0 +1,44 @@
+"""Create a keymap directory from a configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', help='Configurator JSON export')
+@cli.entrypoint('Compile a QMK Configurator export.')
+def main(cli):
+    """Compile a QMK Configurator export.
+
+    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        exit(1)
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    # Compile the keymap
+    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
new file mode 100755
index 0000000000..9ce765a4b5
--- /dev/null
+++ b/lib/python/qmk/cli/doctor.py
@@ -0,0 +1,47 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import shutil
+import platform
+import os
+
+from milc import cli
+
+
+@cli.entrypoint('Basic QMK environment checks')
+def main(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Run the binaries to make sure they work
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+
+    cli.log.info('QMK Doctor is Checking your environment')
+
+    ok = True
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            ok = False
+
+    OS = platform.system()
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS")
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}linux")
+        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+        if os.system(test) == 0:
+            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows")
+
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')
diff --git a/lib/python/qmk/cli/hello.py b/lib/python/qmk/cli/hello.py
new file mode 100755
index 0000000000..bc0cb6de18
--- /dev/null
+++ b/lib/python/qmk/cli/hello.py
@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Hello World.')
+def main(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.general.name)
diff --git a/lib/python/qmk/cli/json/__init__.py b/lib/python/qmk/cli/json/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
new file mode 100755
index 0000000000..35fc8f9c0e
--- /dev/null
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -0,0 +1,54 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+import sys
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', help='File to write to')
+@cli.argument('filename', help='Configurator JSON file')
+@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
+def main(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)
-- 
cgit v1.2.3


From 7d557a0514e2cef42a3d460f6cc78771b5df0a30 Mon Sep 17 00:00:00 2001
From: skullydazed <skullydazed@users.noreply.github.com>
Date: Mon, 15 Jul 2019 15:12:35 -0700
Subject: Fix compiling json files. (#6340)

---
 lib/python/qmk/cli/json/keymap.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
index 35fc8f9c0e..e2d0b58093 100755
--- a/lib/python/qmk/cli/json/keymap.py
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -28,8 +28,8 @@ def main(cli):
         exit(1)
 
     # Environment processing
-    if cli.args.output == ('-'):
-        cli.args.output = None
+    if cli.config.general.output == ('-'):
+        cli.config.general.output = None
 
     # Parse the configurator json
     with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
@@ -38,17 +38,17 @@ def main(cli):
     # Generate the keymap
     keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
 
-    if cli.args.output:
-        output_dir = os.path.dirname(cli.args.output)
+    if cli.config.general.output:
+        output_dir = os.path.dirname(cli.config.general.output)
 
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
 
-        output_file = qmk.path.normpath(cli.args.output)
+        output_file = qmk.path.normpath(cli.config.general.output)
         with open(output_file, 'w') as keymap_fd:
             keymap_fd.write(keymap_c)
 
-        cli.log.info('Wrote keymap to %s.', cli.args.output)
+        cli.log.info('Wrote keymap to %s.', cli.config.general.output)
 
     else:
         print(keymap_c)
-- 
cgit v1.2.3


From f22c5c17b6fe069bec1241262a1c27eb89d3d3af Mon Sep 17 00:00:00 2001
From: skullydazed <skullydazed@users.noreply.github.com>
Date: Sun, 25 Aug 2019 11:58:24 -0700
Subject: Refactor `qmk compile-json` to `qmk compile` (#6592)

---
 lib/python/qmk/cli/compile.py          | 53 ++++++++++++++++++++++++++++++++++
 lib/python/qmk/cli/compile/__init__.py |  0
 lib/python/qmk/cli/compile/json.py     | 44 ----------------------------
 3 files changed, 53 insertions(+), 44 deletions(-)
 create mode 100755 lib/python/qmk/cli/compile.py
 delete mode 100644 lib/python/qmk/cli/compile/__init__.py
 delete mode 100755 lib/python/qmk/cli/compile/json.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
new file mode 100755
index 0000000000..7e14ad8fbf
--- /dev/null
+++ b/lib/python/qmk/cli/compile.py
@@ -0,0 +1,53 @@
+"""Compile a QMK Firmware.
+
+You can compile a keymap already in the repo or using a QMK Configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+from argparse import FileType
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', nargs='?', type=FileType('r'), help='The configurator export to compile')
+@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.entrypoint('Compile a QMK Firmware.')
+def main(cli):
+    """Compile a QMK Firmware.
+
+    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+
+    If --keyboard and --keymap are provided this command will build a firmware based on that.
+
+    """
+    if cli.args.filename:
+        # Parse the configurator json
+        user_keymap = json.load(cli.args.filename)
+
+        # Generate the keymap
+        keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+        cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+        qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+        cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+        # Compile the keymap
+        command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+
+    elif cli.config.general.keyboard and cli.config.general.keymap:
+        # Generate the make command for a specific keyboard/keymap.
+        command = ['make', ':'.join((cli.config.general.keyboard, cli.config.general.keymap))]
+
+    else:
+        cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`.')
+        return False
+
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)
diff --git a/lib/python/qmk/cli/compile/__init__.py b/lib/python/qmk/cli/compile/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/lib/python/qmk/cli/compile/json.py b/lib/python/qmk/cli/compile/json.py
deleted file mode 100755
index 89c16b2063..0000000000
--- a/lib/python/qmk/cli/compile/json.py
+++ /dev/null
@@ -1,44 +0,0 @@
-"""Create a keymap directory from a configurator export.
-"""
-import json
-import os
-import sys
-import subprocess
-
-from milc import cli
-
-import qmk.keymap
-import qmk.path
-
-
-@cli.argument('filename', help='Configurator JSON export')
-@cli.entrypoint('Compile a QMK Configurator export.')
-def main(cli):
-    """Compile a QMK Configurator export.
-
-    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
-
-    FIXME(skullydazed): add code to check and warn if the keymap already exists
-    """
-    # Error checking
-    if cli.args.filename == ('-'):
-        cli.log.error('Reading from STDIN is not (yet) supported.')
-        exit(1)
-    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
-        cli.log.error('JSON file does not exist!')
-        exit(1)
-
-    # Parse the configurator json
-    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
-        user_keymap = json.load(fd)
-
-    # Generate the keymap
-    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
-    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
-    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
-
-    # Compile the keymap
-    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
-    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
-    subprocess.run(command)
-- 
cgit v1.2.3


From 95477749629407e2a9e33c6ccf26ecc8b24ab07a Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Thu, 22 Aug 2019 10:18:52 -0700
Subject: CLI command to format C code

---
 lib/python/qmk/cli/cformat.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 lib/python/qmk/cli/cformat.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
new file mode 100644
index 0000000000..f7020f4c5f
--- /dev/null
+++ b/lib/python/qmk/cli/cformat.py
@@ -0,0 +1,27 @@
+"""Format C code according to QMK's style.
+"""
+import os
+import subprocess
+
+from milc import cli
+
+
+@cli.entrypoint("Format C code according to QMK's style.")
+def main(cli):
+    """Format C code according to QMK's style.
+    """
+    clang_format = ['clang-format', '-i']
+    code_files = []
+    for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
+        for dirpath, dirnames, filenames in os.walk(dir):
+            if 'tmk_core/protocol/usb_hid' in dirpath:
+                continue
+            for name in filenames:
+                if name.endswith('.c') or name.endswith('.h') or name.endswith('.cpp'):
+                    code_files.append(os.path.join(dirpath, name))
+
+    try:
+        subprocess.run(clang_format + code_files, check=True)
+        cli.log.info('Successfully formatted the C code.')
+    except subprocess.CalledProcessError:
+        cli.log.error('Error formatting C code!')
-- 
cgit v1.2.3


From 1784d1bfac44a63bf343b6e2098f0cba81d58cb2 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Thu, 22 Aug 2019 13:30:50 -0700
Subject: Add support for passing files at the command line

---
 lib/python/qmk/cli/cformat.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index f7020f4c5f..0c209247d9 100644
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -6,22 +6,24 @@ import subprocess
 from milc import cli
 
 
+@cli.argument('files', nargs='*', help='Filename(s) to format.')
 @cli.entrypoint("Format C code according to QMK's style.")
 def main(cli):
     """Format C code according to QMK's style.
     """
     clang_format = ['clang-format', '-i']
-    code_files = []
-    for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
-        for dirpath, dirnames, filenames in os.walk(dir):
-            if 'tmk_core/protocol/usb_hid' in dirpath:
-                continue
-            for name in filenames:
-                if name.endswith('.c') or name.endswith('.h') or name.endswith('.cpp'):
-                    code_files.append(os.path.join(dirpath, name))
+    if not cli.args.files:
+        for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
+            for dirpath, dirnames, filenames in os.walk(dir):
+                if 'tmk_core/protocol/usb_hid' in dirpath:
+                    continue
+                for name in filenames:
+                    if name.endswith('.c') or name.endswith('.h') or name.endswith('.cpp'):
+                        cli.args.files.append(os.path.join(dirpath, name))
 
     try:
-        subprocess.run(clang_format + code_files, check=True)
+        subprocess.run(clang_format + cli.args.files, check=True)
         cli.log.info('Successfully formatted the C code.')
     except subprocess.CalledProcessError:
         cli.log.error('Error formatting C code!')
+        return False
-- 
cgit v1.2.3


From 2d688ad14e727bd3437f26a53bd3d92079e5b3c2 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Thu, 22 Aug 2019 13:33:34 -0700
Subject: readability enhancements

---
 lib/python/qmk/cli/cformat.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index 0c209247d9..91e650368b 100644
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -12,18 +12,23 @@ def main(cli):
     """Format C code according to QMK's style.
     """
     clang_format = ['clang-format', '-i']
+
+    # Find the list of files to format
     if not cli.args.files:
         for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
             for dirpath, dirnames, filenames in os.walk(dir):
                 if 'tmk_core/protocol/usb_hid' in dirpath:
                     continue
+
                 for name in filenames:
                     if name.endswith('.c') or name.endswith('.h') or name.endswith('.cpp'):
                         cli.args.files.append(os.path.join(dirpath, name))
 
+    # Run clang-format on the files we've found
     try:
         subprocess.run(clang_format + cli.args.files, check=True)
         cli.log.info('Successfully formatted the C code.')
+
     except subprocess.CalledProcessError:
         cli.log.error('Error formatting C code!')
         return False
-- 
cgit v1.2.3


From 5b7a5b2a7629fbb667d23a55836dce3c6c46a203 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Wed, 21 Aug 2019 23:40:24 -0700
Subject: Setup a python test framework

---
 lib/python/qmk/cli/doctor.py | 30 ++++++++++++++++++++++--------
 lib/python/qmk/cli/nose2.py  | 18 ++++++++++++++++++
 2 files changed, 40 insertions(+), 8 deletions(-)
 create mode 100644 lib/python/qmk/cli/nose2.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 9ce765a4b5..c5a144363c 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -2,9 +2,11 @@
 
 Check up for QMK environment.
 """
-import shutil
-import platform
 import os
+import platform
+import shutil
+import subprocess
+from glob import glob
 
 from milc import cli
 
@@ -16,32 +18,44 @@ def main(cli):
     This is currently very simple, it just checks that all the expected binaries are on your system.
 
     TODO(unclaimed):
-        * [ ] Run the binaries to make sure they work
         * [ ] Compile a trivial program with each compiler
         * [ ] Check for udev entries on linux
     """
 
     binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+    binaries += glob('bin/qmk-*')
 
-    cli.log.info('QMK Doctor is Checking your environment')
+    cli.log.info('QMK Doctor is checking your environment')
 
     ok = True
     for binary in binaries:
         res = shutil.which(binary)
         if res is None:
-            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            cli.log.error("{fg_red}QMK can't find %s in your path", binary)
             ok = False
+        else:
+            try:
+                subprocess.run([binary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5, check=True)
+            except subprocess.CalledProcessError:
+                cli.log.error("{fg_red}Can't run `%s --version`", binary)
+                ok = False
 
     OS = platform.system()
     if OS == "Darwin":
         cli.log.info("Detected {fg_cyan}macOS")
     elif OS == "Linux":
         cli.log.info("Detected {fg_cyan}linux")
-        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
-        if os.system(test) == 0:
-            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+        if shutil.which('systemctl'):
+            test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+            if os.system(test) == 0:
+                cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+        else:
+            cli.log.warn("Can't find systemctl to check for ModemManager.")
     else:
         cli.log.info("Assuming {fg_cyan}Windows")
 
     if ok:
         cli.log.info('{fg_green}QMK is ready to go')
+    else:
+        cli.log.info('{fg_yellow}Problems detected, please fix these problems before proceeding.')
+        # FIXME(skullydazed): Link to a document about troubleshooting, or discord or something
diff --git a/lib/python/qmk/cli/nose2.py b/lib/python/qmk/cli/nose2.py
new file mode 100644
index 0000000000..c6c9c67b30
--- /dev/null
+++ b/lib/python/qmk/cli/nose2.py
@@ -0,0 +1,18 @@
+"""QMK Python Unit Tests
+
+QMK script to run unit and integration tests against our python code.
+"""
+from milc import cli
+
+
+@cli.entrypoint('QMK Python Unit Tests')
+def main(cli):
+    """Use nose2 to run unittests
+    """
+    try:
+        import nose2
+    except ImportError:
+        cli.log.error('Could not import nose2! Please install it with {fg_cyan}pip3 install nose2')
+        return False
+
+    nose2.discover()
-- 
cgit v1.2.3


From 533d6d6a464d41d23a39cecfe42d95d2e400d335 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Thu, 22 Aug 2019 09:38:10 -0700
Subject: Make the modem manager check more pythonic

---
 lib/python/qmk/cli/doctor.py | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index c5a144363c..5a713b20f5 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -21,17 +21,17 @@ def main(cli):
         * [ ] Compile a trivial program with each compiler
         * [ ] Check for udev entries on linux
     """
+    cli.log.info('QMK Doctor is checking your environment.')
 
+    # Make sure the basic CLI tools we need are available and can be executed.
     binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
     binaries += glob('bin/qmk-*')
-
-    cli.log.info('QMK Doctor is checking your environment')
-
     ok = True
+
     for binary in binaries:
         res = shutil.which(binary)
         if res is None:
-            cli.log.error("{fg_red}QMK can't find %s in your path", binary)
+            cli.log.error("{fg_red}QMK can't find %s in your path.", binary)
             ok = False
         else:
             try:
@@ -40,20 +40,36 @@ def main(cli):
                 cli.log.error("{fg_red}Can't run `%s --version`", binary)
                 ok = False
 
+    # Determine our OS and run platform specific tests
     OS = platform.system()
+
     if OS == "Darwin":
-        cli.log.info("Detected {fg_cyan}macOS")
+        cli.log.info("Detected {fg_cyan}macOS.")
+
     elif OS == "Linux":
-        cli.log.info("Detected {fg_cyan}linux")
+        cli.log.info("Detected {fg_cyan}Linux.")
         if shutil.which('systemctl'):
-            test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
-            if os.system(test) == 0:
-                cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+            mm_check = subprocess.run(['systemctl', 'list-unit-files'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10)
+            if mm_check.returncode == 0:
+                mm = True
+                for line in mm_check.stdout.split('\n'):
+                    if 'ModemManager' in line and 'enabled' in line:
+                        mm = False
+
+                if mm:
+                    cli.log.warn("{bg_yellow}Detected ModemManager. Please disable it if you are using a Pro-Micro.")
+
+            else:
+                cli.log.error('{bg_red}Could not run `systemctl list-unit-files`:')
+                cli.log.error(mm_check.stderr)
+
         else:
             cli.log.warn("Can't find systemctl to check for ModemManager.")
+
     else:
-        cli.log.info("Assuming {fg_cyan}Windows")
+        cli.log.info("Assuming {fg_cyan}Windows.")
 
+    # Report a summary of our findings to the user
     if ok:
         cli.log.info('{fg_green}QMK is ready to go')
     else:
-- 
cgit v1.2.3


From deb6fa6a87b12fcdbf577837c6faeb854cd287c7 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Thu, 22 Aug 2019 09:40:12 -0700
Subject: Add a command to format python code

---
 lib/python/qmk/cli/pyformat.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100755 lib/python/qmk/cli/pyformat.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
new file mode 100755
index 0000000000..b1f8c02b28
--- /dev/null
+++ b/lib/python/qmk/cli/pyformat.py
@@ -0,0 +1,16 @@
+"""Format python code according to QMK's style.
+"""
+from milc import cli
+
+import subprocess
+
+
+@cli.entrypoint("Format python code according to QMK's style.")
+def main(cli):
+    """Format python code according to QMK's style.
+    """
+    try:
+        subprocess.run(['yapf', '-vv', '-ri', 'bin/qmk', 'lib/python'], check=True)
+        cli.log.info('Successfully formatted the python code in `bin/qmk` and `lib/python`.')
+    except subprocess.CalledProcessError:
+        cli.log.error('Error formatting python code!')
-- 
cgit v1.2.3


From 595232ec989499d0ee0b8db0b55f6e8ca418c786 Mon Sep 17 00:00:00 2001
From: Kenny Hoang <kenny.hoang87@gmail.com>
Date: Tue, 10 Sep 2019 08:14:25 -0400
Subject: Created new_keymap.py, python version of new_keymap.sh (#6066)

* Created python version of new_keymap.sh: new_keymap.py

* Updated usage message

* Updated new_keymap.py to use python3.5+ syntax & be more similar to new_keyboard.sh

* Updated complete message

* Updated usage in argparser and removed incorrect usage_message

* Reverted the fstrings back to strings that use .format() & updated docstring convention

* Added helper to recursively cd .. until at qmk_firmware root directory

* Revert "Added helper to recursively cd .. until at qmk_firmware root directory"

This reverts commit 61a0ff3b25f91901287bec8d58eb51a1f126e2ad.

* Updated new_keymap.py to use printf-style format strings

* First draft lib/python/qmk/cli/new/keymap.py with milc

* Removed shebang & syspath appending lines

* Added optional args & resolved some cr comemnts

* Added a docstring and updated strings
---
 lib/python/qmk/cli/__init__.py     |  0
 lib/python/qmk/cli/new/__init__.py |  0
 lib/python/qmk/cli/new/keymap.py   | 41 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 lib/python/qmk/cli/__init__.py
 create mode 100644 lib/python/qmk/cli/new/__init__.py
 create mode 100755 lib/python/qmk/cli/new/keymap.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/new/__init__.py b/lib/python/qmk/cli/new/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
new file mode 100755
index 0000000000..b378e5ab43
--- /dev/null
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -0,0 +1,41 @@
+"""This script automates the copying of the default keymap into your own keymap.
+"""
+import os
+import shutil
+
+from milc import cli
+
+
+@cli.argument('-k', '--keyboard', help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
+@cli.argument('-u', '--username', help='Specify any name for the new keymap directory')
+@cli.entrypoint('Creates a new keymap for the keyboard of your choosing')
+def main(cli):
+    """Creates a new keymap for the keyboard of your choosing.
+    """
+    # ask for user input if keyboard or username was not provided in the command line
+    keyboard = cli.config.general.keyboard if cli.config.general.keyboard else input("Keyboard Name: ")
+    username = cli.config.general.username if cli.config.general.username else input("Username: ")
+
+    # generate keymap paths
+    kb_path = os.path.join(os.getcwd(), "keyboards", keyboard)
+    keymap_path_default = os.path.join(kb_path, "keymaps/default")
+    keymap_path = os.path.join(kb_path, "keymaps/%s" % username)
+
+    # check directories
+    if not os.path.exists(kb_path):
+        cli.log.error('Keyboard %s does not exist!', kb_path)
+        exit(1)
+    if not os.path.exists(keymap_path_default):
+        cli.log.error('Keyboard default %s does not exist!', keymap_path_default)
+        exit(1)
+    if os.path.exists(keymap_path):
+        cli.log.error('Keymap %s already exists!', keymap_path)
+        exit(1)
+
+    # create user directory with default keymap files
+    shutil.copytree(keymap_path_default, keymap_path, symlinks=True)
+
+    # end message to user
+    cli.log.info("%s keymap directory created in: %s\n" +
+                 "Compile a firmware file with your new keymap by typing: \n" +
+                 "qmk compile -kb %s -km %s", username, keymap_path, keyboard, username)
-- 
cgit v1.2.3


From d569f0877155efc752994f8a21f5cf001f9d6ae6 Mon Sep 17 00:00:00 2001
From: skullydazed <skullydazed@users.noreply.github.com>
Date: Sun, 22 Sep 2019 13:25:33 -0700
Subject: Configuration system for CLI (#6708)

* Rework how bin/qmk handles subcommands

* qmk config wip

* Code to show all configs

* Fully working `qmk config` command

* Mark some CLI arguments so they don't pollute the config file

* Fleshed out config support, nicer subcommand support

* sync with installable cli

* pyformat

* Add a test for subcommand_modules

* Documentation for the `qmk config` command

* split config_token on space so qmk config is more predictable

* Rework how subcommands are imported

* Document `arg_only`

* Document deleting from CLI

* Document how multiple operations work

* Add cli config to the doc index

* Add tests for the cli commands

* Make running the tests more reliable

* Be more selective about building all default keymaps

* Update new-keymap to fit the new subcommand style

* Add documentation about writing CLI scripts

* Document new-keyboard

* Update docs/cli_configuration.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Address yan's comments.

* Apply suggestions from code review

suggestions from @noahfrederick

Co-Authored-By: Noah Frederick <code@noahfrederick.com>

* Apply suggestions from code review

Co-Authored-By: Noah Frederick <code@noahfrederick.com>

* Remove pip3 from the test runner
---
 lib/python/qmk/cli/__init__.py      | 13 +++++
 lib/python/qmk/cli/cformat.py       |  6 +--
 lib/python/qmk/cli/compile.py       | 10 ++--
 lib/python/qmk/cli/config.py        | 96 +++++++++++++++++++++++++++++++++++++
 lib/python/qmk/cli/doctor.py        |  5 +-
 lib/python/qmk/cli/hello.py         |  6 +--
 lib/python/qmk/cli/json/__init__.py |  5 ++
 lib/python/qmk/cli/json/keymap.py   | 20 ++++----
 lib/python/qmk/cli/new/__init__.py  |  1 +
 lib/python/qmk/cli/new/keymap.py    | 17 ++++---
 lib/python/qmk/cli/nose2.py         | 18 -------
 lib/python/qmk/cli/pyformat.py      |  5 +-
 lib/python/qmk/cli/pytest.py        | 20 ++++++++
 13 files changed, 170 insertions(+), 52 deletions(-)
 create mode 100644 lib/python/qmk/cli/config.py
 delete mode 100644 lib/python/qmk/cli/nose2.py
 create mode 100644 lib/python/qmk/cli/pytest.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index e69de29bb2..fb4e0ecb46 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -0,0 +1,13 @@
+"""QMK CLI Subcommands
+
+We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
+"""
+from . import cformat
+from . import compile
+from . import config
+from . import doctor
+from . import hello
+from . import json
+from . import new
+from . import pyformat
+from . import pytest
diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index 91e650368b..d2382bdbde 100644
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -6,9 +6,9 @@ import subprocess
 from milc import cli
 
 
-@cli.argument('files', nargs='*', help='Filename(s) to format.')
-@cli.entrypoint("Format C code according to QMK's style.")
-def main(cli):
+@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
+@cli.subcommand("Format C code according to QMK's style.")
+def cformat(cli):
     """Format C code according to QMK's style.
     """
     clang_format = ['clang-format', '-i']
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 7e14ad8fbf..6646891b30 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -14,11 +14,11 @@ import qmk.keymap
 import qmk.path
 
 
-@cli.argument('filename', nargs='?', type=FileType('r'), help='The configurator export to compile')
+@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile')
 @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
-@cli.entrypoint('Compile a QMK Firmware.')
-def main(cli):
+@cli.subcommand('Compile a QMK Firmware.')
+def compile(cli):
     """Compile a QMK Firmware.
 
     If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists.
@@ -41,9 +41,9 @@ def main(cli):
         # Compile the keymap
         command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
 
-    elif cli.config.general.keyboard and cli.config.general.keymap:
+    elif cli.config.compile.keyboard and cli.config.compile.keymap:
         # Generate the make command for a specific keyboard/keymap.
-        command = ['make', ':'.join((cli.config.general.keyboard, cli.config.general.keymap))]
+        command = ['make', ':'.join((cli.config.compile.keyboard, cli.config.compile.keymap))]
 
     else:
         cli.log.error('You must supply a configurator export or both `--keyboard` and `--keymap`.')
diff --git a/lib/python/qmk/cli/config.py b/lib/python/qmk/cli/config.py
new file mode 100644
index 0000000000..d6c774e651
--- /dev/null
+++ b/lib/python/qmk/cli/config.py
@@ -0,0 +1,96 @@
+"""Read and write configuration settings
+"""
+import os
+import subprocess
+
+from milc import cli
+
+
+def print_config(section, key):
+    """Print a single config setting to stdout.
+    """
+    cli.echo('%s.%s{fg_cyan}={fg_reset}%s', section, key, cli.config[section][key])
+
+
+@cli.argument('-ro', '--read-only', action='store_true', help='Operate in read-only mode.')
+@cli.argument('configs', nargs='*', arg_only=True, help='Configuration options to read or write.')
+@cli.subcommand("Read and write configuration settings.")
+def config(cli):
+    """Read and write config settings.
+
+    This script iterates over the config_tokens supplied as argument. Each config_token has the following form:
+
+        section[.key][=value]
+
+    If only a section (EG 'compile') is supplied all keys for that section will be displayed.
+
+    If section.key is supplied the value for that single key will be displayed.
+
+    If section.key=value is supplied the value for that single key will be set.
+
+    If section.key=None is supplied the key will be deleted.
+
+    No validation is done to ensure that the supplied section.key is actually used by qmk scripts.
+    """
+    if not cli.args.configs:
+        # Walk the config tree
+        for section in cli.config:
+            for key in cli.config[section]:
+                print_config(section, key)
+
+        return True
+
+    # Process config_tokens
+    save_config = False
+
+    for argument in cli.args.configs:
+        # Split on space in case they quoted multiple config tokens
+        for config_token in argument.split(' '):
+            # Extract the section, config_key, and value to write from the supplied config_token.
+            if '=' in config_token:
+                key, value = config_token.split('=')
+            else:
+                key = config_token
+                value = None
+
+            if '.' in key:
+                section, config_key = key.split('.', 1)
+            else:
+                section = key
+                config_key = None
+
+            # Validation
+            if config_key and '.' in config_key:
+                cli.log.error('Config keys may not have more than one period! "%s" is not valid.', key)
+                return False
+
+            # Do what the user wants
+            if section and config_key and value:
+                # Write a config key
+                log_string = '%s.%s{fg_cyan}:{fg_reset} %s {fg_cyan}->{fg_reset} %s'
+                if cli.args.read_only:
+                    log_string += ' {fg_red}(change not written)'
+
+                cli.echo(log_string, section, config_key, cli.config[section][config_key], value)
+
+                if not cli.args.read_only:
+                    if value == 'None':
+                        del cli.config[section][config_key]
+                    else:
+                        cli.config[section][config_key] = value
+                    save_config = True
+
+            elif section and config_key:
+                # Display a single key
+                print_config(section, config_key)
+
+            elif section:
+                # Display an entire section
+                for key in cli.config[section]:
+                    print_config(section, key)
+
+    # Ending actions
+    if save_config:
+        cli.save_config()
+
+    return True
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 5a713b20f5..3474422a89 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -11,8 +11,8 @@ from glob import glob
 from milc import cli
 
 
-@cli.entrypoint('Basic QMK environment checks')
-def main(cli):
+@cli.subcommand('Basic QMK environment checks')
+def doctor(cli):
     """Basic QMK environment checks.
 
     This is currently very simple, it just checks that all the expected binaries are on your system.
@@ -36,6 +36,7 @@ def main(cli):
         else:
             try:
                 subprocess.run([binary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5, check=True)
+                cli.log.info('Found {fg_cyan}%s', binary)
             except subprocess.CalledProcessError:
                 cli.log.error("{fg_red}Can't run `%s --version`", binary)
                 ok = False
diff --git a/lib/python/qmk/cli/hello.py b/lib/python/qmk/cli/hello.py
index bc0cb6de18..bee28c3013 100755
--- a/lib/python/qmk/cli/hello.py
+++ b/lib/python/qmk/cli/hello.py
@@ -6,8 +6,8 @@ from milc import cli
 
 
 @cli.argument('-n', '--name', default='World', help='Name to greet.')
-@cli.entrypoint('QMK Hello World.')
-def main(cli):
+@cli.subcommand('QMK Hello World.')
+def hello(cli):
     """Log a friendly greeting.
     """
-    cli.log.info('Hello, %s!', cli.config.general.name)
+    cli.log.info('Hello, %s!', cli.config.hello.name)
diff --git a/lib/python/qmk/cli/json/__init__.py b/lib/python/qmk/cli/json/__init__.py
index e69de29bb2..f4ebfc45b4 100644
--- a/lib/python/qmk/cli/json/__init__.py
+++ b/lib/python/qmk/cli/json/__init__.py
@@ -0,0 +1,5 @@
+"""QMK CLI JSON Subcommands
+
+We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
+"""
+from . import keymap
diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
index e2d0b58093..a65acd6197 100755
--- a/lib/python/qmk/cli/json/keymap.py
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -9,10 +9,10 @@ from milc import cli
 import qmk.keymap
 
 
-@cli.argument('-o', '--output', help='File to write to')
-@cli.argument('filename', help='Configurator JSON file')
-@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
-def main(cli):
+@cli.argument('-o', '--output', arg_only=True, help='File to write to')
+@cli.argument('filename', arg_only=True, help='Configurator JSON file')
+@cli.subcommand('Create a keymap.c from a QMK Configurator export.')
+def json_keymap(cli):
     """Generate a keymap.c from a configurator export.
 
     This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
@@ -28,8 +28,8 @@ def main(cli):
         exit(1)
 
     # Environment processing
-    if cli.config.general.output == ('-'):
-        cli.config.general.output = None
+    if cli.args.output == ('-'):
+        cli.args.output = None
 
     # Parse the configurator json
     with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
@@ -38,17 +38,17 @@ def main(cli):
     # Generate the keymap
     keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
 
-    if cli.config.general.output:
-        output_dir = os.path.dirname(cli.config.general.output)
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
 
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
 
-        output_file = qmk.path.normpath(cli.config.general.output)
+        output_file = qmk.path.normpath(cli.args.output)
         with open(output_file, 'w') as keymap_fd:
             keymap_fd.write(keymap_c)
 
-        cli.log.info('Wrote keymap to %s.', cli.config.general.output)
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
 
     else:
         print(keymap_c)
diff --git a/lib/python/qmk/cli/new/__init__.py b/lib/python/qmk/cli/new/__init__.py
index e69de29bb2..c6a26939b8 100644
--- a/lib/python/qmk/cli/new/__init__.py
+++ b/lib/python/qmk/cli/new/__init__.py
@@ -0,0 +1 @@
+from . import keymap
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index b378e5ab43..5efb81c93f 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -6,15 +6,15 @@ import shutil
 from milc import cli
 
 
-@cli.argument('-k', '--keyboard', help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
-@cli.argument('-u', '--username', help='Specify any name for the new keymap directory')
-@cli.entrypoint('Creates a new keymap for the keyboard of your choosing')
-def main(cli):
+@cli.argument('-kb', '--keyboard', help='Specify keyboard name. Example: 1upkeyboards/1up60hse')
+@cli.argument('-km', '--keymap', help='Specify the name for the new keymap directory')
+@cli.subcommand('Creates a new keymap for the keyboard of your choosing')
+def new_keymap(cli):
     """Creates a new keymap for the keyboard of your choosing.
     """
     # ask for user input if keyboard or username was not provided in the command line
-    keyboard = cli.config.general.keyboard if cli.config.general.keyboard else input("Keyboard Name: ")
-    username = cli.config.general.username if cli.config.general.username else input("Username: ")
+    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
+    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
 
     # generate keymap paths
     kb_path = os.path.join(os.getcwd(), "keyboards", keyboard)
@@ -36,6 +36,5 @@ def main(cli):
     shutil.copytree(keymap_path_default, keymap_path, symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s\n" +
-                 "Compile a firmware file with your new keymap by typing: \n" +
-                 "qmk compile -kb %s -km %s", username, keymap_path, keyboard, username)
+    cli.log.info("%s keymap directory created in: %s", username, keymap_path)
+    cli.log.info("Compile a firmware with your new keymap by typing: \n" + "qmk compile -kb %s -km %s", keyboard, username)
diff --git a/lib/python/qmk/cli/nose2.py b/lib/python/qmk/cli/nose2.py
deleted file mode 100644
index c6c9c67b30..0000000000
--- a/lib/python/qmk/cli/nose2.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""QMK Python Unit Tests
-
-QMK script to run unit and integration tests against our python code.
-"""
-from milc import cli
-
-
-@cli.entrypoint('QMK Python Unit Tests')
-def main(cli):
-    """Use nose2 to run unittests
-    """
-    try:
-        import nose2
-    except ImportError:
-        cli.log.error('Could not import nose2! Please install it with {fg_cyan}pip3 install nose2')
-        return False
-
-    nose2.discover()
diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
index b1f8c02b28..a53ba40c0a 100755
--- a/lib/python/qmk/cli/pyformat.py
+++ b/lib/python/qmk/cli/pyformat.py
@@ -5,12 +5,13 @@ from milc import cli
 import subprocess
 
 
-@cli.entrypoint("Format python code according to QMK's style.")
-def main(cli):
+@cli.subcommand("Format python code according to QMK's style.")
+def pyformat(cli):
     """Format python code according to QMK's style.
     """
     try:
         subprocess.run(['yapf', '-vv', '-ri', 'bin/qmk', 'lib/python'], check=True)
         cli.log.info('Successfully formatted the python code in `bin/qmk` and `lib/python`.')
+
     except subprocess.CalledProcessError:
         cli.log.error('Error formatting python code!')
diff --git a/lib/python/qmk/cli/pytest.py b/lib/python/qmk/cli/pytest.py
new file mode 100644
index 0000000000..14613e1d96
--- /dev/null
+++ b/lib/python/qmk/cli/pytest.py
@@ -0,0 +1,20 @@
+"""QMK Python Unit Tests
+
+QMK script to run unit and integration tests against our python code.
+"""
+import sys
+from milc import cli
+
+
+@cli.subcommand('QMK Python Unit Tests')
+def pytest(cli):
+    """Use nose2 to run unittests
+    """
+    try:
+        import nose2
+
+    except ImportError:
+        cli.log.error('Could not import nose2! Please install it with {fg_cyan}pip3 install nose2')
+        return False
+
+    nose2.discover(argv=['nose2', '-v'])
-- 
cgit v1.2.3


From 9067dc817aefb7fe43d23a995604555642c6c897 Mon Sep 17 00:00:00 2001
From: Ayman Bagabas <ayman.bagabas@gmail.com>
Date: Tue, 1 Oct 2019 20:56:16 -0400
Subject: Fix qmk doctor 'bytes-like object is required' on linux
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes the following issue related to encoding on linux systems. Add
`universal_newlines=True` to subprocess.

<class 'TypeError'>
☒ a bytes-like object is required, not 'str'
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/milc.py", line 564, in __call__
    return self.__call__()
  File "/usr/local/lib/python3.7/site-packages/milc.py", line 569, in __call__
    return self._entrypoint(self)
  File "$HOME/qmk_firmware/lib/python/qmk/cli/doctor.py", line 56, in doctor
    for line in mm_check.stdout.split('\n'):
TypeError: a bytes-like object is required, not 'str'
---
 lib/python/qmk/cli/doctor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 3474422a89..2b6a03e443 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -50,7 +50,7 @@ def doctor(cli):
     elif OS == "Linux":
         cli.log.info("Detected {fg_cyan}Linux.")
         if shutil.which('systemctl'):
-            mm_check = subprocess.run(['systemctl', 'list-unit-files'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10)
+            mm_check = subprocess.run(['systemctl', 'list-unit-files'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, universal_newlines=True)
             if mm_check.returncode == 0:
                 mm = True
                 for line in mm_check.stdout.split('\n'):
-- 
cgit v1.2.3


From 78f01eef2e4586f59b56703d0a4f1a57a1af40a1 Mon Sep 17 00:00:00 2001
From: "St. John Johnson" <st.john.johnson@gmail.com>
Date: Sat, 5 Oct 2019 23:41:15 -0700
Subject: Use `keymap` instead of `username` variable for `qmk new_keymap`
 (#6885)

Username is not defined and this causes `qmk new_keymap` to error.  This
appears to have originated from a partial update in
https://github.com/qmk/qmk_firmware/pull/6708/files#diff-d5208bcbc79aa428556a743b6ff41086.  This change completes the migration from `username` to `keymap`
---
 lib/python/qmk/cli/new/keymap.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 5efb81c93f..96525e28e1 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -12,14 +12,14 @@ from milc import cli
 def new_keymap(cli):
     """Creates a new keymap for the keyboard of your choosing.
     """
-    # ask for user input if keyboard or username was not provided in the command line
+    # ask for user input if keyboard or keymap was not provided in the command line
     keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ")
     keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ")
 
     # generate keymap paths
     kb_path = os.path.join(os.getcwd(), "keyboards", keyboard)
     keymap_path_default = os.path.join(kb_path, "keymaps/default")
-    keymap_path = os.path.join(kb_path, "keymaps/%s" % username)
+    keymap_path = os.path.join(kb_path, "keymaps/%s" % keymap)
 
     # check directories
     if not os.path.exists(kb_path):
@@ -36,5 +36,5 @@ def new_keymap(cli):
     shutil.copytree(keymap_path_default, keymap_path, symlinks=True)
 
     # end message to user
-    cli.log.info("%s keymap directory created in: %s", username, keymap_path)
-    cli.log.info("Compile a firmware with your new keymap by typing: \n" + "qmk compile -kb %s -km %s", keyboard, username)
+    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path)
+    cli.log.info("Compile a firmware with your new keymap by typing: \n" + "qmk compile -kb %s -km %s", keyboard, keymap)
-- 
cgit v1.2.3


From f04e58dad6f56cdbd5d369c9e00405dcdb47c8ea Mon Sep 17 00:00:00 2001
From: Dan McClain <dan@danmcclain.net>
Date: Mon, 7 Oct 2019 14:32:30 -0400
Subject: [CLI] Add `qmk list_keyboards` (#6927)

`list_keyboards` replicates the `make list-keyboards` by globbing for all paths
that include `rules.mk` and then removing the paths that include `keymaps`.

This basis of this cli command could be reused in the future as a util, but is
not done so here since this would be the only place that would use it currently

Resolves #6911
---
 lib/python/qmk/cli/__init__.py       |  1 +
 lib/python/qmk/cli/list/__init__.py  |  1 +
 lib/python/qmk/cli/list/keyboards.py | 26 ++++++++++++++++++++++++++
 3 files changed, 28 insertions(+)
 create mode 100644 lib/python/qmk/cli/list/__init__.py
 create mode 100644 lib/python/qmk/cli/list/keyboards.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index fb4e0ecb46..e982a75fc8 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -8,6 +8,7 @@ from . import config
 from . import doctor
 from . import hello
 from . import json
+from . import list
 from . import new
 from . import pyformat
 from . import pytest
diff --git a/lib/python/qmk/cli/list/__init__.py b/lib/python/qmk/cli/list/__init__.py
new file mode 100644
index 0000000000..c36ba69548
--- /dev/null
+++ b/lib/python/qmk/cli/list/__init__.py
@@ -0,0 +1 @@
+from . import keyboards
diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
new file mode 100644
index 0000000000..53a7af75c6
--- /dev/null
+++ b/lib/python/qmk/cli/list/keyboards.py
@@ -0,0 +1,26 @@
+"""List the keyboards currently defined within QMK
+"""
+import os
+import re
+import glob
+
+from milc import cli
+
+@cli.subcommand("List the keyboards currently defined within QMK")
+def list_keyboards(cli):
+    """List the keyboards currently defined within QMK
+    """
+
+    base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
+    kb_path_wildcard = os.path.join(base_path, "**", "rules.mk")
+
+    # find everywhere we have rules.mk where keymaps isn't in the path
+    paths = [path for path in glob.iglob(kb_path_wildcard, recursive=True) if 'keymaps' not in path]
+
+    # strip the keyboard directory path prefix and rules.mk suffix and alphabetize
+    find_name = lambda path: path.replace(base_path, "").replace(os.path.sep + "rules.mk", "")
+    names = sorted(map(find_name, paths))
+
+    for name in names:
+        # We echo instead of cli.log.info to allow easier piping of this output
+        cli.echo(name)
-- 
cgit v1.2.3


From 2707652c9877e5fc7dd67670f8e14962c31baf42 Mon Sep 17 00:00:00 2001
From: fauxpark <fauxpark@gmail.com>
Date: Wed, 9 Oct 2019 05:06:26 +1100
Subject: [Docs] CLI command to serve docs locally (#6956)

* CLI command to serve docs locally

* Document it

* Default port

* Use `with` and subclass `SimpleHTTPRequestHandler` to set working dir

* Apply suggestions from code review

Co-Authored-By: skullydazed <skullydazed@users.noreply.github.com>

* Update docs/cli.md
---
 lib/python/qmk/cli/__init__.py |  1 +
 lib/python/qmk/cli/docs.py     | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 lib/python/qmk/cli/docs.py

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index e982a75fc8..e41cc3dcb2 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -5,6 +5,7 @@ We list each subcommand here explicitly because all the reliable ways of searchi
 from . import cformat
 from . import compile
 from . import config
+from . import docs
 from . import doctor
 from . import hello
 from . import json
diff --git a/lib/python/qmk/cli/docs.py b/lib/python/qmk/cli/docs.py
new file mode 100644
index 0000000000..a0888ec388
--- /dev/null
+++ b/lib/python/qmk/cli/docs.py
@@ -0,0 +1,22 @@
+"""Serve QMK documentation locally
+"""
+import http.server
+
+from milc import cli
+
+
+class DocsHandler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory='docs', **kwargs)
+
+
+@cli.argument('-p', '--port', default=8936, type=int, help='Port number to use.')
+@cli.subcommand('Run a local webserver for QMK documentation.')
+def docs(cli):
+    """Spin up a local HTTPServer instance for the QMK docs.
+    """
+    with http.server.HTTPServer(('', cli.config.docs.port), DocsHandler) as httpd:
+        cli.log.info("Serving QMK docs at http://localhost:%d/", cli.config.docs.port)
+        cli.log.info("Press Control+C to exit.")
+
+        httpd.serve_forever()
-- 
cgit v1.2.3


From 076d8babbbd762f9a316a26144d966238b9b71cc Mon Sep 17 00:00:00 2001
From: fauxpark <fauxpark@gmail.com>
Date: Sat, 12 Oct 2019 15:41:58 +1100
Subject: [CLI] `qmk docs` graceful shutdown on Ctrl+C (#6989)

---
 lib/python/qmk/cli/docs.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/docs.py b/lib/python/qmk/cli/docs.py
index a0888ec388..b419891396 100644
--- a/lib/python/qmk/cli/docs.py
+++ b/lib/python/qmk/cli/docs.py
@@ -19,4 +19,9 @@ def docs(cli):
         cli.log.info("Serving QMK docs at http://localhost:%d/", cli.config.docs.port)
         cli.log.info("Press Control+C to exit.")
 
-        httpd.serve_forever()
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            cli.log.info("Stopping HTTP server...")
+        finally:
+            httpd.shutdown()
-- 
cgit v1.2.3


From f64d9b06215bb08d7f77aeba126c0804fffd0064 Mon Sep 17 00:00:00 2001
From: Harry Wada <harrywada@gmail.com>
Date: Sun, 20 Oct 2019 09:33:58 -0500
Subject: Fix detection of ModemManager (#7076)

---
 lib/python/qmk/cli/doctor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 2b6a03e443..309de0c671 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -52,10 +52,10 @@ def doctor(cli):
         if shutil.which('systemctl'):
             mm_check = subprocess.run(['systemctl', 'list-unit-files'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, universal_newlines=True)
             if mm_check.returncode == 0:
-                mm = True
+                mm = False
                 for line in mm_check.stdout.split('\n'):
                     if 'ModemManager' in line and 'enabled' in line:
-                        mm = False
+                        mm = True
 
                 if mm:
                     cli.log.warn("{bg_yellow}Detected ModemManager. Please disable it if you are using a Pro-Micro.")
-- 
cgit v1.2.3


From e0e26957d43018998c405783a2609b99f0e098a7 Mon Sep 17 00:00:00 2001
From: "St. John Johnson" <st.john.johnson@gmail.com>
Date: Mon, 28 Oct 2019 18:24:36 -0700
Subject: Fix the CLI docs (#6979)

- Sort the commands alphabetically
- Add missing `json_keymap`
- Correct underscore to dash
---
 lib/python/qmk/cli/json/keymap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
index a65acd6197..207ac278ca 100755
--- a/lib/python/qmk/cli/json/keymap.py
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -11,7 +11,7 @@ import qmk.keymap
 
 @cli.argument('-o', '--output', arg_only=True, help='File to write to')
 @cli.argument('filename', arg_only=True, help='Configurator JSON file')
-@cli.subcommand('Create a keymap.c from a QMK Configurator export.')
+@cli.subcommand('Creates a keymap.c from a QMK Configurator export.')
 def json_keymap(cli):
     """Generate a keymap.c from a configurator export.
 
-- 
cgit v1.2.3


From 5421ba11dedd9912967b1fa5ed1f664687221143 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Tue, 12 Nov 2019 17:17:12 -0800
Subject: Add support for newer versions of clang-format, if installed

---
 lib/python/qmk/cli/cformat.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index d2382bdbde..17ca91b3b5 100644
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -2,6 +2,7 @@
 """
 import os
 import subprocess
+from shutil import which
 
 from milc import cli
 
@@ -11,10 +12,18 @@ from milc import cli
 def cformat(cli):
     """Format C code according to QMK's style.
     """
+    # Determine which version of clang-format to use
     clang_format = ['clang-format', '-i']
+    for clang_version in [10, 9, 8, 7]:
+        binary = 'clang-format-%d' % clang_version
+        if which(binary):
+            clang_format[0] = binary
+            break
 
     # Find the list of files to format
-    if not cli.args.files:
+    if cli.args.files:
+        cli.args.files = [os.path.join(os.environ['ORIG_CWD'], file) for file in cli.args.files]
+    else:
         for dir in ['drivers', 'quantum', 'tests', 'tmk_core']:
             for dirpath, dirnames, filenames in os.walk(dir):
                 if 'tmk_core/protocol/usb_hid' in dirpath:
-- 
cgit v1.2.3


From d1b6c11b7f4cc24c50d2b640a94925acf6450da6 Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Tue, 12 Nov 2019 17:21:33 -0800
Subject: When checking program returncodes treat both 0 and 1 as installed

---
 lib/python/qmk/cli/doctor.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 309de0c671..c2723bfcbb 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -24,8 +24,7 @@ def doctor(cli):
     cli.log.info('QMK Doctor is checking your environment.')
 
     # Make sure the basic CLI tools we need are available and can be executed.
-    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
-    binaries += glob('bin/qmk-*')
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc', 'bin/qmk']
     ok = True
 
     for binary in binaries:
@@ -34,10 +33,10 @@ def doctor(cli):
             cli.log.error("{fg_red}QMK can't find %s in your path.", binary)
             ok = False
         else:
-            try:
-                subprocess.run([binary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5, check=True)
+            check = subprocess.run([binary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5)
+            if check.returncode in [0, 1]:
                 cli.log.info('Found {fg_cyan}%s', binary)
-            except subprocess.CalledProcessError:
+            else:
                 cli.log.error("{fg_red}Can't run `%s --version`", binary)
                 ok = False
 
-- 
cgit v1.2.3


From 79edb7c5942108774e667c172550a1519c5543ac Mon Sep 17 00:00:00 2001
From: skullY <skullydazed@gmail.com>
Date: Tue, 12 Nov 2019 17:27:08 -0800
Subject: Small CLI cleanups

* yapf changes
* Fix the cformat test
* Make the normpath test work when run from /
* `qmk config`: Mark `--read-only` as arg_only
---
 lib/python/qmk/cli/config.py         | 2 +-
 lib/python/qmk/cli/list/keyboards.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib/python/qmk/cli')

diff --git a/lib/python/qmk/cli/config.py b/lib/python/qmk/cli/config.py
index d6c774e651..c4ee20cba5 100644
--- a/lib/python/qmk/cli/config.py
+++ b/lib/python/qmk/cli/config.py
@@ -12,7 +12,7 @@ def print_config(section, key):
     cli.echo('%s.%s{fg_cyan}={fg_reset}%s', section, key, cli.config[section][key])
 
 
-@cli.argument('-ro', '--read-only', action='store_true', help='Operate in read-only mode.')
+@cli.argument('-ro', '--read-only', arg_only=True, action='store_true', help='Operate in read-only mode.')
 @cli.argument('configs', nargs='*', arg_only=True, help='Configuration options to read or write.')
 @cli.subcommand("Read and write configuration settings.")
 def config(cli):
diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
index