Merge pull request #4 from jackhumbert/master

Merge from jackhumbert

3 files changed, 319 insertions, 186 deletions

diff --git a/keyboards/ergodox/keymaps/algernon/tools/heatmap-adore-layout.json b/keyboards/ergodox/keymaps/algernon/tools/heatmap-layout.ADORE.json
index 544f61ce87..4dfa877a78 100644
--- a/keyboards/ergodox/keymaps/algernon/tools/heatmap-adore-layout.json
+++ b/keyboards/ergodox/keymaps/algernon/tools/heatmap-layout.ADORE.json
@@ -75,9 +75,9 @@
     "STENO",
     {
       "x": 4.5,
-      "f": 6
+      "f": 3
     },
-    "<i class='mss mss-Unicode-Option-3'></i>",
+    "F12",
     {
       "c": "#7adabd",
       "a": 4,
@@ -94,12 +94,20 @@
     {
       "y": -0.875,
       "c": "#ffb2d2",
-      "f": 3,
+      "f": 9,
+      "a": 6,
       "w": 1.5
     },
-    "\n\n~\n`",
+    "\n\n<i class='kb kb-Multimedia-Play-Pause'></i>",
     {
-      "t": "#0d0d0b"
+      "t": "#0d0d0b",
+      "f": 3,
+      "a": 4,
+      "fa": [
+          0,
+          0,
+          2
+      ]
     },
     "!\n1\nF1",
     {
@@ -120,7 +128,7 @@
       "t": "#000000",
       "a": 6
     },
-    "L",
+    "G",
     {
       "x": 10.5
     },
@@ -132,16 +140,16 @@
       "x": 2.5,
       "c": "#bfbad1",
       "t": "#0d0d0b",
-      "a": 4
+      "a": 6
     },
-    ">\n.",
+    "W",
     {
       "x": 1,
       "c": "#7adabd",
       "t": "#000000",
       "a": 6
     },
-    "W",
+    "L",
     {
       "x": 8.5
     },
@@ -164,14 +172,15 @@
     {
       "c": "#93c9b7",
       "a": 4,
+      "fa": [0, 0, 0],
       "h": 1.5
     },
-    "{\n[",
+    "{\n(\n[",
     {
       "x": 4.5,
       "h": 1.5
     },
-    "}\n]",
+    "}\n)\n]",
     {
       "c": "#7adabd",
       "a": 6
@@ -183,21 +192,22 @@
       "y": -0.875,
       "c": "#ffb07b",
       "t": "#0d0d0b",
-      "f": 6,
+      "f": 3,
+      "a": 4,
       "w": 1.5
     },
-    "<i class='fa fa-fast-backward'></i>\n\n<i class='fa fa-fast-forward'></i>",
+    "\n\n~\n`",
     {
       "c": "#ffb2d2",
-      "a": 4,
+      "a": 6,
       "f": 3
     },
-    "<\n,",
+    "Y",
     {
       "x": 14.5,
       "a": 6
     },
-    "Y",
+    "X",
     {
       "a": 4,
       "w": 1.5
@@ -300,15 +310,14 @@
       "x": 6.5,
       "c": "#93c9b7",
       "t": "#000000",
-      "a": 7,
       "h": 1.5
     },
-    "(",
+    ";\n:",
     {
       "x": 4.5,
       "h": 1.5
     },
-    ")"
+    "_\n-"
   ],
   [
     {
@@ -320,7 +329,9 @@
     },
     "\"\n'",
     {
-      "x": 10.5
+      "x": 10.5,
+      "a": 6,
+      "f": 3
     },
     "V"
   ],
@@ -329,19 +340,22 @@
       "y": -0.875,
       "x": 2.5,
       "c": "#bfbad1",
-      "t": "#0d0d0b"
+      "t": "#0d0d0b",
+      "a": 6
     },
-    "Z",
+    "Q",
     {
       "x": 1,
       "c": "#7adabd",
-      "t": "#000000"
+      "t": "#000000",
+      "a": 4
     },
-    "K",
+    "<\n,",
     {
-      "x": 8.5
+      "x": 8.5,
+      "a": 6
     },
-    "G",
+    "K",
     {
       "x": 1,
       "c": "#bfbad1",
@@ -354,11 +368,13 @@
       "y": -0.875,
       "x": 5.5,
       "c": "#7adabd",
-      "t": "#000000"
+      "t": "#000000",
+      "a": 4
     },
-    "X",
+    ">\n.",
     {
-      "x": 6.5
+      "x": 6.5,
+      "a": 6
     },
     "B"
   ],
@@ -367,28 +383,32 @@
       "y": -0.875,
       "c": "#ffb07b",
       "f": 9,
-      "w": 1.5
+      "w": 1.5,
+      "g": true
     },
-    "\n\n<i class='kb kb-Multimedia-Play-Pause'></i>",
+    "",
     {
       "c": "#ffb2d2",
       "t": "#0d0d0b",
-      "a": 4,
-      "f": 3
+      "a": 6,
+      "f": 3,
+      "g": false
     },
-    "?\n/",
+    "Z",
     {
       "x": 14.5,
-      "a": 6
+      "a": 4
     },
-    "Q",
+    "?\n/",
     {
       "c": "#ffb07b",
       "t": "#000000",
       "f": 9,
-      "w": 1.5
+      "g": true,
+      "w": 1.5,
+      "a": 4
     },
-    "<i class='kb kb-Multimedia-Stop'></i>"
+    ""
   ],
   [
     {
@@ -414,14 +434,14 @@
     {
       "x": 1,
       "c": "#d4872a",
-      "g": false,
+      "g": true,
       "a": 5
     },
-    ";\n:",
+    "",
     {
       "x": 8.5
     },
-    "_\n-",
+    "",
     {
       "x": 1,
       "c": "#d9dae0",
@@ -503,13 +523,13 @@
     },
     "MEDIA",
     {},
-    "1HAND"
+    "DEL"
   ],
   [
     {
       "x": -3
     },
-    "LEAD",
+    "HUN",
     {
       "c": "#d4872a",
       "f": 9,
@@ -528,6 +548,6 @@
       "c": "#f9cd31",
       "f": 2
     },
-    "HUN"
+    "LEAD"
   ]
 ]
diff --git a/keyboards/ergodox/keymaps/algernon/tools/heatmap-base-layout.json b/keyboards/ergodox/keymaps/algernon/tools/heatmap-layout.Dvorak.json
index 148bb1f23b..1248d36af7 100644
--- a/keyboards/ergodox/keymaps/algernon/tools/heatmap-base-layout.json
+++ b/keyboards/ergodox/keymaps/algernon/tools/heatmap-layout.Dvorak.json
@@ -75,9 +75,9 @@
     "STENO",
     {
       "x": 4.5,
-      "f": 6
+      "f": 3
     },
-    "<i class='mss mss-Unicode-Option-3'></i>",
+    "F12",
     {
       "c": "#7adabd",
       "a": 4,
@@ -94,12 +94,21 @@
     {
       "y": -0.875,
       "c": "#ffb2d2",
-      "f": 3,
+      "f": 6,
+      "a": 6,
       "w": 1.5
     },
-    "\n\n~\n`",
+    "<i class='fa fa-fast-backward'></i>\n\n<i class='fa fa-fast-forward'></i>",
     {
-      "t": "#0d0d0b"
+      "f": 3,
+      "t": "#0d0d0b",
+      "a": 4,
+      "fa": [
+          0,
+          0,
+          2
+      ]
+
     },
     "!\n1\nF1",
     {
@@ -167,12 +176,12 @@
       "a": 4,
       "h": 1.5
     },
-    "{\n[",
+    "{\n(\n[",
     {
       "x": 4.5,
       "h": 1.5
     },
-    "}\n]",
+    "}\n)\n]",
     {
       "c": "#7adabd",
       "a": 6
@@ -184,10 +193,10 @@
       "y": -0.875,
       "c": "#ffb07b",
       "t": "#0d0d0b",
-      "f": 6,
+      "f": 3,
       "w": 1.5
     },
-    "<i class='fa fa-fast-backward'></i>\n\n<i class='fa fa-fast-forward'></i>",
+    "\n\n~\n`",
     {
       "c": "#ffb2d2",
       "a": 4,
@@ -503,7 +512,7 @@
     },
     "MEDIA",
     {},
-    "1HAND"
+    "DEL"
   ],
   [
     {
diff --git a/keyboards/ergodox/keymaps/algernon/tools/log-to-heatmap.py b/keyboards/ergodox/keymaps/algernon/tools/log-to-heatmap.py
index a13d12e81a..68585843be 100755
--- a/keyboards/ergodox/keymaps/algernon/tools/log-to-heatmap.py
+++ b/keyboards/ergodox/keymaps/algernon/tools/log-to-heatmap.py
@@ -3,143 +3,247 @@ import json
 import os
 import sys
 import re
+import argparse
 
 from math import floor
+from os.path import dirname
+
+class Heatmap(object):
+    coords = [
+        [
+            # Row 0
+            [ 4,  0], [ 4,  2], [ 2,  0], [ 1,  0], [ 2,  2], [ 3,  0], [ 3,  2],
+            [ 3,  4], [ 3,  6], [ 2,  4], [ 1,  2], [ 2,  6], [ 4,  4], [ 4,  6],
+        ],
+        [
+            # Row 1
+            [ 8,  0], [ 8,  2], [ 6,  0], [ 5,  0], [ 6,  2], [ 7,  0], [ 7,  2],
+            [ 7,  4], [ 7,  6], [ 6,  4], [ 5,  2], [ 6,  6], [ 8,  4], [ 8,  6],
+        ],
+        [
+            # Row 2
+            [12,  0], [12,  2], [10,  0], [ 9,  0], [10,  2], [11, 0], [     ],
+            [      ], [11,  2], [10,  4], [ 9,  2], [10,  6], [12, 4], [12, 6],
+        ],
+        [
+            # Row 3
+            [17,  0], [17,  2], [15,  0], [14,  0], [15,  2], [16,  0], [13,  0],
+            [13,  2], [16,  2], [15,  4], [14,  2], [15,  6], [17,  4], [17,  6],
+        ],
+        [
+            # Row 4
+            [20,  0], [20,  2], [19,  0], [18,  0], [19,  2], [], [], [], [],
+            [19,  4], [18,  2], [19,  6], [20,  4], [20,  6],
+        ],
+        [
+            # Row 5
+            [     ], [23,  0], [22,  2], [22,  0], [22,  4], [21,  0], [21,  2],
+            [24, 0], [24,  2], [25,  0], [25,  4], [25,  2], [26,  0], [      ],
+        ],
+    ]
+
+    def set_attr_at(self, block, n, attr, fn, val):
+        blk = self.heatmap[block][n]
+        if attr in blk:
+            blk[attr] = fn(blk[attr], val)
+        else:
+            blk[attr] = fn(None, val)
+
+    def coord(self, col, row):
+        return self.coords[row][col]
+
+    @staticmethod
+    def set_attr(orig, new):
+        return new
+
+    def set_bg(self, (block, n), color):
+        self.set_attr_at(block, n, "c", self.set_attr, color)
+        #self.set_attr_at(block, n, "g", self.set_attr, False)
+
+    def set_tap_info(self, (block, n), count, cap):
+        def _set_tap_info(o, _count, _cap):
+            ns = 4 - o.count ("\n")
+            return o + "\n" * ns + "%.02f%%" % (float(_count) / float(_cap) * 100)
+
+        if not cap:
+            cap = 1
+        self.heatmap[block][n + 1] = _set_tap_info (self.heatmap[block][n + 1], count, cap)
+
+    @staticmethod
+    def heatmap_color (v):
+        colors = [ [0.3, 0.3, 1], [0.3, 1, 0.3], [1, 1, 0.3], [1, 0.3, 0.3]]
+        fb = 0
+        if v <= 0:
+            idx1, idx2 = 0, 0
+        elif v >= 1:
+            idx1, idx2 = len(colors) - 1, len(colors) - 1
+        else:
+            val = v * (len(colors) - 1)
+            idx1 = int(floor(val))
+            idx2 = idx1 + 1
+            fb = val - float(idx1)
+
+        r = (colors[idx2][0] - colors[idx1][0]) * fb + colors[idx1][0]
+        g = (colors[idx2][1] - colors[idx1][1]) * fb + colors[idx1][1]
+        b = (colors[idx2][2] - colors[idx1][2]) * fb + colors[idx1][2]
+
+        r, g, b = [x * 255 for x in r, g, b]
+        return "#%02x%02x%02x" % (r, g, b)
+
+    def __init__(self, layout):
+        self.log = {}
+        self.total = 0
+        self.max_cnt = 0
+        self.layout = layout
+
+    def update_log(self, (c, r)):
+        if not (c, r) in self.log:
+            self.log[(c, r)] = 0
+        self.log[(c, r)] = self.log[(c, r)] + 1
+        self.total = self.total + 1
+        if self.max_cnt < self.log[(c, r)]:
+            self.max_cnt = self.log[(c, r)]
+
+    def get_heatmap(self):
+        with open("%s/heatmap-layout.%s.json" % (dirname(sys.argv[0]), self.layout), "r") as f:
+            self.heatmap = json.load (f)
+
+        ## Reset colors
+        for row in self.coords:
+            for coord in row:
+                if coord != []:
+                    self.set_bg (coord, "#d9dae0")
+
+        for (c, r) in self.log:
+            coords = self.coord(c, r)
+            b, n = coords
+            cap = self.max_cnt
+            if cap == 0:
+                cap = 1
+            v = float(self.log[(c, r)]) / cap
+            self.set_bg (coords, self.heatmap_color (v))
+            self.set_tap_info (coords, self.log[(c, r)], self.total)
+        return self.heatmap
+
+    def get_stats(self):
+        usage = [
+            # left hand
+            [0, 0, 0, 0, 0],
+            # right hand
+            [0, 0, 0, 0, 0]
+        ]
+        finger_map = [0, 0, 1, 2, 3, 4, 4]
+        for (c, r) in self.log:
+            if r == 5: # thumb cluster
+                if c <= 6: # left side
+                    usage[0][4] = usage[0][4] + self.log[(c, r)]
+                else:
+                    usage[1][4] = usage[1][4] + self.log[(c, r)]
+            else:
+                fc = c
+                hand = 0
+                if fc >= 7:
+                    fc = fc - 7
+                    hand = 1
+                fm = finger_map[fc]
+                usage[hand][fm] = usage[hand][fm] + self.log[(c, r)]
+        hand_usage = [0, 0]
+        for f in usage[0]:
+            hand_usage[0] = hand_usage[0] + f
+        for f in usage[1]:
+            hand_usage[1] = hand_usage[1] + f
+
+        total = self.total
+        if total == 0:
+            total = 1
+        stats = {
+            "hands": {
+                "left": {
+                    "usage": float(hand_usage[0]) / total * 100,
+                    "fingers": {
+                        "0 - pinky": 0,
+                        "1 - ring": 0,
+                        "2 - middle": 0,
+                        "3 - index": 0,
+                        "4 - thumb": 0,
+                    }
+                },
+                "right": {
+                    "usage": float(hand_usage[1]) / total * 100,
+                    "fingers": {
+                        "0 - thumb": 0,
+                        "1 - index": 0,
+                        "2 - middle": 0,
+                        "3 - ring": 0,
+                        "4 - pinky": 0,
+                    }
+                },
+            }
+        }
+
+        hmap = ['left', 'right']
+        fmap = ['0 - pinky', '1 - ring', '2 - middle', '3 - index', '4 - thumb',
+                '0 - thumb', '1 - index', '2 - middle', '3 - ring', '4 - pinky']
+        for hand_idx in range(len(usage)):
+            hand = usage[hand_idx]
+            for finger_idx in range(len(hand)):
+                stats['hands'][hmap[hand_idx]]['fingers'][fmap[finger_idx + hand_idx * 5]] = float(hand[finger_idx]) / total * 100
+        return stats
+
+def dump_all(out_dir, heatmaps):
+    for layer in heatmaps.keys():
+        if len(heatmaps[layer].log) == 0:
+            continue
 
-cr_coord_map = [
-    [
-        # Row 0
-        [ 4,  0], [ 4,  2], [ 2,  0], [ 1,  0], [ 2,  2], [ 3,  0], [ 3,  2],
-        [ 3,  4], [ 3,  6], [ 2,  4], [ 1,  2], [ 2,  6], [ 4,  4], [ 4,  6],
-    ],
-    [
-        # Row 1
-        [ 8,  0], [ 8,  2], [ 6,  0], [ 5,  0], [ 6,  2], [ 7,  0], [ 7,  2],
-        [ 7,  4], [ 7,  6], [ 6,  4], [ 5,  2], [ 6,  6], [ 8,  4], [ 8,  6],
-    ],
-    [
-        # Row 2
-        [12,  0], [12,  2], [10,  0], [ 9,  0], [10,  2], [11, 0], [     ],
-        [      ], [11,  2], [10,  4], [ 9,  2], [10,  6], [12, 4], [12, 6],
-    ],
-    [
-        # Row 3
-        [17,  0], [17,  2], [15,  0], [14,  0], [15,  2], [16,  0], [13,  0],
-        [13,  2], [16,  2], [15,  4], [14,  2], [15,  6], [17,  4], [17,  6],
-    ],
-    [
-        # Row 4
-        [20,  0], [20,  2], [19,  0], [18,  0], [19,  2], [], [], [], [],
-        [19,  4], [18,  2], [19,  6], [20,  4], [20,  6],
-    ],
-    [
-        # Row 5
-        [     ], [23,  0], [22,  2], [22,  0], [22,  4], [21,  0], [21,  2],
-        [24, 0], [24,  2], [25,  0], [25,  4], [25,  2], [26,  0], [      ],
-    ],
-]
-
-def set_attr_at(j, b, n, attr, fn, val):
-    blk = j[b][n]
-    if attr in blk:
-        blk[attr] = fn(blk[attr], val)
-    else:
-        blk[attr] = fn(None, val)
-
-def coord(col, row):
-    return cr_coord_map[row][col]
-
-def set_attr(orig, new):
-    return new
-
-def set_bg(j, (b, n), color):
-    set_attr_at(j, b, n, "c", set_attr, color)
-    #set_attr_at(j, b, n, "g", set_attr, False)
-
-def _set_tap_info(o, count, cap):
-    ns = 4 - o.count ("\n")
-    return o + "\n" * ns + "%.02f%%" % (float(count) / float(cap) * 100)
-
-def set_tap_info(j, (b, n), count, cap):
-    j[b][n + 1] = _set_tap_info (j[b][n + 1], count, cap)
-
-def heatmap_color (v):
-    colors = [ [0.3, 0.3, 1], [0.3, 1, 0.3], [1, 1, 0.3], [1, 0.3, 0.3]]
-    fb = 0
-    if v <= 0:
-        idx1, idx2 = 0, 0
-    elif v >= 1:
-        idx1, idx2 = len(colors) - 1, len(colors) - 1
-    else:
-        val = v * (len(colors) - 1)
-        idx1 = int(floor(val))
-        idx2 = idx1 + 1
-        fb = val - float(idx1)
-
-    r = (colors[idx2][0] - colors[idx1][0]) * fb + colors[idx1][0]
-    g = (colors[idx2][1] - colors[idx1][1]) * fb + colors[idx1][1]
-    b = (colors[idx2][2] - colors[idx1][2]) * fb + colors[idx1][2]
-
-    r, g, b = [x * 255 for x in r, g, b]
-    return "#%02x%02x%02x" % (r, g, b)
-
-# Load the keylog
-def load_keylog(fname, restrict_row):
-    keylog = {}
-    total = 0
-    with open(fname, "r") as f:
-        lines = f.readlines()
-    for line in lines:
-        m = re.search ('KL: col=(\d+), row=(\d+)', line)
+        with open ("%s/%s.json" % (out_dir, layer), "w") as f:
+            json.dump(heatmaps[layer].get_heatmap(), f)
+        print >>sys.stderr, "%s stats:" % (layer)
+        json.dump (heatmaps[layer].get_stats(), sys.stderr,
+                   indent = 4, sort_keys = True)
+        print >>sys.stderr, ""
+        print >>sys.stderr, ""
+
+def main(opts):
+
+    heatmaps = {"Dvorak": Heatmap("Dvorak"),
+                "ADORE": Heatmap("ADORE")
+    }
+    cnt = 0
+    restrict_row = opts.restrict_row
+    out_dir = opts.outdir
+
+    while True:
+        line = sys.stdin.readline()
+        if not line:
+            break
+        m = re.search ('KL: col=(\d+), row=(\d+), pressed=(\d+), layer=(.*)', line)
         if not m:
             continue
-        (c, r) = (int(m.group (2)), int(m.group (1)))
-        if restrict_row != None and r != int(restrict_row):
+
+        cnt = cnt + 1
+        (c, r, l) = (int(m.group (2)), int(m.group (1)), m.group (4))
+        if restrict_row != -1 and r != restrict_row:
+            continue
+        if c in opts.ignore_columns:
             continue
-        if (c, r) in keylog:
-            keylog[(c, r)] = keylog[(c, r)] + 1
-        else:
-            keylog[(c, r)] = 1
-        total = total + 1
-    return total / 2, keylog
-
-def l_flat(s):
-    f = s.split("\n")
-    return ", ".join (f)
-
-def main(base_fn, log_fn, restrict_row = None):
-
-    with open(base_fn, "r") as f:
-        layout = json.load (f)
-
-    ## Reset colors
-    for row in cr_coord_map:
-        for col in row:
-            if col != []:
-                set_bg (layout, col, "#d9dae0")
-                #set_attr_at (layout, col[0], col[1], "g", set_attr, True)
-
-    total, log = load_keylog (log_fn, restrict_row)
-    max_cnt = 0
-    for (c, r) in log:
-        max_cnt = max(max_cnt, log[(c, r)])
-
-    # Create the heatmap
-    for (c, r) in log:
-        coords = coord(c, r)
-        b, n = coords
-        cap = max_cnt
-        v = float(log[(c, r)]) / cap
-        print >> sys.stderr, "%s => %d/%d => %f = %s" % (l_flat(layout[b][n+1]), log[(c,r)], cap, v, heatmap_color(v))
-        set_bg (layout, coord(c, r), heatmap_color (v))
-        set_tap_info (layout, coord (c, r), log[(c, r)], total)
-
-    print json.dumps(layout)
 
-if __name__ == "__main__":
-    if len(sys.argv) < 3:
-        print """Log to Heatmap -- creates a heatmap out of keyboard logs
+        heatmaps[l].update_log ((c, r))
+
+        if opts.dump_interval != -1 and cnt >= opts.dump_interval:
+            cnt = 0
+            dump_all(out_dir, heatmaps)
 
-Usage: log-to-heatmap.py base-layout.json logfile [row] >layout.json"""
-        sys.exit (1)
-    main(*sys.argv[1:])
+    dump_all (out_dir, heatmaps)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser (description = "keylog to heatmap processor")
+    parser.add_argument ('outdir', action = 'store',
+                         help = 'Output directory')
+    parser.add_argument ('--row', dest = 'restrict_row', action = 'store', type = int,
+                         default = -1, help = 'Restrict processing to this row only')
+    parser.add_argument ('--dump-interval', dest = 'dump_interval', action = 'store', type = int,
+                         default = 100, help = 'Dump stats and heatmap at every Nth event, -1 for dumping at EOF only')
+    parser.add_argument ('--ignore-column', dest = 'ignore_columns', action = 'append', type = int,
+                         default = [], help = 'Ignore the specified columns')
+    args = parser.parse_args()
+    main(args)