Extensible split data sync (#11930)

* Extensible split data sync capability through transactions.

- Split common transport has been split up between the transport layer
  and data layer.
- Split "transactions" model used, with convergence between I2C and
  serial data definitions.
- Slave matrix "generation count" is used to determine if the full slave
  matrix needs to be retrieved.
- Encoders get the same "generation count" treatment.
- All other blocks of data are synchronised when a change is detected.
- All transmissions have a globally-configurable deadline before a
  transmission is forced (`FORCED_SYNC_THROTTLE_MS`, default 100ms).
- Added atomicity for all core-synced data, preventing partial updates
- Added retries to AVR i2c_master's i2c_start, to minimise the number of
  failed transactions when interrupts are disabled on the slave due to
  atomicity checks.
- Some keyboards have had slight modifications made in order to ensure
  that they still build due to firmware size restrictions.

* Fixup LED_MATRIX compile.

* Parameterise ERROR_DISCONNECT_COUNT.

5 files changed, 81 insertions, 124 deletions

diff --git a/drivers/avr/i2c_master.c b/drivers/avr/i2c_master.c
index b1e4885298..2773e00778 100644
--- a/drivers/avr/i2c_master.c
+++ b/drivers/avr/i2c_master.c
@@ -28,8 +28,14 @@
 #    define F_SCL 400000UL  // SCL frequency
 #endif
 
+#ifndef I2C_START_RETRY_COUNT
+#    define I2C_START_RETRY_COUNT 20
+#endif  // I2C_START_RETRY_COUNT
+
 #define TWBR_val (((F_CPU / F_SCL) - 16) / 2)
 
+#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
+
 void i2c_init(void) {
     TWSR = 0; /* no prescaler */
     TWBR = (uint8_t)TWBR_val;
@@ -47,7 +53,7 @@ void i2c_init(void) {
 #endif
 }
 
-i2c_status_t i2c_start(uint8_t address, uint16_t timeout) {
+static i2c_status_t i2c_start_impl(uint8_t address, uint16_t timeout) {
     // reset TWI control register
     TWCR = 0;
     // transmit START condition
@@ -86,6 +92,17 @@ i2c_status_t i2c_start(uint8_t address, uint16_t timeout) {
     return I2C_STATUS_SUCCESS;
 }
 
+i2c_status_t i2c_start(uint8_t address, uint16_t timeout) {
+    // Retry i2c_start_impl a bunch times in case the remote side has interrupts disabled.
+    uint16_t     timeout_timer = timer_read();
+    uint16_t     time_slice    = MAX(1, (timeout == (I2C_TIMEOUT_INFINITE)) ? 5 : (timeout / (I2C_START_RETRY_COUNT)));  // if it's infinite, wait 1ms between attempts, otherwise split up the entire timeout into the number of retries
+    i2c_status_t status;
+    do {
+        status = i2c_start_impl(address, time_slice);
+    } while ((status < 0) && ((timeout == I2C_TIMEOUT_INFINITE) || (timer_elapsed(timeout_timer) < timeout)));
+    return status;
+}
+
 i2c_status_t i2c_write(uint8_t data, uint16_t timeout) {
     // load data into data register
     TWDR = data;
diff --git a/drivers/avr/i2c_slave.c b/drivers/avr/i2c_slave.c
index 62a378165a..2907f164c0 100644
--- a/drivers/avr/i2c_slave.c
+++ b/drivers/avr/i2c_slave.c
@@ -17,6 +17,7 @@
  * GitHub repository: https://github.com/g4lvanix/I2C-slave-lib
  */
 
+#include <stddef.h>
 #include <avr/io.h>
 #include <util/twi.h>
 #include <avr/interrupt.h>
@@ -24,6 +25,12 @@
 
 #include "i2c_slave.h"
 
+#if defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+#    include "transactions.h"
+
+static volatile bool is_callback_executor = false;
+#endif  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+
 volatile uint8_t i2c_slave_reg[I2C_SLAVE_REG_COUNT];
 
 static volatile uint8_t buffer_address;
@@ -48,11 +55,14 @@ ISR(TWI_vect) {
         case TW_SR_SLA_ACK:
             // The device is now a slave receiver
             slave_has_register_set = false;
+#if defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+            is_callback_executor = false;
+#endif  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
             break;
 
         case TW_SR_DATA_ACK:
             // This device is a slave receiver and has received data
-            // First byte is the location then the bytes will be writen in buffer with auto-incriment
+            // First byte is the location then the bytes will be writen in buffer with auto-increment
             if (!slave_has_register_set) {
                 buffer_address = TWDR;
 
@@ -60,10 +70,25 @@ ISR(TWI_vect) {
                     ack            = 0;
                     buffer_address = 0;
                 }
-                slave_has_register_set = true;  // address has been receaved now fill in buffer
+                slave_has_register_set = true;  // address has been received now fill in buffer
+
+#if defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+                // Work out if we're attempting to execute a callback
+                is_callback_executor = buffer_address == split_transaction_table[I2C_EXECUTE_CALLBACK].initiator2target_offset;
+#endif  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
             } else {
                 i2c_slave_reg[buffer_address] = TWDR;
                 buffer_address++;
+
+#if defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+                // If we're intending to execute a transaction callback, do so, as we've just received the transaction ID
+                if (is_callback_executor) {
+                    split_transaction_desc_t *trans = &split_transaction_table[split_shmem->transaction_id];
+                    if (trans->slave_callback) {
+                        trans->slave_callback(trans->initiator2target_buffer_size, split_trans_initiator2target_buffer(trans), trans->target2initiator_buffer_size, split_trans_target2initiator_buffer(trans));
+                    }
+                }
+#endif  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
             }
             break;
 
diff --git a/drivers/avr/i2c_slave.h b/drivers/avr/i2c_slave.h
index 1cd0625ef4..a8647c9da3 100644
--- a/drivers/avr/i2c_slave.h
+++ b/drivers/avr/i2c_slave.h
@@ -22,7 +22,18 @@
 
 #pragma once
 
-#define I2C_SLAVE_REG_COUNT 30
+#ifndef I2C_SLAVE_REG_COUNT
+
+#    if defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+#        include "transport.h"
+#        define I2C_SLAVE_REG_COUNT sizeof(split_shared_memory_t)
+#    else  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+#        define I2C_SLAVE_REG_COUNT 30
+#    endif  // defined(USE_I2C) && defined(SPLIT_COMMON_TRANSACTIONS)
+
+#endif  // I2C_SLAVE_REG_COUNT
+
+_Static_assert(I2C_SLAVE_REG_COUNT < 256, "I2C target registers must be single byte");
 
 extern volatile uint8_t i2c_slave_reg[I2C_SLAVE_REG_COUNT];
 
diff --git a/drivers/avr/serial.c b/drivers/avr/serial.c
index 3647bee0d3..9a7345a53d 100644
--- a/drivers/avr/serial.c
+++ b/drivers/avr/serial.c
@@ -224,15 +224,8 @@
 #    define SERIAL_DELAY_HALF2 (SERIAL_DELAY - SERIAL_DELAY / 2)
 
 #    define SLAVE_INT_WIDTH_US 1
-#    ifndef SERIAL_USE_MULTI_TRANSACTION
-#        define SLAVE_INT_RESPONSE_TIME SERIAL_DELAY
-#    else
-#        define SLAVE_INT_ACK_WIDTH_UNIT 2
-#        define SLAVE_INT_ACK_WIDTH 4
-#    endif
-
-static SSTD_t *Transaction_table      = NULL;
-static uint8_t Transaction_table_size = 0;
+#    define SLAVE_INT_ACK_WIDTH_UNIT 2
+#    define SLAVE_INT_ACK_WIDTH 4
 
 inline static void serial_delay(void) ALWAYS_INLINE;
 inline static void serial_delay(void) { _delay_us(SERIAL_DELAY); }
@@ -259,16 +252,12 @@ inline static void serial_low(void) { writePinLow(SOFT_SERIAL_PIN); }
 inline static void serial_high(void) ALWAYS_INLINE;
 inline static void serial_high(void) { writePinHigh(SOFT_SERIAL_PIN); }
 
-void soft_serial_initiator_init(SSTD_t *sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
+void soft_serial_initiator_init(void) {
     serial_output();
     serial_high();
 }
 
-void soft_serial_target_init(SSTD_t *sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
+void soft_serial_target_init(void) {
     serial_input_with_pullup();
 
     // Enable INT0-INT7
@@ -395,19 +384,14 @@ static inline uint8_t nibble_bits_count(uint8_t bits) {
 
 // interrupt handle to be used by the target device
 ISR(SERIAL_PIN_INTERRUPT) {
-#    ifndef SERIAL_USE_MULTI_TRANSACTION
-    serial_low();
-    serial_output();
-    SSTD_t *trans = Transaction_table;
-#    else
     // recive transaction table index
     uint8_t tid, bits;
     uint8_t pecount = 0;
     sync_recv();
-    bits = serial_read_chunk(&pecount, 7);
+    bits = serial_read_chunk(&pecount, 8);
     tid  = bits >> 3;
-    bits = (bits & 7) != nibble_bits_count(tid);
-    if (bits || pecount > 0 || tid > Transaction_table_size) {
+    bits = (bits & 7) != (nibble_bits_count(tid) & 7);
+    if (bits || pecount > 0 || tid > NUM_TOTAL_TRANSACTIONS) {
         return;
     }
     serial_delay_half1();
@@ -415,18 +399,22 @@ ISR(SERIAL_PIN_INTERRUPT) {
     serial_high();  // response step1 low->high
     serial_output();
     _delay_sub_us(SLAVE_INT_ACK_WIDTH_UNIT * SLAVE_INT_ACK_WIDTH);
-    SSTD_t *trans = &Transaction_table[tid];
+    split_transaction_desc_t *trans = &split_transaction_table[tid];
     serial_low();  // response step2 ack high->low
-#    endif
+
+    // If the transaction has a callback, we can execute it now
+    if (trans->slave_callback) {
+        trans->slave_callback(trans->initiator2target_buffer_size, split_trans_initiator2target_buffer(trans), trans->target2initiator_buffer_size, split_trans_target2initiator_buffer(trans));
+    }
 
     // target send phase
-    if (trans->target2initiator_buffer_size > 0) serial_send_packet((uint8_t *)trans->target2initiator_buffer, trans->target2initiator_buffer_size);
+    if (trans->target2initiator_buffer_size > 0) serial_send_packet((uint8_t *)split_trans_target2initiator_buffer(trans), trans->target2initiator_buffer_size);
     // target switch to input
     change_sender2reciver();
 
     // target recive phase
     if (trans->initiator2target_buffer_size > 0) {
-        if (serial_recive_packet((uint8_t *)trans->initiator2target_buffer, trans->initiator2target_buffer_size)) {
+        if (serial_recive_packet((uint8_t *)split_trans_initiator2target_buffer(trans), trans->initiator2target_buffer_size)) {
             *trans->status = TRANSACTION_ACCEPTED;
         } else {
             *trans->status = TRANSACTION_DATA_ERROR;
@@ -448,14 +436,12 @@ ISR(SERIAL_PIN_INTERRUPT) {
 //    TRANSACTION_NO_RESPONSE
 //    TRANSACTION_DATA_ERROR
 // this code is very time dependent, so we need to disable interrupts
-#    ifndef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_transaction(void) {
-    SSTD_t *trans = Transaction_table;
-#    else
 int soft_serial_transaction(int sstd_index) {
-    if (sstd_index > Transaction_table_size) return TRANSACTION_TYPE_ERROR;
-    SSTD_t *trans = &Transaction_table[sstd_index];
-#    endif
+    if (sstd_index > NUM_TOTAL_TRANSACTIONS) return TRANSACTION_TYPE_ERROR;
+    split_transaction_desc_t *trans = &split_transaction_table[sstd_index];
+
+    if (!trans->status) return TRANSACTION_TYPE_ERROR;  // not registered
+
     cli();
 
     // signal to the target that we want to start a transaction
@@ -463,27 +449,11 @@ int soft_serial_transaction(int sstd_index) {
     serial_low();
     _delay_us(SLAVE_INT_WIDTH_US);
 
-#    ifndef SERIAL_USE_MULTI_TRANSACTION
-    // wait for the target response
-    serial_input_with_pullup();
-    _delay_us(SLAVE_INT_RESPONSE_TIME);
-
-    // check if the target is present
-    if (serial_read_pin()) {
-        // target failed to pull the line low, assume not present
-        serial_output();
-        serial_high();
-        *trans->status = TRANSACTION_NO_RESPONSE;
-        sei();
-        return TRANSACTION_NO_RESPONSE;
-    }
-
-#    else
     // send transaction table index
     int tid = (sstd_index << 3) | (7 & nibble_bits_count(sstd_index));
     sync_send();
     _delay_sub_us(TID_SEND_ADJUST);
-    serial_write_chunk(tid, 7);
+    serial_write_chunk(tid, 8);
     serial_delay_half1();
 
     // wait for the target response (step1 low->high)
@@ -504,12 +474,11 @@ int soft_serial_transaction(int sstd_index) {
         }
         _delay_sub_us(SLAVE_INT_ACK_WIDTH_UNIT);
     }
-#    endif
 
     // initiator recive phase
     // if the target is present syncronize with it
     if (trans->target2initiator_buffer_size > 0) {
-        if (!serial_recive_packet((uint8_t *)trans->target2initiator_buffer, trans->target2initiator_buffer_size)) {
+        if (!serial_recive_packet((uint8_t *)split_trans_target2initiator_buffer(trans), trans->target2initiator_buffer_size)) {
             serial_output();
             serial_high();
             *trans->status = TRANSACTION_DATA_ERROR;
@@ -523,7 +492,7 @@ int soft_serial_transaction(int sstd_index) {
 
     // initiator send phase
     if (trans->initiator2target_buffer_size > 0) {
-        serial_send_packet((uint8_t *)trans->initiator2target_buffer, trans->initiator2target_buffer_size);
+        serial_send_packet((uint8_t *)split_trans_initiator2target_buffer(trans), trans->initiator2target_buffer_size);
     }
 
     // always, release the line when not in use
@@ -534,9 +503,8 @@ int soft_serial_transaction(int sstd_index) {
     return TRANSACTION_END;
 }
 
-#    ifdef SERIAL_USE_MULTI_TRANSACTION
 int soft_serial_get_and_clean_status(int sstd_index) {
-    SSTD_t *trans = &Transaction_table[sstd_index];
+    split_transaction_desc_t *trans = &split_transaction_table[sstd_index];
     cli();
     int retval     = *trans->status;
     *trans->status = 0;
@@ -544,8 +512,6 @@ int soft_serial_get_and_clean_status(int sstd_index) {
     sei();
     return retval;
 }
-#    endif
-
 #endif
 
 // Helix serial.c history
diff --git a/drivers/avr/serial.h b/drivers/avr/serial.h
deleted file mode 100644
index 53e66cf905..0000000000
--- a/drivers/avr/serial.h
+++ /dev/null
@@ -1,62 +0,0 @@
-#pragma once
-
-#include <stdbool.h>
-
-// /////////////////////////////////////////////////////////////////
-// Need Soft Serial defines in config.h
-// /////////////////////////////////////////////////////////////////
-// ex.
-//  #define SOFT_SERIAL_PIN ??   // ?? = D0,D1,D2,D3,E6
-//  OPTIONAL: #define SELECT_SOFT_SERIAL_SPEED ? // ? = 1,2,3,4,5
-//                                               //  1: about 137kbps (default)
-//                                               //  2: about 75kbps
-//                                               //  3: about 39kbps
-//                                               //  4: about 26kbps
-//                                               //  5: about 20kbps
-//
-// //// USE simple API (using signle-type transaction function)
-//   /* nothing */
-// //// USE flexible API (using multi-type transaction function)
-//   #define SERIAL_USE_MULTI_TRANSACTION
-//
-// /////////////////////////////////////////////////////////////////
-
-// Soft Serial Transaction Descriptor
-typedef struct _SSTD_t {
-    uint8_t *status;
-    uint8_t  initiator2target_buffer_size;
-    uint8_t *initiator2target_buffer;
-    uint8_t  target2initiator_buffer_size;
-    uint8_t *target2initiator_buffer;
-} SSTD_t;
-#define TID_LIMIT(table) (sizeof(table) / sizeof(SSTD_t))
-
-// initiator is transaction start side
-void soft_serial_initiator_init(SSTD_t *sstd_table, int sstd_table_size);
-// target is interrupt accept side
-void soft_serial_target_init(SSTD_t *sstd_table, int sstd_table_size);
-
-// initiator resullt
-#define TRANSACTION_END 0
-#define TRANSACTION_NO_RESPONSE 0x1
-#define TRANSACTION_DATA_ERROR 0x2
-#define TRANSACTION_TYPE_ERROR 0x4
-#ifndef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_transaction(void);
-#else
-int soft_serial_transaction(int sstd_index);
-#endif
-
-// target status
-// *SSTD_t.status has
-//   initiator:
-//       TRANSACTION_END
-//    or TRANSACTION_NO_RESPONSE
-//    or TRANSACTION_DATA_ERROR
-//   target:
-//       TRANSACTION_DATA_ERROR
-//    or TRANSACTION_ACCEPTED
-#define TRANSACTION_ACCEPTED 0x8
-#ifdef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_get_and_clean_status(int sstd_index);
-#endif