stats: Add stat_item for value monitoring

This commit adds instrumentation function to gather measurement
and statistical values similar to counter groups.

Multiple values can be stored per item, which can be retrieved in
FIFO order. Getting values from the item does not modify its state to
allow for multiple independant backends (e.g. VTY and statd).

When a new value is set, the oldest value gets silently overwritten.
Lost values are skipped when getting values from the item.

Sponsored-by: On-Waves ehf

9 files changed, 531 insertions, 3 deletions

diff --git a/.gitignore b/.gitignore
index 24c3af7a..598f88af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,7 @@ tests/testsuite.dir/
 tests/testsuite.log
 
 tests/utils/utils_test
+tests/stats/stats_test
 tests/kasumi/kasumi_test
 tests/sms/sms_test
 tests/timer/timer_test
diff --git a/include/Makefile.am b/include/Makefile.am
index 52c6a38f..7f8dd777 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -28,6 +28,7 @@ nobase_include_HEADERS = \
                        osmocom/core/prim.h \
                        osmocom/core/process.h \
                        osmocom/core/rate_ctr.h \
+                       osmocom/core/stat_item.h \
                        osmocom/core/select.h \
                        osmocom/core/signal.h \
                        osmocom/core/socket.h \
diff --git a/include/osmocom/core/stat_item.h b/include/osmocom/core/stat_item.h
new file mode 100644
index 00000000..e166579f
--- /dev/null
+++ b/include/osmocom/core/stat_item.h
@@ -0,0 +1,104 @@
+#pragma once
+
+/*! \defgroup stat_item Statistics value item
+ *  @{
+ */
+
+/*! \file stat_item.h */
+
+#include <stdint.h>
+
+#include <osmocom/core/linuxlist.h>
+
+struct stat_item_desc;
+
+/*! \brief data we keep for each actual value */
+struct stat_item {
+	const struct stat_item_desc *desc;
+	/*! \brief the index of the freshest value */
+	int32_t last_value_index;
+	/*! \brief offset to the freshest value in the value fifo */
+	int16_t last_offs;
+	/*! \brief value fifo */
+	int32_t values[0];
+};
+
+/*! \brief statistics value description */
+struct stat_item_desc {
+	const char *name;	/*!< \brief name of the item */
+	const char *description;/*!< \brief description of the item */
+	const char *unit;	/*!< \brief unit of a value */
+	unsigned int num_values;/*!< \brief number of values to store */
+	int32_t default_value;
+};
+
+/*! \brief description of a statistics value group */
+struct stat_item_group_desc {
+	/*! \brief The prefix to the name of all values in this group */
+	const char *group_name_prefix;
+	/*! \brief The human-readable description of the group */
+	const char *group_description;
+	/*! \brief The number of values in this group */
+	const unsigned int num_items;
+	/*! \brief Pointer to array of value names */
+	const struct stat_item_desc *item_desc;
+};
+
+/*! \brief One instance of a counter group class */
+struct stat_item_group {
+	/*! \brief Linked list of all value groups in the system */
+	struct llist_head list;
+	/*! \brief Pointer to the counter group class */
+	const struct stat_item_group_desc *desc;
+	/*! \brief The index of this value group within its class */
+	unsigned int idx;
+	/*! \brief Actual counter structures below */
+	struct stat_item *items[0];
+};
+
+struct stat_item_group *stat_item_group_alloc(
+	void *ctx,
+	const struct stat_item_group_desc *desc,
+	unsigned int idx);
+
+void stat_item_group_free(struct stat_item_group *grp);
+
+void stat_item_set(struct stat_item *val, int32_t value);
+
+int stat_item_init(void *tall_ctx);
+
+struct stat_item_group *stat_item_get_group_by_name_idx(
+	const char *name, const unsigned int idx);
+
+const struct stat_item *stat_item_get_by_name(
+	const struct stat_item_group *valg, const char *name);
+
+/*! \brief Retrieve the next value from the stat_item object.
+ * If a new value has been set, it is returned. The idx is used to decide
+ * which value to return.
+ * On success, *idx is updated to refer to the next unread value. If
+ * values have been missed due to FIFO overflow, *idx is incremented by
+ * (1 + num_lost).
+ * This way, the stat_item object can be kept stateless from the reader's
+ * perspective and therefore be used by several backends simultaneously.
+ *
+ * \param val	the stat_item object
+ * \param idx	identifies the next value to be read
+ * \param value	a pointer to store the value
+ * \returns  the increment of the index (0: no value has been read,
+ *           1: one value has been taken,
+ *           (1+n): n values have been skipped, one has been taken)
+ */
+int stat_item_get_next(const struct stat_item *val, int32_t *idx, int32_t *value);
+
+/*! \brief Get the last (freshest) value */
+static int32_t stat_item_get_last(const struct stat_item *val);
+
+/*! \brief Skip all values of the item and update idx accordingly */
+int stat_item_discard(const struct stat_item *val, int32_t *idx);
+
+static inline int32_t stat_item_get_last(const struct stat_item *val)
+{
+	return val->values[val->last_offs];
+}
+/*! @} */
diff --git a/src/Makefile.am b/src/Makefile.am
index 4bf3408e..93fc3833 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -15,7 +15,7 @@ libosmocore_la_SOURCES = timer.c select.c signal.c msgb.c bits.c \
 			 gsmtap_util.c crc16.c panic.c backtrace.c \
 			 conv.c application.c rbtree.c strrb.c \
 			 loggingrb.c crc8gen.c crc16gen.c crc32gen.c crc64gen.c \
-			 macaddr.c
+			 macaddr.c stat_item.c
 
 BUILT_SOURCES = crc8gen.c crc16gen.c crc32gen.c crc64gen.c
 
diff --git a/src/stat_item.c b/src/stat_item.c
new file mode 100644
index 00000000..7b169ea0
--- /dev/null
+++ b/src/stat_item.c
@@ -0,0 +1,215 @@
+/* utility routines for keeping conters about events and the event rates */
+
+/* (C) 2015 by Sysmocom s.f.m.c. GmbH
+ * (C) 2009-2010 by Harald Welte <laforge@gnumonks.org>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+/*! \addtogroup stat_item
+ *  @{
+ */
+
+/*! \file stat_item.c */
+
+
+#include <stdint.h>
+#include <string.h>
+
+#include <osmocom/core/utils.h>
+#include <osmocom/core/linuxlist.h>
+#include <osmocom/core/talloc.h>
+#include <osmocom/core/timer.h>
+#include <osmocom/core/stat_item.h>
+
+static LLIST_HEAD(stat_item_groups);
+
+static void *tall_stat_item_ctx;
+
+/*! \brief Allocate a new group of counters according to description
+ *  \param[in] ctx \ref talloc context
+ *  \param[in] desc Statistics item group description
+ *  \param[in] idx Index of new stat item group
+ */
+struct stat_item_group *stat_item_group_alloc(void *ctx,
+					    const struct stat_item_group_desc *desc,
+					    unsigned int idx)
+{
+	unsigned int group_size;
+	unsigned int items_size = 0;
+	unsigned int item_idx;
+	void *items;
+
+	struct stat_item_group *group;
+
+	group_size = sizeof(struct stat_item_group) +
+			desc->num_items * sizeof(struct stat_item *);
+
+	if (!ctx)
+		ctx = tall_stat_item_ctx;
+
+	group = talloc_zero_size(ctx, group_size);
+	if (!group)
+		return NULL;
+
+	group->desc = desc;
+	group->idx = idx;
+
+	/* Get combined size of all items */
+	for (item_idx = 0; item_idx < desc->num_items; item_idx++) {
+		unsigned int size;
+		size = sizeof(struct stat_item) +
+			sizeof(int32_t) * desc->item_desc[item_idx].num_values;
+		/* Align to pointer size */
+		size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
+
+		/* Store offsets into the item array */
+		group->items[item_idx] = (void *)items_size;
+
+		items_size += size;
+	}
+
+	items = talloc_zero_size(group, items_size);
+	if (!items) {
+		talloc_free(group);
+		return NULL;
+	}
+
+	/* Update item pointers */
+	for (item_idx = 0; item_idx < desc->num_items; item_idx++) {
+		struct stat_item *item = (struct stat_item *)
+			((uint8_t *)items + (int)group->items[item_idx]);
+		unsigned int i;
+
+		group->items[item_idx] = item;
+		item->last_offs = desc->item_desc[item_idx].num_values - 1;
+		item->last_value_index = -1;
+		item->desc = &desc->item_desc[item_idx];
+
+		for (i = 0; i <= item->last_offs; i++)
+			item->values[i] = desc->item_desc[item_idx].default_value;
+	}
+
+	llist_add(&group->list, &stat_item_groups);
+
+	return group;
+}
+
+/*! \brief Free the memory for the specified group of counters */
+void stat_item_group_free(struct stat_item_group *grp)
+{
+	llist_del(&grp->list);
+	talloc_free(grp);
+}
+
+void stat_item_set(struct stat_item *item, int32_t value)
+{
+	item->last_offs += 1;
+	if (item->last_offs >= item->desc->num_values)
+		item->last_offs = 0;
+
+	item->last_value_index += 1;
+
+	item->values[item->last_offs] = value;
+}
+
+int stat_item_get_next(const struct stat_item *item, int32_t *next_idx,
+	int32_t *value)
+{
+	int32_t delta = item->last_value_index + 1 - *next_idx;
+	int n_values = 0;
+	int next_offs;
+
+	if (delta == 0)
+		/* All items have been read */
+		return 0;
+
+	if (delta < 0 || delta > item->desc->num_values) {
+		n_values = delta - item->desc->num_values;
+		delta = item->desc->num_values;
+	}
+
+	next_offs = item->last_offs + 1 - delta;
+	if (next_offs < 0)
+		next_offs += item->desc->num_values;
+
+	*value = item->values[next_offs];
+
+	n_values += 1;
+	delta -= 1;
+	*next_idx = item->last_value_index + 1 - delta;
+
+	return n_values;
+}
+
+/*! \brief Skip all values and update idx accordingly */
+int stat_item_discard(const struct stat_item *item, int32_t *idx)
+{
+	int discarded = item->last_value_index + 1 - *idx;
+	*idx = item->last_value_index + 1;
+
+	return discarded;
+}
+
+
+/*! \brief Initialize the stat item module */
+int stat_item_init(void *tall_ctx)
+{
+	tall_stat_item_ctx = tall_ctx;
+
+	return 0;
+}
+
+/*! \brief Search for item group based on group name and index */
+struct stat_item_group *stat_item_get_group_by_name_idx(
+	const char *name, const unsigned int idx)
+{
+	struct stat_item_group *statg;
+
+	llist_for_each_entry(statg, &stat_item_groups, list) {
+		if (!statg->desc)
+			continue;
+
+		if (!strcmp(statg->desc->group_name_prefix, name) &&
+				statg->idx == idx)
+			return statg;
+	}
+	return NULL;
+}
+
+/*! \brief Search for item group based on group name */
+const struct stat_item *stat_item_get_by_name(
+	const struct stat_item_group *statg, const char *name)
+{
+	int i;
+	const struct stat_item_desc *item_desc;
+
+	if (!statg->desc)
+		return NULL;
+
+	for (i = 0; i < statg->desc->num_items; i++) {
+		item_desc = &statg->desc->item_desc[i];
+
+		if (!strcmp(item_desc->name, name)) {
+			return statg->items[i];
+		}
+	}
+	return NULL;
+}
+
+/*! @} */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index cf0977de..223535ff 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -9,7 +9,7 @@ check_PROGRAMS = timer/timer_test sms/sms_test ussd/ussd_test		\
 		 kasumi/kasumi_test logging/logging_test fr/fr_test	\
 		 loggingrb/loggingrb_test strrb/strrb_test              \
 		 vty/vty_test comp128/comp128_test utils/utils_test	\
-		 smscb/gsm0341_test
+		 smscb/gsm0341_test stats/stats_test
 
 if ENABLE_MSGFILE
 check_PROGRAMS += msgfile/msgfile_test
@@ -18,6 +18,9 @@ endif
 utils_utils_test_SOURCES = utils/utils_test.c
 utils_utils_test_LDADD = $(top_builddir)/src/libosmocore.la $(top_builddir)/src/gsm/libosmogsm.la
 
+stats_stats_test_SOURCES = stats/stats_test.c
+stats_stats_test_LDADD = $(top_builddir)/src/libosmocore.la $(top_builddir)/src/gsm/libosmogsm.la
+
 a5_a5_test_SOURCES = a5/a5_test.c
 a5_a5_test_LDADD = $(top_builddir)/src/libosmocore.la $(top_builddir)/src/gsm/libgsmint.la
 
@@ -120,7 +123,7 @@ EXTRA_DIST = testsuite.at $(srcdir)/package.m4 $(TESTSUITE)		\
              fr/fr_test.ok loggingrb/logging_test.ok			\
              loggingrb/logging_test.err	strrb/strrb_test.ok		\
 	     vty/vty_test.ok comp128/comp128_test.ok			\
-	     utils/utils_test.ok
+	     utils/utils_test.ok stats/stats_test.ok
 
 DISTCLEANFILES = atconfig
 
diff --git a/tests/stats/stats_test.c b/tests/stats/stats_test.c
new file mode 100644
index 00000000..b4143853
--- /dev/null
+++ b/tests/stats/stats_test.c
@@ -0,0 +1,198 @@
+/* tests for statistics */
+/*
+ * (C) 2015 Sysmocom s.m.f.c. GmbH
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+#include <osmocom/core/logging.h>
+#include <osmocom/core/utils.h>
+#include <osmocom/core/stat_item.h>
+
+#include <stdio.h>
+
+static void stat_test(void)
+{
+	enum test_items {
+		TEST_A_ITEM,
+		TEST_B_ITEM,
+	};
+
+	static const struct stat_item_desc item_description[] = {
+		{ "item.a", "The A value", "ma", 4, -1 },
+		{ "item.b", "The B value", "kb", 7, -1 },
+	};
+
+	static const struct stat_item_group_desc statg_desc = {
+		.group_name_prefix = "test.one",
+		.group_description = "Test number 1",
+		.num_items = ARRAY_SIZE(item_description),
+		.item_desc = item_description,
+	};
+
+	struct stat_item_group *statg =
+		stat_item_group_alloc(NULL, &statg_desc, 0);
+
+	struct stat_item_group *sgrp2;
+	const struct stat_item *sitem1, *sitem2;
+	int rc;
+	int32_t value;
+	int32_t rd_a = 0;
+	int32_t rd_b = 0;
+	int i;
+
+	OSMO_ASSERT(statg != NULL);
+
+	sgrp2 = stat_item_get_group_by_name_idx("test.one", 0);
+	OSMO_ASSERT(sgrp2 == statg);
+
+	sgrp2 = stat_item_get_group_by_name_idx("test.one", 1);
+	OSMO_ASSERT(sgrp2 == NULL);
+
+	sgrp2 = stat_item_get_group_by_name_idx("test.two", 0);
+	OSMO_ASSERT(sgrp2 == NULL);
+
+	sitem1 = stat_item_get_by_name(statg, "item.c");
+	OSMO_ASSERT(sitem1 == NULL);
+
+	sitem1 = stat_item_get_by_name(statg, "item.a");
+	OSMO_ASSERT(sitem1 != NULL);
+	OSMO_ASSERT(sitem1 == statg->items[TEST_A_ITEM]);
+
+	sitem2 = stat_item_get_by_name(statg, "item.b");
+	OSMO_ASSERT(sitem2 != NULL);
+	OSMO_ASSERT(sitem2 != sitem1);
+	OSMO_ASSERT(sitem2 == statg->items[TEST_B_ITEM]);
+
+	value = stat_item_get_last(statg->items[TEST_A_ITEM]);
+	OSMO_ASSERT(value == -1);
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 0);
+
+	stat_item_set(statg->items[TEST_A_ITEM], 1);
+
+	value = stat_item_get_last(statg->items[TEST_A_ITEM]);
+	OSMO_ASSERT(value == 1);
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 1);
+	OSMO_ASSERT(value == 1);
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 0);
+
+	for (i = 2; i <= 32; i++) {
+		stat_item_set(statg->items[TEST_A_ITEM], i);
+		stat_item_set(statg->items[TEST_B_ITEM], 1000 + i);
+
+		rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == i);
+
+		rc = stat_item_get_next(statg->items[TEST_B_ITEM], &rd_b, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == 1000 + i);
+	}
+
+	/* Keep 2 in FIFO */
+	stat_item_set(statg->items[TEST_A_ITEM], 33);
+	stat_item_set(statg->items[TEST_B_ITEM], 1000 + 33);
+
+	for (i = 34; i <= 64; i++) {
+		stat_item_set(statg->items[TEST_A_ITEM], i);
+		stat_item_set(statg->items[TEST_B_ITEM], 1000 + i);
+
+		rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == i-1);
+
+		rc = stat_item_get_next(statg->items[TEST_B_ITEM], &rd_b, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == 1000 + i-1);
+	}
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 1);
+	OSMO_ASSERT(value == 64);
+
+	rc = stat_item_get_next(statg->items[TEST_B_ITEM], &rd_b, &value);
+	OSMO_ASSERT(rc == 1);
+	OSMO_ASSERT(value == 1000 + 64);
+
+	/* Overrun FIFOs */
+	for (i = 65; i <= 96; i++) {
+		stat_item_set(statg->items[TEST_A_ITEM], i);
+		stat_item_set(statg->items[TEST_B_ITEM], 1000 + i);
+	}
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 93 - 65 + 1);
+	OSMO_ASSERT(value == 93);
+
+	for (i = 94; i <= 96; i++) {
+		rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == i);
+	}
+
+	rc = stat_item_get_next(statg->items[TEST_B_ITEM], &rd_b, &value);
+	OSMO_ASSERT(rc == 90 - 65 + 1);
+	OSMO_ASSERT(value == 1000 + 90);
+
+	for (i = 91; i <= 96; i++) {
+		rc = stat_item_get_next(statg->items[TEST_B_ITEM], &rd_b, &value);
+		OSMO_ASSERT(rc == 1);
+		OSMO_ASSERT(value == 1000 + i);
+	}
+
+	/* Test Discard */
+	stat_item_set(statg->items[TEST_A_ITEM], 97);
+	rc = stat_item_discard(statg->items[TEST_A_ITEM], &rd_a);
+	OSMO_ASSERT(rc == 1);
+
+	rc = stat_item_discard(statg->items[TEST_A_ITEM], &rd_a);
+	OSMO_ASSERT(rc == 0);
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 0);
+
+	stat_item_set(statg->items[TEST_A_ITEM], 98);
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 1);
+	OSMO_ASSERT(value == 98);
+
+	rc = stat_item_get_next(statg->items[TEST_A_ITEM], &rd_a, &value);
+	OSMO_ASSERT(rc == 0);
+
+	stat_item_group_free(statg);
+
+	sgrp2 = stat_item_get_group_by_name_idx("test.one", 0);
+	OSMO_ASSERT(sgrp2 == NULL);
+}
+
+int main(int argc, char **argv)
+{
+	static const struct log_info log_info = {};
+	log_init(&log_info, NULL);
+
+	stat_item_init(NULL);
+
+	stat_test();
+	return 0;
+}
diff --git a/tests/stats/stats_test.ok b/tests/stats/stats_test.ok
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/stats/stats_test.ok
diff --git a/tests/testsuite.at b/tests/testsuite.at
index fe30363c..a5427986 100644
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -136,6 +136,12 @@ cat $abs_srcdir/utils/utils_test.ok > expout
 AT_CHECK([$abs_top_builddir/tests/utils/utils_test], [0], [expout], [ignore])
 AT_CLEANUP
 
+AT_SETUP([stats])
+AT_KEYWORDS([stats])
+cat $abs_srcdir/stats/stats_test.ok > expout
+AT_CHECK([$abs_top_builddir/tests/stats/stats_test], [0], [expout], [ignore])
+AT_CLEANUP
+
 AT_SETUP([bssgp-fc])
 AT_KEYWORDS([bssgp-fc])
 cat $abs_srcdir/gb/bssgp_fc_tests.ok > expout