Add constant page type

src/daemon/global_statistics.c

@@ -70,6 +70,9 @@ static struct global_statistics {
     uint64_t tier0_disk_compressed_bytes;
     uint64_t tier0_disk_uncompressed_bytes;
 
+    uint64_t tier0_raw_pages;
+    uint64_t tier0_constant_pages;
+
     uint64_t db_points_stored_per_tier[RRD_STORAGE_TIERS];
 
 } global_statistics = {
@@ -89,6 +92,9 @@ static struct global_statistics {
         .tier0_hot_gorilla_buffers = 0,
         .tier0_disk_compressed_bytes = 0,
         .tier0_disk_uncompressed_bytes = 0,
+
+        .tier0_raw_pages = 0,
+        .tier0_constant_pages = 0,
 };
 
 void global_statistics_rrdset_done_chart_collection_completed(size_t *points_read_per_tier_array) {
@@ -129,6 +135,18 @@ void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size) {
     __atomic_fetch_add(&global_statistics.tier0_disk_uncompressed_bytes, size, __ATOMIC_RELAXED);
 }
 
+void global_statistic_raw_page_new() {
+    __atomic_fetch_add(&global_statistics.tier0_raw_pages, 1, __ATOMIC_RELAXED);
+}
+
+void global_statistic_constant_page_new() {
+    __atomic_fetch_add(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED);
+}
+
+void global_statistic_constant_page_rm() {
+    __atomic_fetch_sub(&global_statistics.tier0_constant_pages, 1, __ATOMIC_RELAXED);
+}
+
 void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) {
     switch(query_source) {
         case QUERY_SOURCE_API_DATA:
@@ -236,6 +254,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
     gs->tier0_disk_compressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_compressed_bytes, __ATOMIC_RELAXED);
     gs->tier0_disk_uncompressed_bytes = __atomic_load_n(&global_statistics.tier0_disk_uncompressed_bytes, __ATOMIC_RELAXED);
 
+    gs->tier0_raw_pages = __atomic_load_n(&global_statistics.tier0_raw_pages, __ATOMIC_RELAXED);
+    gs->tier0_constant_pages = __atomic_load_n(&global_statistics.tier0_constant_pages, __ATOMIC_RELAXED);
+
     for(size_t tier = 0; tier < storage_tiers ;tier++)
         gs->db_points_stored_per_tier[tier] = __atomic_load_n(&global_statistics.db_points_stored_per_tier[tier], __ATOMIC_RELAXED);
 
@@ -920,6 +941,38 @@ static void global_statistics_charts(void) {
         rrdset_done(st_tier0_compression_info);
     }
 #endif
+
+    {
+        static RRDSET *st_t0_page_type = NULL;
+
+        static RRDDIM *rd_raw_pages = NULL;
+        static RRDDIM *rd_constant_pages = NULL;
+
+        if (unlikely(!st_t0_page_type)) {
+            st_t0_page_type = rrdset_create_localhost(
+                    "netdata"
+                    , "tier0_page_type"
+                    , NULL
+                    , "tier0_page_type"
+                    , NULL
+                    , "Tier 0 page type"
+                    , "pages"
+                    , "netdata"
+                    , "stats"
+                    , 131006
+                    , localhost->rrd_update_every
+                    , RRDSET_TYPE_LINE
+            );
+
+            rd_raw_pages = rrddim_add(st_t0_page_type, "raw", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            rd_constant_pages = rrddim_add(st_t0_page_type, "constant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+        }
+
+        rrddim_set_by_pointer(st_t0_page_type, rd_raw_pages, (collected_number) gs.tier0_raw_pages);
+        rrddim_set_by_pointer(st_t0_page_type, rd_constant_pages, (collected_number) gs.tier0_constant_pages);
+
+        rrdset_done(st_t0_page_type);
+    }
 }
 
 // ----------------------------------------------------------------------------

src/daemon/global_statistics.h

@@ -50,6 +50,10 @@ void global_statistics_gorilla_buffer_add_hot();
 void global_statistics_tier0_disk_compressed_bytes(uint32_t size);
 void global_statistics_tier0_disk_uncompressed_bytes(uint32_t size);
 
+void global_statistic_raw_page_new();
+void global_statistic_constant_page_new();
+void global_statistic_constant_page_rm();
+
 void global_statistics_web_request_completed(uint64_t dt,
                                              uint64_t bytes_received,
                                              uint64_t bytes_sent,

src/daemon/main.c

@@ -1169,11 +1169,14 @@ static void get_netdata_configured_variables() {
     // ------------------------------------------------------------------------
     // get default Database Engine page type
 
-    const char *page_type = config_get(CONFIG_SECTION_DB, "dbengine page type", "raw");
+    const char *page_type = config_get(CONFIG_SECTION_DB, "dbengine page type", "constant");
     if (strcmp(page_type, "gorilla") == 0) {
         tier_page_type[0] = PAGE_GORILLA_METRICS;
+    }
+    else if (strcmp(page_type, "constant") == 0) {
+        tier_page_type[0] = PAGE_CONSTANT_METRICS;
     } else if (strcmp(page_type, "raw") != 0) {
-        netdata_log_error("Invalid dbengine page type ''%s' given. Defaulting to 'raw'.", page_type);
+        netdata_log_error("Invalid dbengine page type '%s' (expected one of 'constant', 'gorilla', 'raw'). Defaulting to 'constant'.", page_type);
     }
 
     // ------------------------------------------------------------------------

src/database/engine/page.c

@@ -2,6 +2,7 @@
 
 #include "page.h"
 
+#include "daemon/global_statistics.h"
 #include "libnetdata/libnetdata.h"
 
 typedef enum __attribute__((packed)) {
@@ -20,6 +21,10 @@ typedef struct {
     uint32_t size;
 } page_raw_t;
 
+typedef struct {
+    storage_number v;
+    uint32_t n;
+} page_constant_t;
 
 typedef struct {
     size_t num_buffers;
@@ -45,6 +50,7 @@ struct pgd {
     union {
         page_raw_t raw;
         page_gorilla_t gorilla;
+        page_constant_t constant;
     };
 };
 
@@ -165,7 +171,9 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
     pg->states = PGD_STATE_CREATED_FROM_COLLECTOR;
 
     switch (type) {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
+            global_statistic_raw_page_new();
+            // intentional fall-through
         case PAGE_TIER: {
             uint32_t size = slots * page_type_size[type];
 
@@ -176,6 +184,16 @@ PGD *pgd_create(uint8_t type, uint32_t slots)
             pg->raw.data = pgd_data_aral_alloc(size);
             break;
         }
+        case PAGE_CONSTANT_METRICS: {
+            global_statistic_constant_page_new();
+
+            internal_fatal(slots == 1,
+                      "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
+
+            pg->constant.v = SN_EMPTY_SLOT;
+            pg->constant.n = 0;
+            break;
+        }
         case PAGE_GORILLA_METRICS: {
             internal_fatal(slots == 1,
                       "DBENGINE: invalid number of slots (%u) or page type (%u)", slots, type);
@@ -222,7 +240,7 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
 
     switch (type)
     {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER:
             pg->raw.size = size;
             pg->used = size / page_type_size[type];
@@ -231,6 +249,12 @@ PGD *pgd_create_from_disk_data(uint8_t type, void *base, uint32_t size)
             pg->raw.data = pgd_data_aral_alloc(size);
             memcpy(pg->raw.data, base, size);
             break;
+        case PAGE_CONSTANT_METRICS: {
+            memcpy(&pg->constant.v, base, sizeof(storage_number));
+            memcpy(&pg->constant.n, base + sizeof(storage_number), sizeof(uint32_t));
+            pg->used = pg->slots = pg->constant.n;
+            break;
+        }
         case PAGE_GORILLA_METRICS:
             internal_fatal(size == 0, "Asked to create page with 0 data!!!");
             internal_fatal(size % sizeof(uint32_t), "Unaligned gorilla buffer size");
@@ -268,10 +292,12 @@ void pgd_free(PGD *pg)
 
     switch (pg->type)
     {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER:
             pgd_data_aral_free(pg->raw.data, pg->raw.size);
             break;
+        case PAGE_CONSTANT_METRICS:
+            break;
         case PAGE_GORILLA_METRICS: {
             if (pg->states & PGD_STATE_CREATED_FROM_DISK)
             {
@@ -365,10 +391,13 @@ uint32_t pgd_memory_footprint(PGD *pg)
 
     size_t footprint = 0;
     switch (pg->type) {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER:
             footprint = sizeof(PGD) + pg->raw.size;
             break;
+        case PAGE_CONSTANT_METRICS:
+            footprint = sizeof(PGD);
+            break;
         case PAGE_GORILLA_METRICS: {
             if (pg->states & PGD_STATE_CREATED_FROM_DISK)
                 footprint = sizeof(PGD) + pg->raw.size;
@@ -393,14 +422,18 @@ uint32_t pgd_disk_footprint(PGD *pg)
     size_t size = 0;
 
     switch (pg->type) {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER: {
             uint32_t used_size = pg->used * page_type_size[pg->type];
             internal_fatal(used_size > pg->raw.size, "Wrong disk footprint page size");
             size = used_size;
 
             break;
         }
+        case PAGE_CONSTANT_METRICS: {
+            size = sizeof(storage_number) + sizeof(uint32_t);
+            break;
+        }
         case PAGE_GORILLA_METRICS: {
             if (pg->states & PGD_STATE_CREATED_FROM_COLLECTOR ||
                 pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING ||
@@ -443,11 +476,21 @@ void pgd_copy_to_extent(PGD *pg, uint8_t *dst, uint32_t dst_size)
                    pgd_disk_footprint(pg), dst_size);
 
     switch (pg->type) {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER:
             memcpy(dst, pg->raw.data, dst_size);
             break;
-        case PAGE_GORILLA_METRICS: {
+        case PAGE_CONSTANT_METRICS:
+        {
+            memcpy(dst, &pg->constant.v, sizeof(storage_number));
+
+            pg->constant.n = pg->used;
+            memcpy(dst + sizeof(storage_number), &pg->constant.n, sizeof(uint32_t));
+
+            break;
+        }
+        case PAGE_GORILLA_METRICS:
+        {
             if ((pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING) == 0)
                 fatal("Copying to extent is supported only for PGDs that are scheduled for flushing.");
 
@@ -500,7 +543,7 @@ void pgd_append_point(PGD *pg,
         fatal("Data collection on page already scheduled for flushing");
 
     switch (pg->type) {
-        case PAGE_METRICS: {
+        case PAGE_RAW_METRICS: {
             storage_number *tier0_metric_data = (storage_number *)pg->raw.data;
             storage_number t = pack_storage_number(n, flags);
             tier0_metric_data[pg->used++] = t;
@@ -510,6 +553,43 @@ void pgd_append_point(PGD *pg,
 
             break;
         }
+        case PAGE_CONSTANT_METRICS: {
+            storage_number t = pack_storage_number(n, flags);
+
+            if (pg->used == 0)
+            {
+                pg->constant.v = t;
+                pg->used = 1;
+            }
+            else if (pg->constant.v == t)
+            {
+                pg->used++;
+            }
+            else
+            {
+                storage_number v = pg->constant.v;
+
+                global_statistic_constant_page_rm();
+                global_statistic_raw_page_new();
+
+                pg->type = PAGE_RAW_METRICS;
+                uint32_t size = pg->slots * page_type_size[pg->type];
+                pg->raw.size = size;
+                pg->raw.data = pgd_data_aral_alloc(size);
+
+                storage_number *data = (storage_number *) pg->raw.data;
+                for (uint32_t i = 0; i != pg->used; i++)
+                    data[i] = v;
+
+                data[pg->used++] = t;
+                break;
+            }
+
+            if ((pg->options & PAGE_OPTION_ALL_VALUES_EMPTY) && does_storage_number_exist(t))
+                pg->options &= ~PAGE_OPTION_ALL_VALUES_EMPTY;
+
+            break;
+        }
         case PAGE_TIER: {
             storage_number_tier1_t *tier12_metric_data = (storage_number_tier1_t *)pg->raw.data;
             storage_number_tier1_t t;
@@ -560,8 +640,9 @@ static void pgdc_seek(PGDC *pgdc, uint32_t position)
     PGD *pg = pgdc->pgd;
 
     switch (pg->type) {
-        case PAGE_METRICS:
+        case PAGE_RAW_METRICS:
         case PAGE_TIER:
+        case PAGE_CONSTANT_METRICS:
             pgdc->slots = pgdc->pgd->used;
             break;
         case PAGE_GORILLA_METRICS: {
@@ -634,7 +715,7 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
 
     switch (pgdc->pgd->type)
     {
-        case PAGE_METRICS: {
+        case PAGE_RAW_METRICS: {
             storage_number *array = (storage_number *) pgdc->pgd->raw.data;
             storage_number n = array[pgdc->position++];
 
@@ -645,6 +726,17 @@ bool pgdc_get_next_point(PGDC *pgdc, uint32_t expected_position __maybe_unused,
 
             return true;
         }
+        case PAGE_CONSTANT_METRICS: {
+            pgdc->position++;
+            storage_number n = pgdc->pgd->constant.v;
+
+            sp->min = sp->max = sp->sum = unpack_storage_number(n);
+            sp->flags = (SN_FLAGS)(n & SN_USER_FLAGS);
+            sp->count = 1;
+            sp->anomaly_count = is_storage_number_anomalous(n) ? 1 : 0;
+
+            return true;
+        }
         case PAGE_TIER: {
             storage_number_tier1_t *array = (storage_number_tier1_t *) pgdc->pgd->raw.data;
             storage_number_tier1_t n = array[pgdc->position++];