Support time based retention

src/daemon/main.c

@@ -1043,6 +1043,24 @@ static void backwards_compatible_config() {
     config_move(CONFIG_SECTION_GLOBAL,  "dbengine multihost disk space",
                 CONFIG_SECTION_DB,      "dbengine multihost disk space MB");
 
+    config_move(CONFIG_SECTION_DB,      "dbengine disk space MB",
+                CONFIG_SECTION_DB,      "dbengine multihost disk space MB");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine multihost disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 0 disk space MB");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine tier 1 multihost disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 1 disk space MB");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine tier 2 multihost disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 2 disk space MB");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine tier 3 multihost disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 3 disk space MB");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine tier 4 multihost disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 4 disk space MB");
+
     config_move(CONFIG_SECTION_GLOBAL,  "memory deduplication (ksm)",
                 CONFIG_SECTION_DB,      "memory deduplication (ksm)");
 
@@ -1099,7 +1117,14 @@ static int get_hostname(char *buf, size_t buf_size) {
     return gethostname(buf, buf_size);
 }
 
-static void get_netdata_configured_variables() {
+static void get_netdata_configured_variables()
+{
+    legacy_multihost_db_space = config_exists(CONFIG_SECTION_DB, "dbengine multihost disk space MB");
+    if (!legacy_multihost_db_space)
+        legacy_multihost_db_space = config_exists(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space");
+    if (!legacy_multihost_db_space)
+        legacy_multihost_db_space = config_exists(CONFIG_SECTION_GLOBAL, "dbengine disk space");
+
     backwards_compatible_config();
 
     // ------------------------------------------------------------------------
@@ -1201,20 +1226,23 @@ static void get_netdata_configured_variables() {
 
     // ------------------------------------------------------------------------
     // get default Database Engine disk space quota in MiB
+//
+//    //    if (!config_exists(CONFIG_SECTION_DB, "dbengine disk space MB") && !config_exists(CONFIG_SECTION_DB, "dbengine multihost disk space MB"))
+//
+//    default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
+//    if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
+//        netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
+//        default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
+//        config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
+//    }
+//
+//    default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
+//    if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
+//        netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
+//        default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
+//        config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
+//    }
 
-    default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
-    if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
-        netdata_log_error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
-        default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
-        config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
-    }
-
-    default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
-    if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
-        netdata_log_error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
-        default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
-        config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
-    }
 #else
     if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
        error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");

src/daemon/service.c

@@ -314,6 +314,8 @@ void *service_main(void *ptr)
         }
         real_step = USEC_PER_SEC;
 
+        dbengine_retention_statistics();
+
         svc_rrd_cleanup_obsolete_charts_from_all_hosts();
 
         if (service_running(SERVICE_MAINTENANCE))

src/database/contexts/api_v2.c

@@ -1124,6 +1124,30 @@ void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings
 
 void build_info_to_json_object(BUFFER *b);
 
+static void convert_seconds_to_dhms(time_t seconds, char *result, int result_size) {
+    int days, hours, minutes;
+
+    days = (int) (seconds / (24 * 3600));
+    seconds = (int) (seconds % (24 * 3600));
+    hours = (int) (seconds / 3600);
+    seconds %= 3600;
+    minutes = (int) (seconds / 60);
+    seconds %= 60;
+
+    // Format the result into the provided string buffer
+    BUFFER *buf = buffer_create(128, NULL);
+    if (days)
+        buffer_sprintf(buf,"%d day%s%s", days, days==1 ? "" : "s", hours || minutes ? ", " : "");
+    if (hours)
+        buffer_sprintf(buf,"%d hour%s%s", hours, hours==1 ? "" : "s", minutes ? ", " : "");
+    if (minutes)
+        buffer_sprintf(buf,"%d minute%s%s", minutes, minutes==1 ? "" : "s", seconds ? ", " : "");
+    if (seconds)
+        buffer_sprintf(buf,"%d second%s", (int) seconds, seconds==1 ? "" : "s");
+    strncpyz(result, buffer_tostring(buf), result_size);
+    buffer_free(buf);
+}
+
 void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now_s, bool info, bool array) {
     if(!now_s)
         now_s = now_realtime_sec();
@@ -1151,11 +1175,15 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
         buffer_json_cloud_status(wb, now_s);
 
         buffer_json_member_add_array(wb, "db_size");
+        size_t group_seconds = localhost->rrd_update_every;
         for (size_t tier = 0; tier < storage_tiers; tier++) {
             STORAGE_ENGINE *eng = localhost->db[tier].eng;
             if (!eng) continue;
 
+            group_seconds *= storage_tiers_grouping_iterations[tier];
             uint64_t max = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
+            if (!max)
+                max = get_directory_free_bytes_space(multidb_ctx[tier]);
             uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
             time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
             size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si);
@@ -1168,6 +1196,10 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
 
             buffer_json_add_array_item_object(wb);
             buffer_json_member_add_uint64(wb, "tier", tier);
+            char human_retention[128];
+            convert_seconds_to_dhms((time_t) group_seconds, human_retention, sizeof(human_retention) - 1);
+            buffer_json_member_add_string(wb, "point_every", human_retention);
+
             buffer_json_member_add_uint64(wb, "metrics", storage_engine_metrics(eng->seb, localhost->db[tier].si));
             buffer_json_member_add_uint64(wb, "samples", storage_engine_samples(eng->seb, localhost->db[tier].si));
 
@@ -1178,13 +1210,30 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
             }
 
             if(first_time_s) {
+                time_t retention = now_s - first_time_s;
+
                 buffer_json_member_add_time_t(wb, "from", first_time_s);
                 buffer_json_member_add_time_t(wb, "to", now_s);
-                buffer_json_member_add_time_t(wb, "retention", now_s - first_time_s);
+                buffer_json_member_add_time_t(wb, "retention", retention);
+
+                convert_seconds_to_dhms(retention, human_retention, sizeof(human_retention) - 1);
+                buffer_json_member_add_string(wb, "retention_human", human_retention);
 
-                if(used || max) // we have disk space information
-                    buffer_json_member_add_time_t(wb, "expected_retention",
-                                                  (time_t) ((NETDATA_DOUBLE) (now_s - first_time_s) * 100.0 / percent));
+                if(used || max) { // we have disk space information
+                    time_t time_retention =  multidb_ctx[tier]->config.max_retention_s;
+                    time_t space_retention = (time_t)((NETDATA_DOUBLE)(now_s - first_time_s) * 100.0 / percent);
+                    time_t actual_retention = MIN(space_retention, time_retention ? time_retention : space_retention);
+
+                    if (time_retention) {
+                        convert_seconds_to_dhms(time_retention, human_retention, sizeof(human_retention) - 1);
+                        buffer_json_member_add_time_t(wb, "requested_retention", time_retention);
+                        buffer_json_member_add_string(wb, "requested_retention_human", human_retention);
+                    }
+
+                    convert_seconds_to_dhms(actual_retention, human_retention, sizeof(human_retention) - 1);
+                    buffer_json_member_add_time_t(wb, "expected_retention", actual_retention);
+                    buffer_json_member_add_string(wb, "expected_retention_human", human_retention);
+                }
             }
 
             if(currently_collected_metrics)

src/database/engine/datafile.c

@@ -543,7 +543,7 @@ int init_data_files(struct rrdengine_instance *ctx)
         if (ctx->loading.create_new_datafile_pair)
             create_new_datafile_pair(ctx, false);
 
-        while(rrdeng_ctx_exceeded_disk_quota(ctx))
+        while(rrdeng_ctx_tier_cap_exceeded(ctx))
             datafile_delete(ctx, ctx->datafiles.first, false, false);
     }
 

src/database/engine/journalfile.c

@@ -669,6 +669,7 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
     }
 
     time_t now_s = max_acceptable_collected_time();
+    time_t extent_first_time_s = journalfile->v2.first_time_s ? journalfile->v2.first_time_s : LONG_MAX;
     for (i = 0; i < count ; ++i) {
         nd_uuid_t *temp_id;
         uint8_t page_type = jf_metric_data->descr[i].type;
@@ -728,8 +729,18 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
                 journalfile->datafile,
                 jf_metric_data->extent_offset, jf_metric_data->extent_size, jf_metric_data->descr[i].page_length);
 
+        extent_first_time_s = MIN(extent_first_time_s, vd.start_time_s);
+
         mrg_metric_release(main_mrg, metric);
     }
+
+    journalfile->v2.first_time_s = extent_first_time_s;
+
+    time_t old = __atomic_load_n(&ctx->atomic.first_time_s, __ATOMIC_RELAXED);;
+    do {
+        if(old <= extent_first_time_s)
+            break;
+    } while(!__atomic_compare_exchange_n(&ctx->atomic.first_time_s, &old, extent_first_time_s, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
 }
 
 /*

src/database/engine/pagecache.h

@@ -33,15 +33,6 @@ struct page_descr_with_data {
     } link;
 };
 
-#define PAGE_INFO_SCRATCH_SZ (8)
-struct rrdeng_page_info {
-    uint8_t scratch[PAGE_INFO_SCRATCH_SZ]; /* scratch area to be used by page-cache users */
-
-    usec_t start_time_ut;
-    usec_t end_time_ut;
-    uint32_t page_length;
-};
-
 struct pg_alignment {
     uint32_t refcount;
 };