Skip to content

Commit

Permalink
Allow tier disk space to be 0 in which case the current disk space av…
Browse files Browse the repository at this point in the history 
…ailable will be used for calculations
  • Loading branch information
@stelfrag
stelfrag committed Apr 22, 2024
1 parent 7cbe3ef commit db8ff30
Show file tree
Hide file tree
Showing 5 changed files with 55 additions and 26 deletions.
2 changes: 2 additions & 0 deletions src/database/contexts/api_v2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1142,6 +1142,8 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now

group_seconds *= storage_tiers_grouping_iterations[tier];
uint64_t max = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
if (!max)
max = get_directory_free_bytes_space(multidb_ctx[tier]);
uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si);
Expand Down
18 changes: 14 additions & 4 deletions src/database/engine/rrdengine.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1759,6 +1759,15 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w
work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);
}

uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
{
uint64_t free_bytes = 0;
struct statvfs buff_statvfs;
if (statvfs(ctx->config.dbfiles_path, &buff_statvfs) == 0)
free_bytes = buff_statvfs.f_bavail * buff_statvfs.f_bsize;
return free_bytes;
}

void dbengine_retention_statistics(void)
{
static bool init = false;
Expand Down Expand Up @@ -1795,10 +1804,11 @@ void dbengine_retention_statistics(void)
uint64_t disk_space = get_used_disk_space(multidb_ctx[tier]);
time_t retention = get_tier_retention(multidb_ctx[tier]);

collected_number disk_percentage =
(collected_number)multidb_ctx[tier]->config.max_disk_space ?
100 * (collected_number)disk_space / (collected_number)multidb_ctx[tier]->config.max_disk_space :
0;
uint64_t config_disk_space = multidb_ctx[tier]->config.max_disk_space;
if (!config_disk_space)
config_disk_space = get_directory_free_bytes_space(multidb_ctx[tier]);

collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);

collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
100 * retention / multidb_ctx[tier]->config.max_retention_s :
Expand Down
1 change: 1 addition & 0 deletions src/database/engine/rrdengine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,5 +556,6 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
}

void dbengine_retention_statistics(void);
uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);

#endif /* NETDATA_RRDENGINE_H */
12 changes: 1 addition & 11 deletions src/database/engine/rrdengineapi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1162,17 +1162,7 @@ int rrdeng_init(
if (disk_space_mb && disk_space_mb < RRDENG_MIN_DISK_SPACE_MB)
disk_space_mb = RRDENG_MIN_DISK_SPACE_MB;

if (!disk_space_mb) {
struct statvfs buff_statvfs;
if (statvfs(ctx->config.dbfiles_path, &buff_statvfs) == 0) {
ctx->config.max_disk_space = buff_statvfs.f_bavail * buff_statvfs.f_bsize;
ctx->config.max_disk_space = ctx->config.max_disk_space - (uint64_t ) (ctx->config.max_disk_space * 10 / 100);
}
else
ctx->config.max_disk_space = 0;
}
else
ctx->config.max_disk_space = disk_space_mb * 1048576LLU;
ctx->config.max_disk_space = disk_space_mb * 1048576LLU;

ctx->config.max_retention_s = max_retention_s;

Expand Down
48 changes: 37 additions & 11 deletions src/database/rrdhost.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ size_t storage_tiers = 3;
bool use_direct_io = true;
size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
size_t storage_tiers_collection_per_sec[RRD_STORAGE_TIERS] = {1, 60, 3600, 8 * 3600, 24 * 3600};
//int space_per_tier[RRD_STORAGE_TIERS];
double storage_tiers_retention_days[RRD_STORAGE_TIERS] = {14, 90, 2 * 365, 0, 0};


static void calculate_tier_grouping_iterations(int update_every)
{
size_t grouping = update_every;
Expand Down Expand Up @@ -929,14 +929,44 @@ void dbengine_init(char *hostname) {

struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};

bool tiers_adjusted = false;
// bool tiers_adjusted = false;
size_t created_tiers = 0;
char dbenginepath[FILENAME_MAX + 1];
char dbengineconfig[200 + 1];
int divisor = 1;
calculate_tier_grouping_iterations(default_rrd_update_every);

default_backfill = get_dbengine_backfill(RRD_BACKFILL_NEW);
// Pvoid_t space_JudyL = NULL;
// Pvoid_t *PValue;
//
// // Query the disk space per tier
// size_t iterations = 1;
// for (size_t tier = 0; tier < storage_tiers; tier++) {
// if (tier == 0)
// snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
// else
// snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);
//
// int ret = mkdir(dbenginepath, 0775);
// if (ret != 0 && errno != EEXIST) {
// nd_log(NDLS_DAEMON, NDLP_CRIT, "DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
// continue;
// }
// struct statvfs buff_statvfs;
// if (statvfs(dbenginepath, &buff_statvfs) == 0) {
// space_per_tier[tier] = (int)((buff_statvfs.f_bavail * buff_statvfs.f_bsize) / 1048576LLU);
// space_per_tier[tier] = space_per_tier[tier] - (space_per_tier[tier] * 10 / 100);
// PValue = JudyLIns(&space_JudyL, buff_statvfs.f_fsid, PJE0);
// if (PValue && !*PValue)
// *(Word_t *)PValue = (Word_t)space_per_tier[tier];
// } else
// space_per_tier[tier] = 0;
//
// // Bytes per tier
// iterations *= storage_tiers_grouping_iterations[tier];
// //storage_tiers_grouping_iterations[tier] * page_type_size[tier_page_type[tier]];
// }

for (size_t tier = 0; tier < storage_tiers; tier++) {

Expand All @@ -946,6 +976,9 @@ void dbengine_init(char *hostname) {
snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu disk space MB", tier);
disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, tier_quota_mb[tier]);

// if (!disk_space_mb)
// disk_space_mb = (int) space_per_tier[tier];

snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu retention days", tier);
storage_tiers_retention_days[tier] = config_get_float(CONFIG_SECTION_DB, dbengineconfig, storage_tiers_retention_days[tier]);
}
Expand All @@ -960,6 +993,7 @@ void dbengine_init(char *hostname) {
snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine tier %zu multihost disk space MB", tier);
else
snprintfz(dbengineconfig, sizeof(dbengineconfig) - 1, "dbengine multihost disk space MB");

snprintfz(dbengineconfig_new, sizeof(dbengineconfig_new) - 1, "dbengine tier %zu MB", tier);
config_move(CONFIG_SECTION_DB, dbengineconfig, CONFIG_SECTION_DB, dbengineconfig_new);
disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig_new, tier_quota_mb[tier]);
Expand All @@ -970,15 +1004,9 @@ void dbengine_init(char *hostname) {
else
snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%zu", netdata_configured_cache_dir, tier);

int ret = mkdir(dbenginepath, 0775);
if (ret != 0 && errno != EEXIST) {
nd_log(NDLS_DAEMON, NDLP_CRIT, "DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
break;
}

internal_error(true, "DBENGINE tier %zu grouping iterations is set to %zu", tier, storage_tiers_grouping_iterations[tier]);

tiers_init[tier].disk_space_mb = disk_space_mb;
tiers_init[tier].disk_space_mb = (int) disk_space_mb;
tiers_init[tier].tier = tier;
tiers_init[tier].retention_seconds = (size_t) (86400.0 * storage_tiers_retention_days[tier]);
strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);
Expand All @@ -992,8 +1020,6 @@ void dbengine_init(char *hostname) {
else
dbengine_tier_init(&tiers_init[tier]);
}
if (tiers_adjusted)
config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);

for(size_t tier = 0; tier < storage_tiers ;tier++) {
void *ptr;
Expand Down

0 comments on commit db8ff30

Please sign in to comment.