From: Jaroslav Kysela Date: Thu, 3 Dec 2015 09:19:52 +0000 (+0100) Subject: DVR: space maintenance cleanups, add high watermark handling X-Git-Tag: v4.2.1~1394 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=18a4a8c8faa21739663cd58996be2fd0bd9198d8;p=thirdparty%2Ftvheadend.git DVR: space maintenance cleanups, add high watermark handling --- diff --git a/src/dvr/dvr.h b/src/dvr/dvr.h index 1a012c18d..70ed2cfd4 100644 --- a/src/dvr/dvr.h +++ b/src/dvr/dvr.h @@ -54,7 +54,8 @@ typedef struct dvr_config { uint32_t dvr_extra_time_post; uint32_t dvr_update_window; int dvr_running; - uint32_t dvr_cleanup_threshold; + uint32_t dvr_cleanup_threshold_low; + uint32_t dvr_cleanup_threshold_high; muxer_config_t dvr_muxcnf; @@ -129,13 +130,15 @@ typedef enum { DVR_RET_1WEEK = 7, DVR_RET_2WEEK = 14, DVR_RET_3WEEK = 21, - DVR_RET_1MONTH = 30, - DVR_RET_2MONTH = 60, - DVR_RET_3MONTH = 90, - DVR_RET_6MONTH = 180, - DVR_RET_1YEAR = 365, + DVR_RET_1MONTH = (30+1), + DVR_RET_2MONTH = (60+1), + DVR_RET_3MONTH = (90+2), + DVR_RET_6MONTH = (180+2), + DVR_RET_1YEAR = (365+1), + DVR_RET_2YEARS = (2*365+1), + DVR_RET_3YEARS = (3*366+1), DVR_RET_ONREMOVE = UINT32_MAX-1, // for retention only - DVR_RET_SPACENEED = UINT32_MAX-1, // for removal only + DVR_RET_SPACE = UINT32_MAX-1, // for removal only DVR_RET_FOREVER = UINT32_MAX } dvr_retention_t; diff --git a/src/dvr/dvr_config.c b/src/dvr/dvr_config.c index 4bdd14284..08ed46562 100644 --- a/src/dvr/dvr_config.c +++ b/src/dvr/dvr_config.c @@ -186,7 +186,8 @@ dvr_config_create(const char *name, const char *uuid, htsmsg_t *conf) cfg->dvr_warm_time = 30; cfg->dvr_update_window = 24 * 3600; cfg->dvr_pathname = strdup("$t$n.$x"); - cfg->dvr_cleanup_threshold = 2000; + cfg->dvr_cleanup_threshold_low = 200; + cfg->dvr_cleanup_threshold_high = 2000; /* Muxer config */ cfg->dvr_muxcnf.m_cache = MC_CACHE_DONTKEEP; @@ -516,8 +517,10 @@ dvr_config_save(dvr_config_t *cfg) lock_assert(&global_lock); dvr_config_storage_check(cfg); - if (cfg->dvr_cleanup_threshold < 100) - cfg->dvr_cleanup_threshold = 100; // as checking is only periodically, lower is not save + if (cfg->dvr_cleanup_threshold_low < 50) + cfg->dvr_cleanup_threshold_low = 50; // as checking is only periodically, lower is not save + if (cfg->dvr_cleanup_threshold_high < cfg->dvr_cleanup_threshold_high) + cfg->dvr_cleanup_threshold_high = cfg->dvr_cleanup_threshold_low + 50; if (cfg->dvr_removal_days != DVR_RET_FOREVER && cfg->dvr_removal_days > cfg->dvr_retention_days) cfg->dvr_retention_days = DVR_RET_ONREMOVE; @@ -714,7 +717,7 @@ dvr_config_class_removal_list ( void *o, const char *lang ) { N_("3 months"), DVR_RET_3MONTH }, { N_("6 months"), DVR_RET_6MONTH }, { N_("1 year"), DVR_RET_1YEAR }, - { N_("Until space needed"), DVR_RET_SPACENEED }, + { N_("Maintained space"), DVR_RET_SPACE }, { N_("Forever"), DVR_RET_FOREVER }, }; return strtab2htsmsg_u32(tab, 1, lang); @@ -735,6 +738,8 @@ dvr_config_class_retention_list ( void *o, const char *lang ) { N_("3 months"), DVR_RET_3MONTH }, { N_("6 months"), DVR_RET_6MONTH }, { N_("1 year"), DVR_RET_1YEAR }, + { N_("2 years"), DVR_RET_2YEARS }, + { N_("3 years"), DVR_RET_3YEARS }, { N_("On file removal"), DVR_RET_ONREMOVE }, { N_("Forever"), DVR_RET_FOREVER }, }; @@ -979,10 +984,18 @@ const idclass_t dvr_config_class = { }, { .type = PT_U32, - .id = "storage-cleanup", - .name = N_("\"Until space needed\" cleanup below (MB)"), - .off = offsetof(dvr_config_t, dvr_cleanup_threshold), - .def.i = 2000, //2000 MB + .id = "storage-cleanup-low", + .name = N_("Maintain free storage space (MiB)"), + .off = offsetof(dvr_config_t, dvr_cleanup_threshold_low), + .def.i = 200, + .group = 2, + }, + { + .type = PT_U32, + .id = "storage-cleanup-high", + .name = N_("Maintain used storage space (MiB)"), + .off = offsetof(dvr_config_t, dvr_cleanup_threshold_high), + .def.i = 2000, .group = 2, }, { diff --git a/src/dvr/dvr_db.c b/src/dvr/dvr_db.c index 3b7b5758a..b744f3721 100644 --- a/src/dvr/dvr_db.c +++ b/src/dvr/dvr_db.c @@ -277,9 +277,9 @@ dvr_entry_get_removal_string ( dvr_entry_t *de ) char buf[24]; uint32_t removal = dvr_entry_get_removal_days(de); - if (removal < DVR_RET_SPACENEED) + if (removal < DVR_RET_SPACE) snprintf(buf, sizeof(buf), "%i days", removal); - else if (removal == DVR_RET_SPACENEED) + else if (removal == DVR_RET_SPACE) return strdup("Until space needed"); else return strdup("Forever"); @@ -388,7 +388,7 @@ dvr_entry_retention_timer(dvr_entry_t *de) uint32_t retention = dvr_entry_get_retention_days(de); stop = de->de_stop + removal * (time_t)86400; - if ((removal > 0 || retention == 0) && removal < DVR_RET_SPACENEED) { + if ((removal > 0 || retention == 0) && removal < DVR_RET_SPACE) { if (stop > dispatch_clock) { dvr_entry_retention_arm(de, dvr_timer_remove_files, stop); return; @@ -2219,7 +2219,7 @@ dvr_entry_class_removal_list ( void *o, const char *lang ) { N_("3 months"), DVR_RET_3MONTH }, { N_("6 months"), DVR_RET_6MONTH }, { N_("1 year"), DVR_RET_1YEAR }, - { N_("Until space needed"), DVR_RET_SPACENEED }, + { N_("Maintained space"), DVR_RET_SPACE }, { N_("Forever"), DVR_RET_FOREVER }, }; return strtab2htsmsg_u32(tab, 1, lang); diff --git a/src/dvr/dvr_vfsmgr.c b/src/dvr/dvr_vfsmgr.c index a44776c43..7a13576c4 100644 --- a/src/dvr/dvr_vfsmgr.c +++ b/src/dvr/dvr_vfsmgr.c @@ -38,6 +38,9 @@ #include #endif +#define MIB(v) ((int64_t)v*((int64_t)1024*1024)) +#define TOMIB(v) (v/((int64_t)1024*1024)) + static int dvr_disk_space_config_idx; static int dvr_disk_space_config_size; static time_t dvr_disk_space_config_lastdelete; @@ -56,7 +59,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) { dvr_entry_t *de, *oldest; time_t stoptime; - int64_t requiredBytes, availBytes; + int64_t requiredBytes, maximalBytes, availBytes, usedBytes, diskBytes; int64_t clearedBytes = 0, fileSize; unsigned long int filesystemId; struct statvfs diskdata; @@ -70,7 +73,10 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) filesystemId = diskdata.f_fsid; availBytes = diskdata.f_bsize * (int64_t)diskdata.f_bavail; - requiredBytes = (int64_t)cfg->dvr_cleanup_threshold*(int64_t)1024*(int64_t)1024; + requiredBytes = MIB(cfg->dvr_cleanup_threshold_low); + diskBytes = diskdata.f_bsize * (int64_t)diskdata.f_blocks; + usedBytes = diskBytes - availBytes; + maximalBytes = MIB(cfg->dvr_cleanup_threshold_high); configName = cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile"; /* When deleting a file from the disk, the system needs some time to actually do this */ @@ -81,16 +87,17 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) return -1; } - if (diskdata.f_bsize * (int64_t)diskdata.f_blocks < requiredBytes) { - tvhlog(LOG_WARNING, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MB\" is smaller than the total disk size!", + if (diskBytes < requiredBytes) { + tvhlog(LOG_WARNING, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MiB\" is smaller than the total disk size!", configName, requiredBytes/(int64_t)1024/(int64_t)1024); - return -1; + if (maximalBytes >= usedBytes) + return -1; } - tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MB\", current free space \"%ld MB\"", - configName, requiredBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024); + tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", required/current free space \"%ld/%ld MiB\", required/current used space \"%ld/%ld MB\"", + configName, TOMIB(requiredBytes), TOMIB(availBytes), TOMIB(maximalBytes), TOMIB(usedBytes)); - while (availBytes < requiredBytes) { + while (availBytes < requiredBytes || maximalBytes < usedBytes) { oldest = NULL; stoptime = dispatch_clock; @@ -102,7 +109,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) if (dvr_entry_get_stop_time(de) > stoptime) continue; - if (dvr_entry_get_removal_days(de) != DVR_RET_SPACENEED) // only remove the allowed ones + if (dvr_entry_get_removal_days(de) != DVR_RET_SPACE) // only remove the allowed ones continue; if (dvr_get_filename(de) == NULL || dvr_get_filesize(de) <= 0) @@ -124,6 +131,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) fileSize = dvr_get_filesize(oldest); availBytes += fileSize; clearedBytes += fileSize; + usedBytes -= fileSize; localtime_r(&stoptime, &tm); if (strftime(tbuf, sizeof(tbuf), "%F %T", &tm) <= 0) @@ -149,8 +157,8 @@ dvr_disk_space_cleanup(dvr_config_t *cfg) } finish: - tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", cleared \"%ld MB\" of disk space, new free disk space \"%ld MB\"", - configName, clearedBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024); + tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", cleared \"%ld MB\" of disk space, new free disk space \"%ld MiB\", new used disk space \"%ld MiB\"", + configName, TOMIB(clearedBytes), TOMIB(availBytes), TOMIB(usedBytes)); return clearedBytes; } @@ -165,7 +173,7 @@ dvr_disk_space_check() dvr_config_t *cfg; dvr_entry_t *de; struct statvfs diskdata; - int64_t requiredBytes, availBytes; + int64_t requiredBytes, maximalBytes, availBytes, usedBytes; int idx = 0, cleanupDone = 0; pthread_mutex_lock(&global_lock); @@ -183,18 +191,26 @@ dvr_disk_space_check() statvfs(cfg->dvr_storage, &diskdata) != -1) { availBytes = diskdata.f_bsize * (int64_t)diskdata.f_bavail; - requiredBytes = (int64_t)cfg->dvr_cleanup_threshold*(int64_t)1024*(int64_t)1024; + usedBytes = (diskdata.f_bsize * (int64_t)diskdata.f_blocks) - availBytes; + requiredBytes = MIB(cfg->dvr_cleanup_threshold_low); + maximalBytes = MIB(cfg->dvr_cleanup_threshold_high); - if (availBytes < requiredBytes) { + if (availBytes < requiredBytes || maximalBytes > usedBytes) { LIST_FOREACH(de, &dvrentries, de_global_link) { /* only start cleanup if we are actually writing files right now */ if (de->de_sched_state != DVR_RECORDING || !de->de_config || de->de_config != cfg) - continue; - - tvhlog(LOG_WARNING, "dvr","running out of free disk space for dvr config \"%s\", required free space \"%ld MB\", current free space \"%ld MB\"", - cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile", - requiredBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024); + goto checking; + + if (availBytes < requiredBytes) { + tvhlog(LOG_WARNING, "dvr","running out of free disk space for dvr config \"%s\", required free space \"%ld MiB\", current free space \"%ld MiB\"", + cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile", + TOMIB(requiredBytes), TOMIB(availBytes)); + } else { + tvhlog(LOG_WARNING, "dvr","running out of used disk space for dvr config \"%s\", required used space \"%ld MiB\", current used space \"%ld MiB\"", + cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile", + TOMIB(maximalBytes), TOMIB(usedBytes)); + } /* only cleanup one directory at the time as the system needs time to delete the actual files */ dvr_disk_space_cleanup(de->de_config); @@ -202,10 +218,10 @@ dvr_disk_space_check() dvr_disk_space_config_idx = idx + 1; break; } - } - else { - tvhlog(LOG_DEBUG, "dvr","checking free disk space for config \"%s\" : OK", - cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile"); + } else { +checking: + tvhlog(LOG_DEBUG, "dvr","checking free and used disk space for config \"%s\" : OK", + cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile"); } } }