From: Jaroslav Kysela <perex@perex.cz>
Date: Sun, 13 Mar 2016 18:19:50 +0000 (+0100)
Subject: add htsmsg/htsmsg_field to memoryinfo statistics
X-Git-Tag: v4.2.1~871
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=72a312c7000c3d462a6e614f876570fe455701a7;p=thirdparty%2Ftvheadend.git

add htsmsg/htsmsg_field to memoryinfo statistics
---

diff --git a/configure b/configure
index dac82a135..ff9ac8496 100755
--- a/configure
+++ b/configure
@@ -55,6 +55,7 @@ OPTIONS=(
   "android:no"
   "tsdebug:no"
   "gtimer_check:no"
+  "slow_memoryinfo:no"
   "libsystemd_daemon:no"
   "bintray_cache:yes"
 )
diff --git a/src/htsmsg.c b/src/htsmsg.c
index 2238f04da..844b14b3f 100644
--- a/src/htsmsg.c
+++ b/src/htsmsg.c
@@ -23,13 +23,19 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include "build.h"
 #include "htsmsg.h"
 #include "misc/dbl.h"
 #include "htsmsg_json.h"
+#include "memoryinfo.h"
+
+#if ENABLE_SLOW_MEMORYINFO
+memoryinfo_t htsmsg_memoryinfo = { .my_name = "htsmsg" };
+memoryinfo_t htsmsg_field_memoryinfo = { .my_name = "htsmsg field" };
+#endif
 
 static void htsmsg_clear(htsmsg_t *msg);
-static htsmsg_t *
-htsmsg_field_get_msg ( htsmsg_field_t *f, int islist );
+static htsmsg_t *htsmsg_field_get_msg ( htsmsg_field_t *f, int islist );
 
 /**
  *
@@ -44,13 +50,21 @@ htsmsg_field_data_destroy(htsmsg_field_t *f)
     break;
 
   case HMF_STR:
-    if(f->hmf_flags & HMF_ALLOCED)
+    if(f->hmf_flags & HMF_ALLOCED) {
+#if ENABLE_SLOW_MEMORYINFO
+      memoryinfo_remove(&htsmsg_field_memoryinfo, strlen(f->hmf_str) + 1);
+#endif
       free((void *)f->hmf_str);
+    }
     break;
 
   case HMF_BIN:
-    if(f->hmf_flags & HMF_ALLOCED)
+    if(f->hmf_flags & HMF_ALLOCED) {
+#if ENABLE_SLOW_MEMORYINFO
+      memoryinfo_remove(&htsmsg_field_memoryinfo, f->hmf_binsize);
+#endif
       free((void *)f->hmf_bin);
+    }
     break;
   default:
     break;
@@ -64,12 +78,24 @@ htsmsg_field_data_destroy(htsmsg_field_t *f)
 void
 htsmsg_field_destroy(htsmsg_t *msg, htsmsg_field_t *f)
 {
+#if ENABLE_SLOW_MEMORYINFO
+  size_t asize = 0;
+#endif
+
   TAILQ_REMOVE(&msg->hm_fields, f, hmf_link);
 
   htsmsg_field_data_destroy(f);
 
-  if(f->hmf_flags & HMF_NAME_ALLOCED)
+  if (f->hmf_flags & HMF_NAME_ALLOCED) {
+#if ENABLE_SLOW_MEMORYINFO
+    asize += strlen(f->hmf_name);
+#endif
     free((void *)f->hmf_name);
+  }
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_free(&htsmsg_field_memoryinfo,
+                  sizeof(*f) + f->hmf_edata_size + asize);
+#endif
   free(f);
 }
 
@@ -94,6 +120,9 @@ htsmsg_field_add(htsmsg_t *msg, const char *name, int type, int flags, size_t es
 {
   size_t nsize = 0;
   htsmsg_field_t *f;
+#if ENABLE_SLOW_MEMORYINFO
+  size_t asize = 0;
+#endif
   
   if((flags & HMF_NAME_INALLOCED) && name)
     nsize = strlen(name) + 1;
@@ -112,18 +141,32 @@ htsmsg_field_add(htsmsg_t *msg, const char *name, int type, int flags, size_t es
     if (name) {
       f->hmf_name = f->hmf_edata;
       strcpy(f->hmf_edata, name);
+    } else {
+      f->hmf_name = NULL;
     }
   } else if(flags & HMF_NAME_ALLOCED) {
     f->hmf_name = name ? strdup(name) : NULL;
+#if ENABLE_SLOW_MEMORYINFO
+    asize = name ? strlen(name) + 1 : 0;
+#endif
   } else {
     f->hmf_name = name;
   }
 
-  if(esize)
-    f->hmf_str = f->hmf_edata + nsize;
+  if(esize) {
+    if(type == HMF_STR)
+      f->hmf_str = f->hmf_edata + nsize;
+    else if(type == HMF_BIN)
+      f->hmf_bin = f->hmf_edata + nsize;
+  }
 
   f->hmf_type = type;
   f->hmf_flags = flags;
+#if ENABLE_SLOW_MEMORYINFO
+  f->hmf_edata_size = nsize + esize;
+  memoryinfo_alloc(&htsmsg_field_memoryinfo,
+                   sizeof(htsmsg_field_t) + f->hmf_edata_size + asize);
+#endif
   return f;
 }
 
@@ -196,9 +239,15 @@ htsmsg_create_map(void)
   htsmsg_t *msg;
 
   msg = malloc(sizeof(htsmsg_t));
-  TAILQ_INIT(&msg->hm_fields);
-  msg->hm_data = NULL;
-  msg->hm_islist = 0;
+  if (msg) {
+    TAILQ_INIT(&msg->hm_fields);
+    msg->hm_data = NULL;
+    msg->hm_data_size = 0;
+    msg->hm_islist = 0;
+#if ENABLE_SLOW_MEMORYINFO
+    memoryinfo_alloc(&htsmsg_memoryinfo, sizeof(htsmsg_t));
+#endif
+  }
   return msg;
 }
 
@@ -211,9 +260,15 @@ htsmsg_create_list(void)
   htsmsg_t *msg;
 
   msg = malloc(sizeof(htsmsg_t));
-  TAILQ_INIT(&msg->hm_fields);
-  msg->hm_data = NULL;
-  msg->hm_islist = 1;
+  if (msg) {
+    TAILQ_INIT(&msg->hm_fields);
+    msg->hm_data = NULL;
+    msg->hm_data_size = 0;
+    msg->hm_islist = 1;
+#if ENABLE_SLOW_MEMORYINFO
+    memoryinfo_alloc(&htsmsg_memoryinfo, sizeof(htsmsg_t));
+#endif
+  }
   return msg;
 }
 
@@ -224,11 +279,23 @@ htsmsg_create_list(void)
 void
 htsmsg_destroy(htsmsg_t *msg)
 {
+#if ENABLE_SLOW_MEMORYINFO
+  size_t size = 0;
+#endif
+
   if(msg == NULL)
     return;
 
   htsmsg_clear(msg);
-  free((void *)msg->hm_data);
+  if (msg->hm_data) {
+    free((void *)msg->hm_data);
+#if ENABLE_SLOW_MEMORYINFO
+    size += msg->hm_data_size;
+#endif
+  }
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_free(&htsmsg_memoryinfo, sizeof(htsmsg_t) + size);
+#endif
   free(msg);
 }
 
@@ -289,6 +356,7 @@ htsmsg_add_str(htsmsg_t *msg, const char *name, const char *str)
   htsmsg_field_t *f = htsmsg_field_add(msg, name, HMF_STR, HMF_NAME_INALLOCED,
                                        strlen(str) + 1);
   strcpy((char *)f->hmf_str, str);
+  f->hmf_flags |= HMF_INALLOCED;
 }
 
 /*
@@ -328,10 +396,24 @@ htsmsg_field_set_str(htsmsg_field_t *f, const char *str)
 {
   if (f->hmf_type != HMF_STR)
     return 1;
-  if (f->hmf_flags & HMF_ALLOCED)
+  if (f->hmf_flags & HMF_ALLOCED) {
+#if ENABLE_SLOW_MEMORYINFO
+    memoryinfo_remove(&htsmsg_field_memoryinfo, strlen(f->hmf_str) + 1);
+#endif
     free((void *)f->hmf_str);
+  }
+  else if (f->hmf_flags & HMF_INALLOCED) {
+    if (strlen(f->hmf_str) >= strlen(str)) {
+      strcpy((char *)f->hmf_str, str);
+      return 0;
+    }
+    f->hmf_flags &= ~HMF_INALLOCED;
+  }
   f->hmf_flags |= HMF_ALLOCED;
   f->hmf_str = strdup(str);
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_alloc(&htsmsg_field_memoryinfo, strlen(str) + 1);
+#endif
   return 0;
 }
 
@@ -342,8 +424,10 @@ int
 htsmsg_set_str(htsmsg_t *msg, const char *name, const char *str)
 {
   htsmsg_field_t *f = htsmsg_field_find(msg, name);
-  if (!f)
-    f = htsmsg_field_add(msg, name, HMF_STR, HMF_NAME_INALLOCED, 0);
+  if (!f) {
+    htsmsg_add_str(msg, name, str);
+    return 0;
+  }
   return htsmsg_field_set_str(f, str);
 }
 
@@ -356,6 +440,7 @@ htsmsg_add_bin(htsmsg_t *msg, const char *name, const void *bin, size_t len)
   htsmsg_field_t *f = htsmsg_field_add(msg, name, HMF_BIN, HMF_NAME_INALLOCED, len);
   f->hmf_bin = f->hmf_str;
   f->hmf_binsize = len;
+  f->hmf_flags |= HMF_INALLOCED;
   memcpy((void *)f->hmf_bin, bin, len);
 }
 
@@ -379,8 +464,12 @@ htsmsg_field_set_msg(htsmsg_field_t *f, htsmsg_t *sub)
 {
   assert(sub->hm_data == NULL);
   f->hmf_msg.hm_data = NULL;
+  f->hmf_msg.hm_data_size = 0;
   f->hmf_msg.hm_islist = sub->hm_islist;
   TAILQ_MOVE(&f->hmf_msg.hm_fields, &sub->hm_fields, hmf_link);
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_free(&htsmsg_memoryinfo, sizeof(htsmsg_t));
+#endif
   free(sub);
 
   if (f->hmf_type == (f->hmf_msg.hm_islist ? HMF_LIST : HMF_MAP))
@@ -429,8 +518,12 @@ htsmsg_add_msg_extname(htsmsg_t *msg, const char *name, htsmsg_t *sub)
 
   assert(sub->hm_data == NULL);
   f->hmf_msg.hm_data = NULL;
+  f->hmf_msg.hm_data_size = 0;
   TAILQ_MOVE(&f->hmf_msg.hm_fields, &sub->hm_fields, hmf_link);
   f->hmf_msg.hm_islist = sub->hm_islist;
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_free(&htsmsg_memoryinfo, sizeof(htsmsg_t));
+#endif
   free(sub);
 }
 
@@ -688,21 +781,15 @@ htsmsg_field_get_string(htsmsg_field_t *f)
   case HMF_STR:
     break;
   case HMF_BOOL:
-    f->hmf_str = strdup(f->hmf_bool ? "true" : "false");
-    f->hmf_type = HMF_STR;
-    f->hmf_flags |= HMF_ALLOCED;
+    htsmsg_field_set_str(f, f->hmf_bool ? "true" : "false");
     break;
   case HMF_S64:
     snprintf(buf, sizeof(buf), "%"PRId64, f->hmf_s64);
-    f->hmf_str = strdup(buf);
-    f->hmf_type = HMF_STR;
-    f->hmf_flags |= HMF_ALLOCED;
+    htsmsg_field_set_str(f, buf);
     break;
   case HMF_DBL:
     snprintf(buf, sizeof(buf), "%lf", f->hmf_dbl);
-    f->hmf_str = strdup(buf);
-    f->hmf_type = HMF_STR;
-    f->hmf_flags |= HMF_ALLOCED;
+    htsmsg_field_set_str(f, buf);
     break;
   }
   return f->hmf_str;
@@ -816,11 +903,20 @@ htsmsg_field_get_msg ( htsmsg_field_t *f, int islist )
   /* Deserialize JSON (will keep either list or map) */
   if (f->hmf_type == HMF_STR) {
     if ((m = htsmsg_json_deserialize(f->hmf_str))) {
-      free((void*)f->hmf_str);
+      if (f->hmf_flags & HMF_ALLOCED) {
+#if ENABLE_SLOW_MEMORYINFO
+        memoryinfo_free(&htsmsg_field_memoryinfo, strlen(f->hmf_str) + 1);
+#endif
+        free((void*)f->hmf_str);
+      }
       f->hmf_type          = m->hm_islist ? HMF_LIST : HMF_MAP;
       f->hmf_msg.hm_islist = m->hm_islist;
       f->hmf_msg.hm_data   = NULL;
+      f->hmf_msg.hm_data_size = 0;
       TAILQ_MOVE(&f->hmf_msg.hm_fields, &m->hm_fields, hmf_link);
+#if ENABLE_SLOW_MEMORYINFO
+      memoryinfo_free(&htsmsg_memoryinfo, sizeof(htsmsg_t));
+#endif
       free(m);
     }
   }
@@ -1087,7 +1183,6 @@ htsmsg_list_2_csv(htsmsg_t *m, char delim, int human)
   const char *ssep;
   if (!m->hm_islist) return NULL;
 
-#define MAX(a,b) ((a) < (b)) ? (a) : (b)
 #define REALLOC(l)\
   if ((alloc - used) < l) {\
     alloc = MAX((l)*2, alloc*2);\
diff --git a/src/htsmsg.h b/src/htsmsg.h
index 0d3cdf810..0f8996307 100644
--- a/src/htsmsg.h
+++ b/src/htsmsg.h
@@ -20,6 +20,7 @@
 #include <stdlib.h>
 #include <inttypes.h>
 #include "queue.h"
+#include "build.h"
 
 #define HTSMSG_ERR_FIELD_NOT_FOUND       -1
 #define HTSMSG_ERR_CONVERSION_IMPOSSIBLE -2
@@ -41,6 +42,7 @@ typedef struct htsmsg {
    * Data to be free'd when the message is destroyed
    */
   const void *hm_data;
+  size_t hm_data_size;
 } htsmsg_t;
 
 
@@ -59,8 +61,9 @@ typedef struct htsmsg_field {
   uint8_t hmf_flags;
 
 #define HMF_ALLOCED        0x1
-#define HMF_NAME_INALLOCED 0x2
-#define HMF_NAME_ALLOCED   0x4
+#define HMF_INALLOCED      0x2
+#define HMF_NAME_INALLOCED 0x4
+#define HMF_NAME_ALLOCED   0x8
 
   union {
     int64_t  s64;
@@ -73,6 +76,10 @@ typedef struct htsmsg_field {
     double dbl;
     int bool;
   } u;
+
+#if ENABLE_SLOW_MEMORYINFO
+  size_t hmf_edata_size;
+#endif
   char hmf_edata[0];
 } htsmsg_field_t;
 
@@ -418,3 +425,10 @@ const char *htsmsg_get_cdata(htsmsg_t *m, const char *field);
 char *htsmsg_list_2_csv(htsmsg_t *m, char delim, int human);
 
 htsmsg_t *htsmsg_csv_2_list(const char *str, char delim);
+
+/**
+ *
+ */
+struct memoryinfo;
+extern struct memoryinfo htsmsg_memoryinfo;
+extern struct memoryinfo htsmsg_field_memoryinfo;
diff --git a/src/htsmsg_binary.c b/src/htsmsg_binary.c
index 074da77af..0a835eaae 100644
--- a/src/htsmsg_binary.c
+++ b/src/htsmsg_binary.c
@@ -24,6 +24,7 @@
 #include <string.h>
 
 #include "htsmsg_binary.h"
+#include "memoryinfo.h"
 
 /*
  *
@@ -31,12 +32,12 @@
 static int
 htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
 {
-  unsigned type, namelen, datalen;
+  uint_fast32_t type, namelen, datalen;
+  size_t tlen;
   htsmsg_field_t *f;
   htsmsg_t *sub;
-  char *n;
   uint64_t u64;
-  int i;
+  int i, bin = 0;
 
   while(len > 5) {
 
@@ -53,36 +54,43 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
     if(len < namelen + datalen)
       return -1;
 
-    f = malloc(sizeof(htsmsg_field_t));
+    tlen = sizeof(htsmsg_field_t) +
+           (namelen ? namelen + 1 : 0) +
+           (type == HMF_STR ? datalen + 1 : 0);
+    f = malloc(tlen);
+    if (f == NULL)
+      return -1;
+#if ENABLE_SLOW_MEMORYINFO
+    f->hmf_edata_size = tlen - sizeof(htsmsg_field_t);
+    memoryinfo_alloc(&htsmsg_field_memoryinfo, tlen);
+#endif
     f->hmf_type  = type;
 
     if(namelen > 0) {
-      n = malloc(namelen + 1);
-      memcpy(n, buf, namelen);
-      n[namelen] = 0;
+      f->hmf_name = f->hmf_edata;
+      memcpy(f->hmf_edata, buf, namelen);
+      f->hmf_edata[namelen] = 0;
 
       buf += namelen;
       len -= namelen;
-      f->hmf_flags = HMF_NAME_ALLOCED;
+      f->hmf_flags = HMF_NAME_INALLOCED;
 
     } else {
-      n = NULL;
+      f->hmf_name  = NULL;
       f->hmf_flags = 0;
     }
 
-    f->hmf_name  = n;
-
     switch(type) {
     case HMF_STR:
-      f->hmf_str = n = malloc(datalen + 1);
-      memcpy(n, buf, datalen);
-      n[datalen] = 0;
-      f->hmf_flags |= HMF_ALLOCED;
+      f->hmf_str = f->hmf_edata + (namelen ? namelen + 1 : 0);
+      memcpy((char *)f->hmf_str, buf, datalen);
+      ((char *)f->hmf_str)[datalen] = 0;
       break;
 
     case HMF_BIN:
       f->hmf_bin = (const void *)buf;
       f->hmf_binsize = datalen;
+      bin = 1;
       break;
 
     case HMF_S64:
@@ -97,11 +105,16 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
       sub = &f->hmf_msg;
       TAILQ_INIT(&sub->hm_fields);
       sub->hm_data = NULL;
-      if(htsmsg_binary_des0(sub, buf, datalen) < 0) {
-        free(n);
+      i = htsmsg_binary_des0(sub, buf, datalen);
+      if (i < 0) {
+#if ENABLE_SLOW_MEMORYINFO
+        memoryinfo_free(&htsmsg_field_memoryinfo, tlen);
+#endif
         free(f);
         return -1;
       }
+      if (i > 0)
+        bin = 1;
       break;
 
     case HMF_BOOL:
@@ -109,7 +122,9 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
       break;
 
     default:
-      free(n);
+#if ENABLE_SLOW_MEMORYINFO
+      memoryinfo_free(&htsmsg_field_memoryinfo, tlen);
+#endif
       free(f);
       return -1;
     }
@@ -118,7 +133,7 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
     buf += datalen;
     len -= datalen;
   }
-  return 0;
+  return bin;
 }
 
 
@@ -127,15 +142,21 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
  *
  */
 htsmsg_t *
-htsmsg_binary_deserialize(const void *data, size_t len, const void *buf)
+htsmsg_binary_deserialize(void *data, size_t len, const void *buf)
 {
   htsmsg_t *msg = htsmsg_create_map();
-  msg->hm_data = buf;
-
-  if(htsmsg_binary_des0(msg, data, len) < 0) {
+  int r = htsmsg_binary_des0(msg, data, len);
+  if (r < 0) {
     htsmsg_destroy(msg);
     return NULL;
   }
+  if (r > 0) {
+    msg->hm_data = buf;
+    msg->hm_data_size = len;
+#if ENABLE_SLOW_MEMORYINFO
+    memoryinfo_append(&htsmsg_memoryinfo, len);
+#endif
+  }
   return msg;
 }
 
diff --git a/src/htsmsg_binary.h b/src/htsmsg_binary.h
index 74c0e1251..55dc0895e 100644
--- a/src/htsmsg_binary.h
+++ b/src/htsmsg_binary.h
@@ -24,7 +24,7 @@
 /**
  * htsmsg_binary_deserialize
  */
-htsmsg_t *htsmsg_binary_deserialize(const void *data, size_t len,
+htsmsg_t *htsmsg_binary_deserialize(void *data, size_t len,
 				    const void *buf);
 
 int htsmsg_binary_serialize(htsmsg_t *msg, void **datap, size_t *lenp,
diff --git a/src/htsmsg_xml.c b/src/htsmsg_xml.c
index b13f2d802..c9edc1a0e 100644
--- a/src/htsmsg_xml.c
+++ b/src/htsmsg_xml.c
@@ -849,6 +849,7 @@ htsmsg_xml_deserialize(char *src, char *errbuf, size_t errbufsize)
 
   if(xp.xp_srcdataused) {
     m->hm_data = src0;
+    m->hm_data_size = strlen(src0) + 1;
   } else {
     free(src0);
   }
diff --git a/src/main.c b/src/main.c
index 7c191a969..5164a717e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1135,6 +1135,10 @@ main(int argc, char **argv)
 
   /* Memoryinfo */
   memoryinfo_register(&tasklet_memoryinfo);
+#if ENABLE_SLOW_MEMORYINFO
+  memoryinfo_register(&htsmsg_memoryinfo);
+  memoryinfo_register(&htsmsg_field_memoryinfo);
+#endif
   memoryinfo_register(&pkt_memoryinfo);
   memoryinfo_register(&pktbuf_memoryinfo);
   memoryinfo_register(&pktref_memoryinfo);
diff --git a/src/memoryinfo.h b/src/memoryinfo.h
index 44d127f6b..f759a1637 100644
--- a/src/memoryinfo.h
+++ b/src/memoryinfo.h
@@ -75,5 +75,9 @@ static inline void memoryinfo_free(memoryinfo_t *my, int64_t size)
   atomic_dec_s64(&my->my_count, 1);
 }
 
+static inline void memoryinfo_remove(memoryinfo_t *my, int64_t size)
+{
+  atomic_dec_s64(&my->my_size, size);
+}
 
 #endif /* TVHEADEND_MEMORYINFO_H */