From: Jaroslav Kysela <perex@perex.cz>
Date: Mon, 8 Jan 2018 09:31:16 +0000 (+0100)
Subject: use htsmsg binary2 format for epg and config files
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=19066dc1e97d43410f8ced5cd1436269567a1624;p=thirdparty%2Ftvheadend.git

use htsmsg binary2 format for epg and config files
---

diff --git a/src/epgdb.c b/src/epgdb.c
index 56bd42d59..82cc8cd73 100644
--- a/src/epgdb.c
+++ b/src/epgdb.c
@@ -28,6 +28,7 @@
 #include "tvheadend.h"
 #include "sbuf.h"
 #include "htsmsg_binary.h"
+#include "htsmsg_binary2.h"
 #include "settings.h"
 #include "channels.h"
 #include "epg.h"
@@ -282,7 +283,7 @@ static void epg_mmap_sigbus (int sig, siginfo_t *siginfo, void *ptr)
  */
 void epg_init ( void )
 {
-  int fd = -1;
+  int fd = -1, binary2 = 0, r;
   struct stat st;
   size_t remain;
   uint8_t *mem, *rp, *zlib_mem = NULL;
@@ -343,8 +344,10 @@ void epg_init ( void )
   }
 
 #if ENABLE_ZLIB
-  if (remain > 12 && memcmp(rp, "\xff\xffGZIP00", 8) == 0) {
+  if (remain > 12 && memcmp(rp, "\xff\xffGZIP0", 7) == 0 &&
+      (rp[7] == '0' || rp[7] == '1')) {
     uint32_t orig = (rp[8] << 24) | (rp[9] << 16) | (rp[10] << 8) | rp[11];
+    binary2 = rp[7] == '1';
     tvhinfo(LS_EPGDB, "gzip format detected, inflating (ratio %.1f%% deflated size %zd)",
            (float)((remain * 100.0) / orig), remain);
     rp = zlib_mem = tvh_gzip_inflate(rp + 12, remain - 12, orig);
@@ -359,18 +362,19 @@ void epg_init ( void )
   while ( remain > 4 ) {
 
     /* Get message length */
-    uint32_t msglen = (rp[0] << 24) | (rp[1] << 16) | (rp[2] << 8) | rp[3];
-    remain    -= 4;
-    rp        += 4;
+    size_t msglen = remain;
+    htsmsg_t *m;
+    if (binary2) {
+      r = htsmsg_binary2_deserialize(&m, rp, &msglen, NULL);
+    } else {
+      r = htsmsg_binary_deserialize(&m, rp, &msglen, NULL);
+    }
 
     /* Safety check */
-    if ((int64_t)msglen > remain) {
+    if (r) {
       tvherror(LS_EPGDB, "corruption detected, some/all data lost");
       break;
     }
-    
-    /* Extract message */
-    htsmsg_t *m = htsmsg_binary_deserialize(rp, msglen, NULL);
 
     /* Next */
     rp     += msglen;
@@ -446,7 +450,13 @@ static int _epg_write ( sbuf_t *sb, htsmsg_t *m )
   size_t msglen;
   void *msgdata;
   if (m) {
-    int r = htsmsg_binary_serialize(m, &msgdata, &msglen, 0x10000);
+    int r;
+#if ENABLE_ZLIB
+    if (config.epg_compress)
+      r = htsmsg_binary2_serialize(m, &msgdata, &msglen, 0x10000);
+    else
+#endif
+      r = htsmsg_binary_serialize(m, &msgdata, &msglen, 0x10000);
     htsmsg_destroy(m);
     if (!r) {
       ret = 0;
@@ -487,7 +497,7 @@ static void epg_save_tsk_callback ( void *p, int dearmed )
   if (fd >= 0) {
 #if ENABLE_ZLIB
     if (config.epg_compress) {
-      r = tvh_gzip_deflate_fd_header(fd, sb->sb_data, size, &orig, 3) < 0;
+      r = tvh_gzip_deflate_fd_header(fd, sb->sb_data, size, &orig, 3, "01") < 0;
    } else
 #endif
       r = tvh_write(fd, sb->sb_data, orig = size);
diff --git a/src/htsmsg_binary.c b/src/htsmsg_binary.c
index c7b2f3f0d..4cfef0ec6 100644
--- a/src/htsmsg_binary.c
+++ b/src/htsmsg_binary.c
@@ -148,16 +148,17 @@ htsmsg_binary_des0(htsmsg_t *msg, const uint8_t *buf, size_t len)
   return len ? -1 : bin;
 }
 
-
-
 /*
  *
  */
 htsmsg_t *
-htsmsg_binary_deserialize(void *data, size_t len, const void *buf)
+htsmsg_binary_deserialize0(const void *data, size_t len, const void *buf)
 {
-  htsmsg_t *msg = htsmsg_create_map();
-  int r = htsmsg_binary_des0(msg, data, len);
+  htsmsg_t *msg;
+  int r;
+
+  msg = htsmsg_create_map();
+  r = htsmsg_binary_des0(msg, data, len);
   if (r < 0) {
     free((void *)buf);
     htsmsg_destroy(msg);
@@ -175,7 +176,36 @@ htsmsg_binary_deserialize(void *data, size_t len, const void *buf)
   return msg;
 }
 
-
+/*
+ *
+ */
+int
+htsmsg_binary_deserialize(htsmsg_t **msg, const void *data, size_t *len, const void *buf)
+{
+  htsmsg_t *m;
+  const uint8_t *p;
+  uint32_t l, len2;
+
+  len2 = *len;
+  *msg = NULL;
+  *len = 0;
+  if (len2 < 4) {
+    free((void *)buf);
+    return -1;
+  }
+  p = data;
+  l = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3];
+  if (l + 4 > len2) {
+    free((void *)buf);
+    return -1;
+  }
+  m = htsmsg_binary_deserialize0(data + 4, l, buf);
+  if (m == NULL)
+    return -1;
+  *len = l + 4;
+  *msg = m;
+  return 0;
+}
 
 /*
  *
@@ -227,7 +257,6 @@ htsmsg_binary_count(htsmsg_t *msg)
   return len;
 }
 
-
 /*
  *
  */
@@ -316,6 +345,26 @@ htsmsg_binary_write(htsmsg_t *msg, uint8_t *ptr)
   }
 }
 
+/*
+ *
+ */
+int
+htsmsg_binary_serialize0(htsmsg_t *msg, void **datap, size_t *lenp, int maxlen)
+{
+  size_t len;
+  uint8_t *data;
+
+  len = htsmsg_binary_count(msg);
+  if(len > maxlen)
+    return -1;
+
+  data = malloc(len);
+
+  htsmsg_binary_write(msg, data);
+  *datap = data;
+  *lenp  = len;
+  return 0;
+}
 
 /*
  *
diff --git a/src/htsmsg_binary.h b/src/htsmsg_binary.h
index 55dc0895e..d7d1eee79 100644
--- a/src/htsmsg_binary.h
+++ b/src/htsmsg_binary.h
@@ -24,8 +24,14 @@
 /**
  * htsmsg_binary_deserialize
  */
-htsmsg_t *htsmsg_binary_deserialize(void *data, size_t len,
-				    const void *buf);
+htsmsg_t *htsmsg_binary_deserialize0(const void *data, size_t len,
+                                     const void *buf);
+
+int htsmsg_binary_deserialize(htsmsg_t **msg, const void *data, size_t *len,
+                              const void *buf);
+
+int htsmsg_binary_serialize0(htsmsg_t *msg, void **datap, size_t *lenp,
+			     int maxlen);
 
 int htsmsg_binary_serialize(htsmsg_t *msg, void **datap, size_t *lenp,
 			    int maxlen);
diff --git a/src/htsmsg_binary2.c b/src/htsmsg_binary2.c
index 73361550c..3f0c44055 100644
--- a/src/htsmsg_binary2.c
+++ b/src/htsmsg_binary2.c
@@ -49,34 +49,31 @@ static inline uint32_t htsmsg_binary2_get_length(uint8_t const **_p, const uint8
 
 static inline uint8_t *htsmsg_binary2_set_length(uint8_t *p, uint32_t len)
 {
-  if (len == 0) {
-    *p = 0;
-    return p + 1;
-  } else if (len < 0x80) {
+  if (len < 0x80) {
     p[0] = len;
     return p + 1;
   } else if (len < 0x4000) {
     p[0] = 0x80 | (len >> 7);
-    p[1] = len;
+    p[1] = len & 0x7f;
     return p + 2;
   } else if (len < 0x200000) {
     p[0] = 0x80 | (len >> 14);
     p[1] = 0x80 | (len >> 7);
-    p[2] = len;
+    p[2] = len & 0x7f;
     return p + 3;
     p += 2;
   } else if (len < 0x10000000) {
     p[0] = 0x80 | (len >> 21);
     p[1] = 0x80 | (len >> 14);
     p[2] = 0x80 | (len >> 7);
-    p[3] = len;
+    p[3] = len & 0x7f;
     return p + 4;
   } else {
     p[0] = 0x80 | (len >> 28);
     p[1] = 0x80 | (len >> 21);
     p[2] = 0x80 | (len >> 14);
     p[3] = 0x80 | (len >> 7);
-    p[4] = len;
+    p[4] = len & 0x7f;
     return p + 5;
   }
 }
@@ -115,7 +112,7 @@ htsmsg_binary2_des0(htsmsg_t *msg, const uint8_t *buf, uint32_t len)
     buf      = p;
 
     if(len < namelen + datalen)
-      return -1;
+      abort(); // return -1;
 
     nlen = namelen ? namelen + 1 : 0;
     tlen = sizeof(htsmsg_field_t) + nlen;
@@ -124,11 +121,11 @@ htsmsg_binary2_des0(htsmsg_t *msg, const uint8_t *buf, uint32_t len)
     } else if (type == HMF_UUID) {
       tlen += UUID_BIN_SIZE;
       if (datalen != UUID_BIN_SIZE)
-        return -1;
+        abort(); // return -1;
     }
     f = malloc(tlen);
     if (f == NULL)
-      return -1;
+      abort(); // return -1;
 #if ENABLE_SLOW_MEMORYINFO
     f->hmf_edata_size = tlen - sizeof(htsmsg_field_t);
     memoryinfo_alloc(&htsmsg_field_memoryinfo, tlen);
@@ -186,7 +183,7 @@ htsmsg_binary2_des0(htsmsg_t *msg, const uint8_t *buf, uint32_t len)
         memoryinfo_free(&htsmsg_field_memoryinfo, tlen);
 #endif
         free(f);
-        return -1;
+        abort(); // return -1;
       }
       if (i > 0)
         bin = 1;
@@ -201,7 +198,7 @@ htsmsg_binary2_des0(htsmsg_t *msg, const uint8_t *buf, uint32_t len)
       memoryinfo_free(&htsmsg_field_memoryinfo, tlen);
 #endif
       free(f);
-      return -1;
+      abort(); // return -1;
     }
 
     TAILQ_INSERT_TAIL(&msg->hm_fields, f, hmf_link);
@@ -211,13 +208,11 @@ htsmsg_binary2_des0(htsmsg_t *msg, const uint8_t *buf, uint32_t len)
   return len ? -1 : bin;
 }
 
-
-
 /*
  *
  */
 htsmsg_t *
-htsmsg_binary2_deserialize(void *data, size_t len, const void *buf)
+htsmsg_binary2_deserialize0(const void *data, size_t len, const void *buf)
 {
   htsmsg_t *msg;
   int r;
@@ -245,6 +240,40 @@ htsmsg_binary2_deserialize(void *data, size_t len, const void *buf)
   return msg;
 }
 
+/*
+ *
+ */
+int
+htsmsg_binary2_deserialize
+  (htsmsg_t **msg, const void *data, size_t *len, const void *buf)
+{
+  htsmsg_t *m;
+  const uint8_t *p;
+  uint32_t l, l2;
+  size_t len2;
+
+  len2 = *len;
+  *msg = NULL;
+  *len = 0;
+  if (len2 != (len2 & 0xffffffff) || len2 == 0) {
+    free((void *)buf);
+    return -1;
+  }
+  p = data;
+  l = htsmsg_binary2_get_length(&p, data + len2);
+  l2 = l + (p - (uint8_t *)data);
+  if (l2 > len2) {
+    free((void *)buf);
+    return -1;
+  }
+  m = htsmsg_binary2_deserialize0(p, l, buf);
+  if (m == NULL)
+    return -1;
+  *msg = m;
+  *len = l2;
+  return 0;
+}
+
 /*
  *
  */
@@ -285,7 +314,6 @@ htsmsg_binary2_field_length(htsmsg_field_t *f)
   }
 }
 
-
 /*
  *
  */
@@ -304,7 +332,6 @@ htsmsg_binary2_count(htsmsg_t *msg)
   return len;
 }
 
-
 /*
  *
  */
@@ -367,12 +394,35 @@ htsmsg_binary2_write(htsmsg_t *msg, uint8_t *ptr)
   }
 }
 
+/*
+ *
+ */
+int
+htsmsg_binary2_serialize0
+  (htsmsg_t *msg, void **datap, size_t *lenp, size_t maxlen)
+{
+  uint32_t len;
+  uint8_t *data;
+
+  len = htsmsg_binary2_count(msg);
+  if((size_t)len > maxlen)
+    return -1;
+
+  data = malloc(len);
+
+  htsmsg_binary2_write(msg, data);
+
+  *datap = data;
+  *lenp  = len;
+  return 0;
+}
 
 /*
  *
  */
 int
-htsmsg_binary2_serialize(htsmsg_t *msg, void **datap, size_t *lenp, size_t maxlen)
+htsmsg_binary2_serialize
+  (htsmsg_t *msg, void **datap, size_t *lenp, size_t maxlen)
 {
   uint32_t len, llen;
   uint8_t *data, *p;
diff --git a/src/htsmsg_binary2.h b/src/htsmsg_binary2.h
index c32ca6c13..e41eeb7ee 100644
--- a/src/htsmsg_binary2.h
+++ b/src/htsmsg_binary2.h
@@ -21,8 +21,14 @@
 
 #include "htsmsg.h"
 
-htsmsg_t *htsmsg_binary2_deserialize(void *data, size_t len,
-                                     const void *buf);
+htsmsg_t *htsmsg_binary2_deserialize0(const void *data, size_t len,
+                                      const void *buf);
+
+int htsmsg_binary2_deserialize(htsmsg_t **msg, const void *data, size_t *len,
+                               const void *buf);
+
+int htsmsg_binary2_serialize0(htsmsg_t *msg, void **datap, size_t *lenp,
+			      size_t maxlen);
 
 int htsmsg_binary2_serialize(htsmsg_t *msg, void **datap, size_t *lenp,
 			     size_t maxlen);
diff --git a/src/htsp_server.c b/src/htsp_server.c
index fbdb8d9d3..0af1bf744 100644
--- a/src/htsp_server.c
+++ b/src/htsp_server.c
@@ -3169,7 +3169,7 @@ htsp_read_message(htsp_connection_t *htsp, htsmsg_t **mp, int timeout)
 
   /* buf will be tied to the message (on success) */
   /* bellow fcn calls free(buf) (on failure) */
-  *mp = htsmsg_binary_deserialize(buf, len, buf);
+  *mp = htsmsg_binary_deserialize0(buf, len, buf);
   if(*mp == NULL)
     return EBADMSG;
 
diff --git a/src/settings.c b/src/settings.c
index fdf567bf8..b027dff51 100644
--- a/src/settings.c
+++ b/src/settings.c
@@ -29,6 +29,7 @@
 
 #include "htsmsg.h"
 #include "htsmsg_binary.h"
+#include "htsmsg_binary2.h"
 #include "htsmsg_json.h"
 #include "settings.h"
 #include "tvheadend.h"
@@ -182,9 +183,9 @@ hts_settings_save(htsmsg_t *record, const char *pathfmt, ...)
 #if ENABLE_ZLIB
     void *msgdata = NULL;
     size_t msglen;
-    r = htsmsg_binary_serialize(record, &msgdata, &msglen, 2*1024*1024);
+    r = htsmsg_binary2_serialize0(record, &msgdata, &msglen, 2*1024*1024);
     if (!r && msglen >= 4) {
-      r = tvh_gzip_deflate_fd_header(fd, msgdata + 4, msglen - 4, NULL, 3);
+      r = tvh_gzip_deflate_fd_header(fd, msgdata, msglen, NULL, 3, "01");
       if (r)
         ok = 0;
     } else {
@@ -233,7 +234,8 @@ hts_settings_load_one(const char *filename)
 
   /* Decode */
   if(n == size) {
-    if (size > 12 && memcmp(mem, "\xff\xffGZIP00", 8) == 0) {
+    if (size > 12 && memcmp(mem, "\xff\xffGZIP0", 7) == 0 &&
+        (mem[7] == '0' || mem[7] == '1')) {
 #if ENABLE_ZLIB
       uint32_t orig = (mem[8] << 24) | (mem[9] << 16) | (mem[10] << 8) | mem[11];
       if (orig > 10*1024*1024U) {
@@ -242,7 +244,11 @@ hts_settings_load_one(const char *filename)
       } else if (orig > 0) {
         uint8_t *unpacked = tvh_gzip_inflate((uint8_t *)mem + 12, size - 12, orig);
         if (unpacked) {
-          r = htsmsg_binary_deserialize(unpacked, orig, NULL);
+          if (mem[7] == '1') {
+            r = htsmsg_binary2_deserialize0(unpacked, orig, NULL);
+          } else {
+            r = htsmsg_binary_deserialize0(unpacked, orig, NULL);
+          }
           free(unpacked);
         }
       }
diff --git a/src/tvheadend.h b/src/tvheadend.h
index 2e3eb04db..6f7274afb 100644
--- a/src/tvheadend.h
+++ b/src/tvheadend.h
@@ -776,7 +776,7 @@ char *regexp_escape ( const char *str );
 uint8_t *tvh_gzip_inflate ( const uint8_t *data, size_t size, size_t orig );
 uint8_t *tvh_gzip_deflate ( const uint8_t *data, size_t orig, size_t *size );
 int      tvh_gzip_deflate_fd ( int fd, const uint8_t *data, size_t orig, size_t *size, int speed );
-int      tvh_gzip_deflate_fd_header ( int fd, const uint8_t *data, size_t orig, size_t *size, int speed );
+int      tvh_gzip_deflate_fd_header ( int fd, const uint8_t *data, size_t orig, size_t *size, int speed , const char *signature);
 #endif
 
 /* URL decoding */
diff --git a/src/zlib.c b/src/zlib.c
index 2df7b09cb..9d2a5b614 100644
--- a/src/zlib.c
+++ b/src/zlib.c
@@ -154,9 +154,10 @@ int tvh_gzip_deflate_fd ( int fd, const uint8_t *data, size_t orig, size_t *size
 }
 
 int tvh_gzip_deflate_fd_header ( int fd, const uint8_t *data, size_t orig,
-                                 size_t *deflated_size, int speed )
+                                 size_t *deflated_size, int speed,
+                                 const char *signature )
 {
-  uint8_t data2[4];
+  uint8_t data2[6];
   size_t size = 0;
   int r;
 
@@ -169,13 +170,15 @@ int tvh_gzip_deflate_fd_header ( int fd, const uint8_t *data, size_t orig,
     if (r || size > UINT_MAX)
       return 1;
   }
-  r = lseek(fd, 8, SEEK_SET) != (off_t)8;
+  r = lseek(fd, 6, SEEK_SET) != (off_t)6;
   if (r)
     return 1;
   if (deflated_size) *deflated_size = size + 12;
-  data2[0] = (orig >> 24) & 0xff;
-  data2[1] = (orig >> 16) & 0xff;
-  data2[2] = (orig >> 8) & 0xff;
-  data2[3] = (orig & 0xff);
-  return tvh_write(fd, data2, 4);
+  data2[0] = signature ? signature[0] : '0';
+  data2[1] = signature ? signature[1] : '0';
+  data2[2] = (orig >> 24) & 0xff;
+  data2[3] = (orig >> 16) & 0xff;
+  data2[4] = (orig >> 8) & 0xff;
+  data2[5] = (orig & 0xff);
+  return tvh_write(fd, data2, 6);
 }