[PR-153] Provide automatic charset overrides for certain services

Certain services on some networks are transmitted with incorrect
charset encodings. The user has the ability to manually override
these, but this provide initial defaults for known bad services.

Mostly this relates to Polish DVB-S providers at this time.

diff --git a/Makefile b/Makefile
index 76004c65a79308404f3925a4847ce9e4f91f7008..2517db4c9f3079c336af1370f071d83a195eb7c9 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -116,6 +116,7 @@ SRCS += src/epggrab/module.c\
   src/epggrab/module/xmltv.c\
   src/epggrab/module/eit.c \
   src/epggrab/module/opentv.c \
+  src/epggrab/module/encoding.c \
   src/epggrab/support/freesat_huffman.c \
 
 SRCS += src/plumbing/tsfix.c \

diff --git a/data/conf/charset b/data/conf/charset
new file mode 100644 (file)
index 0000000..0824f48
--- /dev/null
+++ b/data/conf/charset
@@ -0,0 +1,415 @@
+[
+       {
+               "tsid": 200,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 13878,
+               "description": "ESP2 NE Polish"
+       },
+       {
+               "tsid": 200,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 13865,
+               "description": "Hotbird Eutelsat (Eurosport)"
+       },
+       {
+               "tsid": 300,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 15206,
+               "description": "Animal Planet HD"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13020,
+               "description": "Hotbird 13.0 Cyfra+ Canal+ HD Polska"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13022,
+               "description": "Hotbird 13.0 Cyfra+ Canal+ Sport HD Polska"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13023,
+               "description": "Hotbird 13.0 Cyfra+ National Geographic HD Polska"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13025,
+               "description": "Hotbird 13.0 Cyfra+ Filmbox HD"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13026,
+               "description": "Hotbird 13.0 Cyfra+ AXN Spin HD"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13027,
+               "description": "Hotbird 13.0 Cyfra+ TVN 7 HD"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13070,
+               "description": "Hotbird 13.0 Cyfra+ ESP HD PL"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13081,
+               "description": "EUROSPORT HD"
+       },
+       {
+               "tsid": 400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 13082,
+               "description": "EUROSPORT HD"
+       },
+       {
+               "tsid": 1000,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 4314,
+               "description": "nSport"
+       },
+       {
+               "tsid": 1000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Grupa ITI"
+       },
+       {
+               "tsid": 1100,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfra+"
+       },
+       {
+               "tsid": 1500,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 5110,
+               "description": "TV Info"
+       },
+       {
+               "tsid": 1500,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 5108,
+               "description": "Fox Polska"
+       },
+       {
+               "tsid": 1500,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 5102,
+               "description": "TVS"
+       },
+       {
+               "tsid": 1500,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 5122,
+               "description": "Animal Planet Poland"
+       },
+       {
+               "tsid": 1500,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfra+"
+       },
+       {
+               "tsid": 1600,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 EskaTV, TVN"
+       },
+       {
+               "tsid": 1800,
+               "onid": 200,
+               "charset": "ISO6937",
+               "sid": 3623,
+               "description": "Hotbird 13.0 Polo TV"
+       },
+       {
+               "tsid": 7400,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfrowy Polsat"
+       },
+       {
+               "tsid": 7800,
+               "onid": 113,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfrowy Polsat"
+       },
+       {
+               "tsid": 7900,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 10510,
+               "description": "TV6 Polska"
+       },
+       {
+               "tsid": 7900,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 10508,
+               "description": "Hotbird 13.0 Cyfrowy Polsat Jim Jam"
+       },
+       {
+               "tsid": 7900,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 10521,
+               "description": "SuperStacja"
+       },
+       {
+               "tsid": 7900,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 10520,
+               "description": "Travel Channel"
+       },
+       {
+               "tsid": 7900,
+               "onid": 113,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfrowy Polsat"
+       },
+       {
+               "tsid": 8100,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14911,
+               "description": "UNIVERSAL POL"
+       },
+       {
+               "tsid": 8100,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14910,
+               "description": "Sport Klub POL"
+       },
+       {
+               "tsid": 8100,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 Eutelsat (Universal)"
+       },
+       {
+               "tsid": 11000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfra+"
+       },
+       {
+               "tsid": 11400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 15716,
+               "description": "Discovery Channel Polska"
+       },
+       {
+               "tsid": 11400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 15701,
+               "description": "Discovery Science Polska"
+       },
+       {
+               "tsid": 11400,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 15703,
+               "description": "Discovery World Polska"
+       },
+       {
+               "tsid": 11400,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfra+"
+       },
+       {
+               "tsid": 11600,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 3514,
+               "description": "Disney XD Polska"
+       },
+       {
+               "tsid": 11600,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 BBC HD, ITI"
+       },
+       {
+               "tsid": 11900,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfra+"
+       },
+       {
+               "tsid": 12200,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Disney Channel Polska, TCM and other"
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 24,
+               "description": "CC Family."
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 6,
+               "description": "Viacom Blink!"
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 22,
+               "description": "MTV Polska."
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 30,
+               "description": "Comedy Central Polska."
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 25,
+               "description": "VIVA Polska."
+       },
+       {
+               "tsid": 12800,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 Viacom ... MTV / VH1 Polska"
+       },
+       {
+               "tsid": 13000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14612,
+               "description": "Investigation Discovery Polska"
+       },
+       {
+               "tsid": 13000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14613,
+               "description": "TLC Polska"
+       },
+       {
+               "tsid": 13000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14602,
+               "description": "BBC Entertainment Poland"
+       },
+       {
+               "tsid": 13000,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 14603,
+               "description": "BBC Knowledge Poland"
+       },
+       {
+               "tsid": 13000,
+               "onid": 318,
+               "charset": "ISO8859-2",
+               "sid": 0,
+               "description": "Hotbird 13.0 BBC Polska, TLC and other"
+       },
+       {
+               "tsid": 13100,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 7322,
+               "description": "TV5 Monde Europe"
+       },
+       {
+               "tsid": 13100,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 7324,
+               "description": "Hotbird 13.0 Crime and Investigation"
+       },
+       {
+               "tsid": 13200,
+               "onid": 113,
+               "charset": "ISO88959-2",
+               "sid": 13104,
+               "description": "Polsat Sport"
+       },
+       {
+               "tsid": 13200,
+               "onid": 113,
+               "charset": "ISO88959-2",
+               "sid": 13105,
+               "description": "Polsat Sport Extra"
+       },
+       {
+               "tsid": 13200,
+               "onid": 113,
+               "charset": "ISO88959-2",
+               "sid": 13107,
+               "description": "TV Biznes"
+       },
+       {
+               "tsid": 13200,
+               "onid": 113,
+               "charset": "ISO6937",
+               "sid": 0,
+               "description": "Hotbird 13.0 Cyfrowy Polsat"
+       },
+       {
+               "tsid": 15700,
+               "onid": 318,
+               "charset": "ISO6937",
+               "sid": 10626,
+               "description": "Hotbird 13.0 Disco TV"
+       }
+]

diff --git a/src/dvb/dvb_support.c b/src/dvb/dvb_support.c
index 73b7c4b78e3779831c909ab00a095c9fbee9a371..f628239a41660dd937844351bc9a7677582d0601 100644 (file)
--- a/src/dvb/dvb_support.c
+++ b/src/dvb/dvb_support.c
@@ -204,7 +204,7 @@ static inline size_t dvb_convert(int conv,
 int
 dvb_get_string(char *dst, size_t dstlen, const uint8_t *src, size_t srclen, char *dvb_charset, dvb_string_conv_t *conv)
 {
-  int ic;
+  int ic, pl_workaround = 0;
   size_t len, outlen;
   int i;
 
@@ -220,13 +220,22 @@ dvb_get_string(char *dst, size_t dstlen, const uint8_t *src, size_t srclen, char
     conv++;
   }
 
+  // check for polish channels encoding workaround
+  if (dvb_charset && (strcmp(dvb_charset, "pl_workaround") == 0)) {
+    pl_workaround = 1;
+    dvb_charset = NULL;
+  }
+
   // automatic charset detection
   switch(src[0]) {
   case 0:
     return -1;
 
   case 0x01 ... 0x0b:
-    ic = convert_iso_8859[src[0] + 4];
+    if (pl_workaround && (src[0] + 4) == 5)
+      ic = convert_iso6937;
+    else
+      ic = convert_iso_8859[src[0] + 4];
     src++; srclen--;
     break;
 
@@ -251,7 +260,10 @@ dvb_get_string(char *dst, size_t dstlen, const uint8_t *src, size_t srclen, char
     return -1;
 
   default:
-    ic = convert_iso6937;
+    if (pl_workaround)
+      ic = convert_iso_8859[2];
+    else
+      ic = convert_iso6937;
     break;
   }
 

diff --git a/src/epggrab/module/eit.c b/src/epggrab/module/eit.c
index a994bf41d402d4dddeeadc4b5417fe054bd089fb..108923844df72a88590863ed5c682d58565f2eab 100644 (file)
--- a/src/epggrab/module/eit.c
+++ b/src/epggrab/module/eit.c
@@ -26,6 +26,7 @@
 #include "epg.h"
 #include "epggrab.h"
 #include "epggrab/private.h"
+#include "encoding.h"
 
 /* ************************************************************************
  * Status handling
@@ -508,6 +509,7 @@ static int _eit_process_event
   epg_episode_t *ee;
   epg_serieslink_t *es;
   eit_event_t ev;
+  static char pl_workaround[] = "pl_workaround";
 
   if ( len < 12 ) return -1;
 
@@ -539,6 +541,20 @@ static int _eit_process_event
   /* Process tags */
   memset(&ev, 0, sizeof(ev));
   ev.default_charset = svc->s_dvb_charset;
+
+  /* Test channels which needs polish workaround */
+  encoding_t *enc;
+  LIST_FOREACH(enc, &encoding_list, link) {
+    // TODO: also check for original network id (onid)
+    //       currently onid is unavailable in tvh but there are additional cases which has
+    //       to be met, so I believe, that it should not harm Russian ISO-8859-5 channels
+    //       until we've got this information here
+    if (!ev.default_charset && svc->s_dvb_mux_instance->tdmi_transport_stream_id == enc->tsid) {
+      ev.default_charset = pl_workaround;
+      break;
+    }
+  }
+
   while (dllen > 2) {
     int r;
     dtag = ptr[0];

diff --git a/src/epggrab/module/encoding.c b/src/epggrab/module/encoding.c
new file mode 100644 (file)
index 0000000..5a6ffed
--- /dev/null
+++ b/src/epggrab/module/encoding.c
@@ -0,0 +1,71 @@
+/*
+ *  tvheadend, encoding list
+ *  Copyright (C) 2012 Mariusz Białończyk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include "tvheadend.h"
+#include "encoding.h"
+#include "../../settings.h"
+
+/*
+ * Process a file
+ */
+static void _encoding_load_file()
+{
+  htsmsg_t *l, *e;
+  htsmsg_field_t *f;
+
+  encoding_t *enc;
+  unsigned int tsid, onid;
+  int i = 0;
+
+  l = hts_settings_load("encoding");
+  if (l)
+  {
+    HTSMSG_FOREACH(f, l) {
+      if ((e = htsmsg_get_map_by_field(f))) {
+        tsid = onid = 0;
+        htsmsg_get_u32(e, "tsid", &tsid);
+        htsmsg_get_u32(e, "onid", &onid);
+
+        if (tsid == 0 || onid == 0)
+          continue;
+
+        enc = calloc(1, sizeof(encoding_t));
+        if (enc)
+        {
+          enc->tsid = tsid;
+          enc->onid = onid;
+          LIST_INSERT_HEAD(&encoding_list, enc, link);
+          i++;
+        }
+      }
+    };
+    htsmsg_destroy(l);
+  };
+
+  if (i > 0)
+    tvhlog(LOG_INFO, "encoding", "%d entries loaded", i);
+}
+
+/*
+ * Initialise the encoding list
+ */
+void encoding_init ( void )
+{
+  _encoding_load_file();
+}

diff --git a/src/epggrab/module/encoding.h b/src/epggrab/module/encoding.h
new file mode 100644 (file)
index 0000000..a6748ad
--- /dev/null
+++ b/src/epggrab/module/encoding.h
@@ -0,0 +1,32 @@
+/*
+ *  tvheadend, encoding list
+ *  Copyright (C) 2012 Mariusz Białończyk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __TVH_ENCODING_H__
+#define __TVH_ENCODING_H__
+
+typedef struct encoding {
+  LIST_ENTRY(encoding) link;
+ unsigned int tsid;
+ unsigned int onid;
+} encoding_t;
+
+LIST_HEAD(,encoding) encoding_list;
+
+void encoding_init ( void );
+
+#endif /* __TVH_ENCODING_H__ */

diff --git a/src/main.c b/src/main.c
index 66d863fc39fad4c6ab0780c2c0c65eae59a232ca..23e4e64b112a59b04390712b5611791e1471f525 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -60,6 +60,7 @@
 #include "ffdecsa/FFdecsa.h"
 #include "muxes.h"
 #include "config2.h"
+#include "epggrab/module/encoding.h"
 
 int running;
 time_t dispatch_clock;
@@ -433,6 +434,7 @@ main(int argc, char **argv)
 
   epggrab_init();
   epg_init();
+  encoding_init();
 
   dvr_init();