From: Jaroslav Kysela Date: Thu, 5 Nov 2015 15:57:27 +0000 (+0100) Subject: added more robust and universal m3u parser (used by IPTV) X-Git-Tag: v4.2.1~1664 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=190bdf74eecc02797d8ea006c8859809df9f62e7;p=thirdparty%2Ftvheadend.git added more robust and universal m3u parser (used by IPTV) --- diff --git a/Makefile b/Makefile index 7545fc2ce..74462a770 100644 --- a/Makefile +++ b/Makefile @@ -182,6 +182,7 @@ SRCS-1 = \ src/htsmsg_xml.c \ src/misc/dbl.c \ src/misc/json.c \ + src/misc/m3u.c \ src/settings.c \ src/htsbuf.c \ src/trap.c \ diff --git a/src/bouquet.c b/src/bouquet.c index 36b77ee92..aceec50b6 100644 --- a/src/bouquet.c +++ b/src/bouquet.c @@ -37,7 +37,7 @@ static void bouquet_remove_service(bouquet_t *bq, service_t *s); static uint64_t bouquet_get_channel_number0(bouquet_t *bq, service_t *t); static void bouquet_download_trigger(bouquet_t *bq); static void bouquet_download_stop(void *aux); -static int bouquet_download_process(void *aux, const char *last_url, char *data, size_t len); +static int bouquet_download_process(void *aux, const char *last_url, const char *host_url, char *data, size_t len); /** * @@ -1099,7 +1099,8 @@ next: } static int -bouquet_download_process(void *aux, const char *last_url, char *data, size_t len) +bouquet_download_process(void *aux, const char *last_url, const char *host_url, + char *data, size_t len) { bouquet_download_t *bqd = aux; while (*data) { diff --git a/src/download.c b/src/download.c index b77c3477f..3b37fcc36 100644 --- a/src/download.c +++ b/src/download.c @@ -65,7 +65,7 @@ download_file(download_t *dn, const char *filename) last_url = strrchr(filename, '/'); if (last_url) last_url++; - res = dn->process(dn->aux, last_url, data, off); + res = dn->process(dn->aux, last_url, NULL, data, off); } else { res = -1; } @@ -95,7 +95,9 @@ download_fetch_complete(http_client_t *hc) { download_t *dn = hc->hc_aux; char *last_url = NULL; - url_t u; + char host_url[512]; + char *s, *p; + url_t u, u2; switch (hc->hc_code) { case HTTP_STATUS_MOVED: @@ -111,6 +113,25 @@ download_fetch_complete(http_client_t *hc) if (last_url) last_url++; } + if ((p = http_arg_get(&hc->hc_args, "Host")) != NULL) { + snprintf(host_url, sizeof(host_url), "%s://%s", + hc->hc_ssl ? "https" : "http", p); + } else if (dn->url) { + s = strdupa(dn->url); + if ((p = strchr(s, '/')) != NULL) { + p++; + if (*p == '/') + p++; + if ((p = strchr(p, '/')) != NULL) + *p = '\0'; + } + urlinit(&u2); + if (!urlparse(s, &u2)) + snprintf(host_url, sizeof(host_url), "%s", s); + urlreset(&u2); + } else { + host_url[0] = '\0'; + } pthread_mutex_lock(&global_lock); @@ -118,7 +139,7 @@ download_fetch_complete(http_client_t *hc) goto out; if (hc->hc_code == HTTP_STATUS_OK && hc->hc_result == 0 && hc->hc_data_size > 0) - dn->process(dn->aux, last_url, hc->hc_data, hc->hc_data_size); + dn->process(dn->aux, last_url, host_url, hc->hc_data, hc->hc_data_size); else tvherror(dn->log, "unable to fetch data from url [%d-%d/%zd]", hc->hc_code, hc->hc_result, hc->hc_data_size); diff --git a/src/download.h b/src/download.h index 0f1b817e5..a050f15a3 100644 --- a/src/download.h +++ b/src/download.h @@ -27,7 +27,8 @@ typedef struct download { char *url; void *aux; int ssl_peer_verify; - int (*process)(void *aux, const char *last_url, char *data, size_t len); + int (*process)(void *aux, const char *last_url, const char *host_url, + char *data, size_t len); void (*stop)(void *aux); /* internal members */ http_client_t *http_client; diff --git a/src/input/mpegts/iptv/iptv_auto.c b/src/input/mpegts/iptv/iptv_auto.c index d820292c3..ff86d7992 100644 --- a/src/input/mpegts/iptv/iptv_auto.c +++ b/src/input/mpegts/iptv/iptv_auto.c @@ -22,7 +22,7 @@ #include "iptv_private.h" #include "channels.h" #include "download.h" -#include "intlconv.h" +#include "misc/m3u.h" #include #include @@ -33,70 +33,15 @@ typedef struct auto_private { gtimer_t in_auto_timer; } auto_private_t; -/* - * - */ -static char *get_m3u_str(char *data, char **res) -{ - char *p = data, first = *data; - - if (first == '"' || first == '\'') { - data++; p++; - while (*data && *data != first) - data++; - } else { - while (*data && *data != ',' && *data > ' ') - data++; - } - if (*data) { - *data = '\0'; - data++; - } - *res = data; - return p; -} - -/* - * - */ -static char *until_eol(char *d) -{ - while (*d && *d != '\r' && *d != '\n') d++; - if (*d) { *d = '\0'; d++; } - while (*d && (*d == '\r' || *d == '\n')) d++; - return d; -} - -/* - * - */ -static int replace_string(char **s, const char *n, const char *charset_id) -{ - char *c; - - if (charset_id && n) { - c = intlconv_to_utf8safestr(charset_id, n, strlen(n)*2); - } else - c = n ? strdup(n) : NULL; - if (strcmp(*s ?: "", n ?: "") == 0) { - free(c); - return 0; - } - free(*s); - *s = c; - return 1; -} - /* * */ static void iptv_auto_network_process_m3u_item(iptv_network_t *in, - const char *last_url, const char *charset_id, + const char *last_url, const http_arg_list_t *remove_args, - const char *url, const char *name, - const char *logo, const char *epgid, - int64_t chnum, int *total, int *count) + int64_t chnum, htsmsg_t *item, + int *total, int *count) { htsmsg_t *conf; mpegts_mux_t *mm; @@ -108,7 +53,10 @@ iptv_auto_network_process_m3u_item(iptv_network_t *in, htsbuf_queue_t q; int delim; size_t l; - char url2[512], custom[512], name2[128], buf[32], *n, *x = NULL, *y; + const char *url, *name, *logo, *epgid; + char url2[512], custom[512], name2[128], buf[32], *n, *y; + + url = htsmsg_get_str(item, "m3u-url"); if (url == NULL || (strncmp(url, "file://", 7) && @@ -128,9 +76,16 @@ iptv_auto_network_process_m3u_item(iptv_network_t *in, chnum += (int64_t)*total * CHANNEL_SPLIT; } + name = htsmsg_get_str(item, "m3u-name"); if (name == NULL) name = ""; + logo = htsmsg_get_str(item, "tvg-logo"); + if (logo == NULL) + logo = htsmsg_get_str(item, "logo"); + + epgid = htsmsg_get_str(item, "tvg-id"); + urlinit(&u); custom[0] = '\0'; @@ -199,12 +154,7 @@ iptv_auto_network_process_m3u_item(iptv_network_t *in, } skip_url: - x = NULL; if (last_url) { - if (charset_id) { - x = intlconv_to_utf8safestr(charset_id, name, strlen(name)*2); - name = x; - } snprintf(n = name2, sizeof(name2), "%s - %s", last_url, name); } else { n = (char *)name; @@ -234,7 +184,11 @@ skip_url: im->mm_iptv_icon = logo ? strdup(logo) : NULL; change = 1; } - change |= replace_string(&im->mm_iptv_epgid, epgid, charset_id); + if (strcmp(im->mm_iptv_epgid ?: "", epgid ?: "")) { + free(im->mm_iptv_epgid); + im->mm_iptv_epgid = epgid ? strdup(epgid) : NULL; + change = 1; + } if (strcmp(im->mm_iptv_hdr ?: "", custom)) { free(im->mm_iptv_hdr); im->mm_iptv_hdr = strdup(custom); @@ -277,7 +231,6 @@ skip_url: } end: - free(x); urlreset(&u); } @@ -287,74 +240,40 @@ end: static int iptv_auto_network_process_m3u(iptv_network_t *in, char *data, const char *last_url, + const char *host_url, http_arg_list_t *remove_args, int64_t chnum) { - char *url, *name = NULL, *logo = NULL, *epgid = NULL; - char *charset_id = intlconv_charset_id(in->in_ctx_charset, 0, 1); int total = 0, count = 0; - - while (*data && *data != '\n') data++; - if (*data) data++; - while (*data) { - if (strncmp(data, "#EXTINF:", 8) == 0) { - name = NULL; - logo = NULL; - epgid = NULL; - data += 8; - while (*data > ' ' && *data != ',') data++; - while (1) { - while (*data && *data <= ' ') data++; - if (*data == ',') break; - if (strncmp(data, "tvg-logo=", 9) == 0) { - logo = get_m3u_str(data + 9, &data); continue; - } else if (strncmp(data, "tvg-id=", 7) == 0) { - epgid = get_m3u_str(data + 7, &data); continue; - } else if (strncmp(data, "logo=", 5) == 0) { - logo = get_m3u_str(data + 5, &data); continue; - } else { - data++; - while (*data && *data != ',' && *data != '=') data++; - if (*data == '=') - get_m3u_str(data + 1, &data); - } - } - if (*data == ',') { - data++; - while (*data && *data <= ' ') data++; - if (*data) - name = data; - } - data = until_eol(data); - continue; - } else if (strncmp(data, "#EXT", 4) == 0) { - data += 4; - while (*data && *data != '\n') data++; - if (*data) data++; - continue; - } - while (*data && *data <= ' ') data++; - url = data; - data = until_eol(data); - if (*url && *url > ' ') - iptv_auto_network_process_m3u_item(in, last_url, charset_id, - remove_args, url, name, - logo, epgid, - chnum, &total, &count); + htsmsg_t *m, *items, *item; + htsmsg_field_t *f; + int ret = 0; + + m = parse_m3u(data, in->in_ctx_charset, host_url); + items = htsmsg_get_list(m, "items"); + HTSMSG_FOREACH(f, items) { + if ((item = htsmsg_field_get_map(f)) == NULL) continue; + iptv_auto_network_process_m3u_item(in, last_url, + remove_args, chnum, + item, + &total, &count); + } - + htsmsg_destroy(m); if (total == 0) - return -1; - tvhinfo("iptv", "m3u parse: %d new mux(es) in network '%s' (total %d)", - count, in->mn_network_name, total); - return 0; + ret = -1; + else + tvhinfo("iptv", "m3u parse: %d new mux(es) in network '%s' (total %d)", + count, in->mn_network_name, total); + return ret; } /* * */ static int -iptv_auto_network_process(void *aux, const char *last_url, char *data, size_t len) +iptv_auto_network_process(void *aux, const char *last_url, + const char *host_url, char *data, size_t len) { auto_private_t *ap = aux; iptv_network_t *in = ap->in_network; @@ -381,7 +300,8 @@ iptv_auto_network_process(void *aux, const char *last_url, char *data, size_t le while (*data && *data <= ' ') data++; if (!strncmp(data, "#EXTM3U", 7)) - r = iptv_auto_network_process_m3u(in, data, last_url, &remove_args, in->in_channel_number); + r = iptv_auto_network_process_m3u(in, data, last_url, host_url, + &remove_args, in->in_channel_number); http_arg_flush(&remove_args); diff --git a/src/input/mpegts/iptv/iptv_http.c b/src/input/mpegts/iptv/iptv_http.c index e7900923b..616e67eed 100644 --- a/src/input/mpegts/iptv/iptv_http.c +++ b/src/input/mpegts/iptv/iptv_http.c @@ -20,38 +20,7 @@ #include "tvheadend.h" #include "iptv_private.h" #include "http.h" - -/* - * M3U parser - */ -static char *until_eol(char *d) -{ - while (*d && *d != '\r' && *d != '\n') d++; - if (*d) { *d = '\0'; d++; } - while (*d && (*d == '\r' || *d == '\n')) d++; - return d; -} - -static char * -iptv_http_m3u(char *data) -{ - char *url; - - while (*data && *data != '\n') data++; - if (*data) data++; - while (*data) { - if (strncmp(data, "#EXT", 4) == 0) { - data = until_eol(data); - continue; - } - while (*data && *data <= ' ') data++; - url = data; - data = until_eol(data); - if (*url && *url > ' ') - return strdup(url); - } - return NULL; -} +#include "misc/m3u.h" /* * Connected @@ -133,56 +102,68 @@ iptv_http_complete ( http_client_t *hc ) { iptv_mux_t *im = hc->hc_aux; - char *url, *url2, *s, *p; + const char *url, *host_url; + char *p; + htsmsg_t *m, *items, *item; + htsmsg_field_t *f; url_t u; + size_t l; int r; if (im->im_m3u_header) { im->im_m3u_header = 0; + sbuf_append(&im->mm_iptv_buffer, "", 1); - url = iptv_http_m3u((char *)im->mm_iptv_buffer.sb_data); + + if ((p = http_arg_get(&hc->hc_args, "Host")) != NULL) { + l = strlen(p) + 16; + host_url = alloca(l); + snprintf((char *)host_url, l, "%s://%s", hc->hc_ssl ? "https" : "http", p); + } else if (im->mm_iptv_url_raw) { + host_url = strdupa(im->mm_iptv_url_raw); + if ((p = strchr(host_url, '/')) != NULL) { + p++; + if (*p == '/') + p++; + if ((p = strchr(p, '/')) != NULL) + *p = '\0'; + } + urlinit(&u); + r = urlparse(host_url, &u); + urlreset(&u); + if (r) + goto end; + } else { + host_url = NULL; + } + + m = parse_m3u((char *)im->mm_iptv_buffer.sb_data, NULL, host_url); + items = htsmsg_get_list(m, "items"); + url = NULL; + HTSMSG_FOREACH(f, items) { + if ((item = htsmsg_field_get_map(f)) == NULL) continue; + url = htsmsg_get_str(items, "m3u-url"); + if (url && url[0]) break; + } tvhtrace("iptv", "m3u url: '%s'", url); - sbuf_reset(&im->mm_iptv_buffer, IPTV_BUF_SIZE); if (url == NULL) { tvherror("iptv", "m3u contents parsing failed"); - return 0; + goto fin; } urlinit(&u); - if (url[0] == '/') { - url2 = malloc(512); - url2[0] = '\0'; - if ((p = http_arg_get(&hc->hc_args, "Host")) != NULL) { - snprintf(url2, 512, "%s://%s%s", - hc->hc_ssl ? "https" : "http", p, url); - } else if (im->mm_iptv_url_raw) { - s = strdupa(im->mm_iptv_url_raw); - if ((p = strchr(s, '/')) != NULL) { - p++; - if (*p == '/') - p++; - if ((p = strchr(p, '/')) != NULL) - *p = '\0'; - } - if (urlparse(s, &u)) - goto invalid; - snprintf(url2, 512, "%s%s", s, url); - } - free(url); - url = url2; - urlinit(&u); - } if (!urlparse(url, &u)) { hc->hc_keepalive = 0; r = http_client_simple_reconnect(hc, &u, HTTP_VERSION_1_1); if (r < 0) tvherror("iptv", "cannot reopen http client: %d'", r); } else { -invalid: tvherror("iptv", "m3u url invalid '%s'", url); } urlreset(&u); - free(url); - return 0; +fin: + htsmsg_destroy(m); +end: + sbuf_reset(&im->mm_iptv_buffer, IPTV_BUF_SIZE); } return 0; } diff --git a/src/misc/m3u.c b/src/misc/m3u.c new file mode 100644 index 000000000..d567240cb --- /dev/null +++ b/src/misc/m3u.c @@ -0,0 +1,251 @@ +/* + * M3U parser + * Copyright (C) 2015 Jaroslav Kysela + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "tvheadend.h" +#include "htsmsg.h" +#include "intlconv.h" +#include "m3u.h" + +/* + * + */ +static char *get_m3u_str(char *data, char **res, int *last) +{ + char *p = data, first = *data; + + if (first == '"' || first == '\'') { + data++; p++; + while (*data && *data != first && *data != '\n' && *data != '\r') + data++; + } else { + while (*data && *data != ',' && *data > ' ') + data++; + } + *last = '\0'; + if (*data) { + *last = *data; + *data = '\0'; + data++; + } + *res = data; + return p; +} + +/* + * + */ +static void get_m3u_str_post(char **data, int delim) +{ + if (delim == '\n' || delim == '\r') { + (*data)--; + **data = delim; + } +} + +/* + * + */ +static char *until_eol(char *d) +{ + while (*d && *d != '\r' && *d != '\n') d++; + if (*d) { *d = '\0'; d++; } + while (*d && (*d == '\r' || *d == '\n')) d++; + return d; +} + +/* + * + */ +static const char *relative_url + (char *buf, size_t buflen, const char *rel, const char *url) +{ + if (url == NULL) + return rel; + if (strncmp(url, "file://", 7) && + strncmp(url, "pipe://", 7) && + strncmp(url, "http://", 7) && + strncmp(url, "https://", 8) && + strncmp(url, "rtsp://", 7) && + strncmp(url, "rtsps://", 8) && + strncmp(url, "udp://", 6) && + strncmp(url, "rtp://", 6)) + return rel; + + snprintf(buf, buflen, "%s%s", url, rel); + return buf; +} + +/* + * Note: text in data pointer is not preserved (must be read/write) + */ +htsmsg_t *parse_m3u + (char *data, const char *charset, const char *url) +{ + char *p, *x, *y; + char *charset_id = intlconv_charset_id(charset, 0, 1); + const char *multi_name; + int delim; + htsmsg_t *m = htsmsg_create_map(); + htsmsg_t *item = NULL, *l = NULL, *t; + char buf[512]; + + while (*data && *data <= ' ') data++; + p = data; + while (*data && *data != '\n') data++; + if (*data) { *data = '\0'; data++; } + if (strcmp(p, "#EXTM3U")) { + htsmsg_add_msg(m, "items", htsmsg_create_list()); + return m; + } + while (*data) { + if (strncmp(data, "#EXTINF:", 8) == 0) { + if (item == NULL) + item = htsmsg_create_map(); + data += 8; + p = data; + if (*data == '-') data++; + while (*data >= '0' && *data <= '9') data++; + delim = *data; + *data = '\0'; + htsmsg_add_s64(item, "m3u-duration", strtoll(p, NULL, 10)); + *data = delim; + while (*data > ' ' && *data != ',') data++; + while (delim && delim != ',' && delim != '\n' && delim != '\r') { + while (*data && *data <= ' ') data++; + if (*data == '\0' || *data == ',') break; + p = data++; + while (*data && *data != ',' && *data != '=') data++; + if (*data == '=') { + *data = '\0'; + x = get_m3u_str(data + 1, &data, &delim); + if (*p && *x) { + y = intlconv_to_utf8safestr(charset_id, x, strlen(x)*2); + htsmsg_add_str(item, p, y); + free(y); + } + get_m3u_str_post(&data, delim); + } + } + p = NULL; + if (*data == ',') { + data++; + while (*data && *data <= ' ' && *data != '\n' && *data != '\r') data++; + if (*data) + p = data; + } + data = until_eol(data); + if (p && *p) { + y = intlconv_to_utf8safestr(charset_id, p, strlen(p)*2); + htsmsg_add_str(item, "m3u-name", y); + free(y); + } + continue; + } else if (strncmp(data, "#EXT-X-VERSION", 14) == 0) { + htsmsg_add_s64(m, "version", strtoll(data + 14, NULL, 10)); + data = until_eol(data + 14); + continue; + } else if (strncmp(data, "#EXT-X-MEDIA-SEQUENCE:", 22) == 0) { + htsmsg_add_s64(m, "media-sequence", strtoll(data + 22, NULL, 10)); + data = until_eol(data + 22); + continue; + } else if (strncmp(data, "#EXT-X-TARGETDURATION:", 22) == 0) { + htsmsg_add_s64(m, "targetduration", strtoll(data + 22, NULL, 10)); + data = until_eol(data + 22); + continue; + } else if (strncmp(data, "#EXT-X-STREAM-INF:", 18) == 0) { + data += 18; + multi_name = "stream-inf"; +multi: + t = htsmsg_create_map(); + delim = 0; + while (*data && delim != '\n' && delim != '\r') { + p = data; + while (*data && *data >= ' ' && *data != '=') data++; + if (*data == '=') { + *data = '\0'; + x = get_m3u_str(data + 1, &data, &delim); + if (*x && *p) + htsmsg_add_str(t, p, x); + get_m3u_str_post(&data, delim); + } else { + while (*data && *data <= ' ' && *data != '\n' && *data != '\r') data++; + if (*data != ',') break; + } + } + if (item == NULL) + item = htsmsg_create_map(); + htsmsg_add_msg(item, multi_name, t); + data = until_eol(data); + continue; + } else if (strncmp(data, "#EXT-X-MEDIA:", 13) == 0) { + data += 13; + multi_name = "x-media"; + goto multi; + } else if (strncmp(data, "#EXT", 4) == 0) { + data = until_eol(data + 4); + continue; + } + while (*data && *data <= ' ') data++; + p = data; + data = until_eol(data); + if (*p && *p > ' ') { + if (item == NULL) + item = htsmsg_create_map(); + if (strncmp(p, "http://", 7) == 0 || + strncmp(p, "https://", 8) == 0) { + delim = 0; + x = p; + while (*x && *x != ' ' && *x != '|') x++; + if (*x) { delim = *x; *x = '\0'; x++; } + t = NULL; + while (*x) { + while (*x && *x <= ' ') x++; + y = x; + while (*x && *x != delim && *x != '&') x++; + if (*x) { *x = '\0'; x++; } + if (*y) { + if (t == NULL) + t = htsmsg_create_list(); + htsmsg_add_str(t, NULL, y); + } + } + if (t) + htsmsg_add_msg(item, "m3u-http-headers", t); + } + + htsmsg_add_str(item, "m3u-url", + p[0] == '/' ? relative_url(buf, sizeof(buf), p, url) : p); + } else if (item) { + htsmsg_destroy(item); + free(item); + item = NULL; + } + + if (item) { + if (l == NULL) + l = htsmsg_create_list(); + htsmsg_add_msg(l, NULL, item); + item = NULL; + } + } + + if (l == NULL) + l = htsmsg_create_list(); + htsmsg_add_msg(m, "items", l); + return m; +} diff --git a/src/misc/m3u.h b/src/misc/m3u.h new file mode 100644 index 000000000..8192138df --- /dev/null +++ b/src/misc/m3u.h @@ -0,0 +1,3 @@ +#pragma once + +htsmsg_t *parse_m3u(char *data, const char *charset, const char *url);