diff --git a/AutoBouquetsMaker/custom/hd_sat_192_sky_deutschland_CustomLCN.xml b/AutoBouquetsMaker/custom/hd_sat_192_sky_deutschland_CustomLCN.xml deleted file mode 100644 index 0789c078..00000000 --- a/AutoBouquetsMaker/custom/hd_sat_192_sky_deutschland_CustomLCN.xml +++ /dev/null @@ -1,119 +0,0 @@ - - yes - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/AutoBouquetsMaker/lib/dvbreader.c b/AutoBouquetsMaker/lib/dvbreader.c index 6adf0aaf..4d1db1ef 100644 --- a/AutoBouquetsMaker/lib/dvbreader.c +++ b/AutoBouquetsMaker/lib/dvbreader.c @@ -1380,6 +1380,59 @@ PyObject *ss_read_fastscan(PyObject *self, PyObject *args) { return ret; } +// Sky Deutschland / Sky Q channel-list carousel (table 0x9E, variable_id 0x0055 on PID 0x08AE). +// One section carries a raw chunk of a larger NDS-ARCHIVE blob; the Python layer reassembles +// all sections of the same version_number and parses the archive. +PyObject *ss_parse_header_skyq(unsigned char *data, int length) +{ + return Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:i}", + "table_id", data[0], + "section_length", length, + "variable_id", (data[3] << 8) | data[4], + "version_number", (data[5] >> 1) & 0x1F, + "current_next_indicator", data[5] & 0x01, + "section_number", data[6], + "last_section_number", data[7]); +} + +PyObject *ss_read_skyq(PyObject *self, PyObject *args) { + PyObject *content = NULL, *header = NULL; + unsigned char buffer[4096], table_id; + int fd; + + if (!PyArg_ParseTuple(args, "ib", &fd, &table_id)) + return Py_None; + + int size = read(fd, buffer, sizeof(buffer)); + if (size < 14) + return Py_None; + + if (buffer[0] != table_id) + return Py_None; + + int section_length = ((buffer[1] & 0x0f) << 8) | buffer[2]; + + if (size != section_length + 3) + return Py_None; + + // payload: bytes 13..section_length-2 inclusive (excludes 8-byte section header, + // 5-byte NDS sub-header and trailing 4-byte CRC32) + int payload_len = section_length - 14; + if (payload_len < 0) + return Py_None; + + header = ss_parse_header_skyq(buffer, section_length); + content = PyBytes_FromStringAndSize((const char *)(buffer + 13), payload_len); + + if (!header || !content) + return Py_None; + + PyObject *ret = Py_BuildValue("{s:O,s:O}", "header", header, "content", content); + Py_DECREF(header); + Py_DECREF(content); + return ret; +} + PyObject *ss_read_nit(PyObject *self, PyObject *args) { PyObject *content = NULL, *header = NULL; unsigned char buffer[4096], table_id_current, table_id_other; @@ -1419,6 +1472,7 @@ static PyMethodDef dvbreaderMethods[] = { { "read_nit", ss_read_nit, METH_VARARGS }, { "read_sdt", ss_read_sdt, METH_VARARGS }, { "read_fastscan", ss_read_fastscan, METH_VARARGS }, + { "read_skyq", ss_read_skyq, METH_VARARGS }, { "read_ts", ss_read_ts, METH_VARARGS }, { NULL, NULL } }; diff --git a/AutoBouquetsMaker/providers/sat_130_ncplus.xml b/AutoBouquetsMaker/providers/sat_130_ncplus.xml index 206534c3..e4eeb40f 100644 --- a/AutoBouquetsMaker/providers/sat_130_ncplus.xml +++ b/AutoBouquetsMaker/providers/sat_130_ncplus.xml @@ -7,7 +7,7 @@ frequency="10719000" symbol_rate="27500000" polarization="1" - fec_inner="5" + fec_inner="4" inversion="2" system="1" modulation="2" @@ -21,7 +21,7 @@
CANAL+
- NC+ + CANAL+ diff --git a/AutoBouquetsMaker/providers/sat_192_sky_deutschland.xml b/AutoBouquetsMaker/providers/sat_192_sky_deutschland.xml index 656f9ea2..b645b55c 100644 --- a/AutoBouquetsMaker/providers/sat_192_sky_deutschland.xml +++ b/AutoBouquetsMaker/providers/sat_192_sky_deutschland.xml @@ -1,7 +1,7 @@ Sky Deutschland dvbs - nolcn + skyde + + Satellit HD + + + + Sky Showcase HD + HISTORY Channel HD +
Entertainment
Sport
@@ -41,6 +54,12 @@ for number in service["numbers"]: if service["service_name"] in blacklist: skip.skip = True +# Sky DE sometimes wraps service names in DVB emphasis bytes (0x86/0x87), +# which defeats startswith() — use a substring check so we catch e.g. +# "†STEST1‡" too. +if "STEST" in service["service_name"]: + skip.skip = True + ]]> diff --git a/AutoBouquetsMaker/src/scanner/dvbscanner.py b/AutoBouquetsMaker/src/scanner/dvbscanner.py index 72c2cc39..acafe420 100644 --- a/AutoBouquetsMaker/src/scanner/dvbscanner.py +++ b/AutoBouquetsMaker/src/scanner/dvbscanner.py @@ -4,13 +4,102 @@ import dvbreader import datetime +import re +import struct import time from Components.config import config +# --- Sky Q / Sky Deutschland NDS-ARCHIVE parser ------------------------------- +# The channel-list carousel on 19.2E (PID 0x08AE, table 0x9E, variable_id 0x0055) +# delivers an NDS-ARCHIVE container holding a nested NDS-ARCHIVE of per-bouquet +# .txt files. Each bouquet file maps LCN to (ONID, TSID, SID) using lines of +# the form "==dvb://..;;". + +_SKYQ_LINE_RE = re.compile(rb"^\s*(\d+)=(\d+)=dvb://([0-9A-Fa-f]+)\.([0-9A-Fa-f]+)\.([0-9A-Fa-f]+)") +_SKYQ_NDS_MAGIC = b"NDS-ARCHIVE\x00" + + +def _skyq_parse_archive(data, magic_off, archive_base): + """Return list of (name, bytes) for one NDS-ARCHIVE container. + + magic_off absolute offset of the 'NDS-ARCHIVE\\0' magic inside data + archive_base origin for the file-offset field in each entry; 0 for the + outer archive, magic_off - 6 empirically for nested + archives (a 6-byte pre-magic header we do not decode). + """ + if data[magic_off:magic_off + 12] != _SKYQ_NDS_MAGIC: + raise ValueError("NDS-ARCHIVE magic not found at 0x%x" % magic_off) + off = magic_off + 12 + # 18-byte post-magic header: FF FF | 32-bit unknown | 16-bit unknown | uint32 nfiles | 6 pad + nfiles = struct.unpack(">I", data[off + 8:off + 12])[0] + off += 18 + entries = [] + for _ in range(nfiles): + if off + 23 > len(data): + break + entry_len = data[off] + file_off = struct.unpack(">I", data[off + 5:off + 9])[0] + file_size = struct.unpack(">I", data[off + 9:off + 13])[0] + fname_len = data[off + 22] + fname = data[off + 23:off + 23 + fname_len].rstrip(b"\x00").decode("latin-1", "replace") + start = archive_base + file_off + entries.append((fname, data[start:start + file_size])) + off += entry_len + return entries + + +def skyq_extract_bouquet_file(archive_data, bouquet_file): + """Locate a given '.txt' filename inside the Sky Q NDS-ARCHIVE blob. + Raises KeyError listing the available channel-list files on miss.""" + magic_positions = [m.start() for m in re.finditer(_SKYQ_NDS_MAGIC, archive_data)] + if len(magic_positions) < 2: + raise RuntimeError("Malformed NDS-ARCHIVE: expected at least 2 magics, got %d" % len(magic_positions)) + root = _skyq_parse_archive(archive_data, magic_positions[0], archive_base=0) + nested = None + for name, body in root: + if name.endswith("sky_de_configuration.nsa"): + nested = body + break + if nested is None: + raise RuntimeError("sky_de_configuration.nsa not found in NDS-ARCHIVE") + nested_magic = nested.find(_SKYQ_NDS_MAGIC) + if nested_magic < 0: + raise RuntimeError("Nested NDS-ARCHIVE magic not found") + inner = _skyq_parse_archive(nested, nested_magic, archive_base=nested_magic - 6) + for name, body in inner: + if name == bouquet_file: + return body + available = sorted(n for n, _ in inner if "channels-" in n and not n.endswith(".headers")) + raise KeyError("bouquet_file %r not in archive; available: %s" % (bouquet_file, available)) + + +def skyq_parse_channel_list(body): + """Parse a bouquet .txt into {(onid, tsid, sid): logical_channel_number}. + LCN 0 entries (data/interactive slots) are skipped.""" + out = {} + for line in body.split(b"\r\n"): + m = _SKYQ_LINE_RE.match(line) + if not m: + continue + lcn = int(m.group(1)) + if lcn == 0: + continue + onid = int(m.group(3), 16) + tsid = int(m.group(4), 16) + sid = int(m.group(5), 16) + key = (onid, tsid, sid) + if key not in out: + out[key] = lcn + return out + +# --- end Sky Q NDS-ARCHIVE parser --------------------------------------------- + + class DvbScanner(): TIMEOUT_SEC = 20 SDT_TIMEOUT = 20 + SKYQ_TIMEOUT = 60 VIDEO_ALLOWED_TYPES = [1, 4, 5, 17, 22, 24, 25, 27, 31, 135] AUDIO_ALLOWED_TYPES = [2, 10] @@ -34,6 +123,9 @@ def __init__(self): self.bat_table_id = 0x4a self.fastscan_pid = 0x00 self.fastscan_table_id = 0x00 + self.skyq_pid = 0x08ae + self.skyq_table_id = 0x9e + self.skyq_variable_id = 0x0055 self.ignore_visible_service_flag = 0 self.extra_debug = config.autobouquetsmaker.extra_debug.value self.namespace_complete = not (config.usage.subnetwork.value if hasattr(config.usage, "subnetwork") else True) # config.usage.subnetwork not available in all images @@ -103,6 +195,18 @@ def setFastscanTableId(self, value): self.fastscan_table_id = value print("[ABM-DvbScanner] Fastscan table id: 0x%x" % self.fastscan_table_id, file=log) + def setSkyqPid(self, value): + self.skyq_pid = value + print("[ABM-DvbScanner] SkyQ pid: 0x%x" % self.skyq_pid, file=log) + + def setSkyqTableId(self, value): + self.skyq_table_id = value + print("[ABM-DvbScanner] SkyQ table id: 0x%x" % self.skyq_table_id, file=log) + + def setSkyqVariableId(self, value): + self.skyq_variable_id = value + print("[ABM-DvbScanner] SkyQ variable id: 0x%x" % self.skyq_variable_id, file=log) + def setVisibleServiceFlagIgnore(self, value): self.ignore_visible_service_flag = value print("[ABM-DvbScanner] Ignore visible service flag: %d" % self.ignore_visible_service_flag, file=log) @@ -727,6 +831,117 @@ def updateAndReadServicesLCN(self, transponders, servicehacks, TSID_ONID_list, l "radio": radio_services } + def readSkyQArchive(self): + """Reassemble the Sky Q / Sky Deutschland channel-list NDS-ARCHIVE. + Returns raw archive bytes, or None on timeout / incomplete delivery.""" + print("[ABM-DvbScanner] Reading Sky Q channel-list carousel (PID 0x%x, table 0x%x, var_id 0x%x)..." + % (self.skyq_pid, self.skyq_table_id, self.skyq_variable_id), file=log) + + fd = dvbreader.open(self.demuxer_device, self.skyq_pid, self.skyq_table_id, 0xff, self.frontend) + if fd < 0: + print("[ABM-DvbScanner] Cannot open the demuxer", file=log) + return None + + version = None + expected = None + received = {} + + timeout = datetime.datetime.now() + timeout += datetime.timedelta(0, self.SKYQ_TIMEOUT) + while True: + if datetime.datetime.now() > timeout: + print("[ABM-DvbScanner] Timed out reading Sky Q archive", file=log) + break + + section = dvbreader.read_skyq(fd, self.skyq_table_id) + if section is None: + time.sleep(0.1) + continue + + header = section["header"] + if header["variable_id"] != self.skyq_variable_id: + continue + + ver = header["version_number"] + if version is None: + version = ver + expected = header["last_section_number"] + 1 + elif ver != version: + # carousel updated mid-read - start over + version = ver + expected = header["last_section_number"] + 1 + received = {} + + sn = header["section_number"] + if sn in received: + continue + received[sn] = section["content"] + if len(received) == expected: + break + + dvbreader.close(fd) + + if not expected or len(received) != expected: + print("[ABM-DvbScanner] Incomplete Sky Q archive (%d/%s sections)" + % (len(received), expected), file=log) + return None + + blob = b"".join(received[n] for n in range(expected)) + print("[ABM-DvbScanner] Sky Q archive v%d assembled, %d bytes, %d sections" + % (version, len(blob), expected), file=log) + return blob + + def updateAndReadServicesSKYDE(self, bouquet_file, transponders, servicehacks, provider_config, lcn_overrides=None): + print("[ABM-DvbScanner] Reading services (SKYDE, bouquet_file=%s)..." % bouquet_file, file=log) + + blob = self.readSkyQArchive() + if not blob: + return {"video": {}, "radio": {}} + + try: + body = skyq_extract_bouquet_file(blob, bouquet_file) + except (KeyError, RuntimeError, ValueError) as e: + print("[ABM-DvbScanner] Sky Q archive: %s" % e, file=log) + return {"video": {}, "radio": {}} + + lcn_map = skyq_parse_channel_list(body) + print("[ABM-DvbScanner] Sky Q channel list %r: %d entries" % (bouquet_file, len(lcn_map)), file=log) + + # Provider-level manual overrides — for services Sky broadcasts but does + # not list in the carousel (e.g. after a transponder re-allocation that + # Sky has not yet propagated to the channel-list .txt). Applied only + # when the carousel itself has no entry for the same (ONID, TSID, SID). + if lcn_overrides: + injected = 0 + for ov in lcn_overrides: + key = (ov["onid"], ov["tsid"], ov["sid"]) + if key not in lcn_map: + lcn_map[key] = ov["lcn"] + injected += 1 + if injected: + print("[ABM-DvbScanner] Injected %d provider LCN override(s) for services missing from carousel" % injected, file=log) + if not lcn_map: + return {"video": {}, "radio": {}} + + # Convert into the shape the shared LCN path expects + TSID_ONID_list = [] + logical_channel_number_dict = {} + for (onid, tsid, sid), lcn in lcn_map.items(): + key = "%x:%x" % (tsid, onid) + if key not in TSID_ONID_list: + TSID_ONID_list.append(key) + servicekey = "%x:%x:%x" % (tsid, onid, sid) + logical_channel_number_dict[servicekey] = { + "transport_stream_id": tsid, + "original_network_id": onid, + "service_id": sid, + "logical_channel_number": lcn, + } + + return self.updateAndReadServicesLCN( + transponders, servicehacks, TSID_ONID_list, + logical_channel_number_dict, {}, "skyde", bouquet_file, provider_config) + def updateAndReadServicesFastscan(self, transponders, servicehacks, logical_channel_number_dict, provider_config): print("[ABM-DvbScanner] Reading services (fastscan)...", file=log) diff --git a/AutoBouquetsMaker/src/scanner/manager.py b/AutoBouquetsMaker/src/scanner/manager.py index dc1bc915..08d6b29a 100644 --- a/AutoBouquetsMaker/src/scanner/manager.py +++ b/AutoBouquetsMaker/src/scanner/manager.py @@ -185,7 +185,7 @@ def read(self, provider_config, providers, motorised): # providers = Providers().read() if provider_key in providers: - if bouquet_key in providers[provider_key]["bouquets"] or providers[provider_key]["protocol"] != "sky": + if bouquet_key in providers[provider_key]["bouquets"] or providers[provider_key]["protocol"] not in ("sky", "skyde"): scanner = DvbScanner() scanner.setAdapter(self.adapter) scanner.setDemuxer(self.demuxer) @@ -261,7 +261,28 @@ def read(self, provider_config, providers, motorised): self.serviceVideoRead += len(list(self.services[provider_key]["video"].keys())) self.serviceAudioRead += len(list(self.services[provider_key]["radio"].keys())) + elif providers[provider_key]["protocol"] == "skyde": + scanner.setSdtPid(providers[provider_key]["transponder"]["sdt_pid"]) + scanner.setSdtCurrentTableId(providers[provider_key]["transponder"]["sdt_current_table_id"]) + scanner.setSdtOtherTableId(providers[provider_key]["transponder"]["sdt_other_table_id"]) + scanner.setSkyqPid(providers[provider_key]["transponder"]["skyq_pid"]) + scanner.setSkyqTableId(providers[provider_key]["transponder"]["skyq_table_id"]) + scanner.setSkyqVariableId(providers[provider_key]["transponder"]["skyq_variable_id"]) + + scanner.updateTransponders(self.transponders, False) + bouquet = providers[provider_key]["bouquets"][bouquet_key] + self.services[provider_key] = scanner.updateAndReadServicesSKYDE( + bouquet["bouquet_file"], self.transponders, + providers[provider_key]["servicehacks"], provider_config, + lcn_overrides=providers[provider_key].get("lcn_overrides", [])) + + ret = len(list(self.services[provider_key]["video"].keys())) > 0 or len(list(self.services[provider_key]["radio"].keys())) > 0 + + self.serviceVideoRead += len(list(self.services[provider_key]["video"].keys())) + self.serviceAudioRead += len(list(self.services[provider_key]["radio"].keys())) + elif providers[provider_key]["protocol"] == "freesat": + scanner.setSdtPid(providers[provider_key]["transponder"]["sdt_pid"]) scanner.setSdtCurrentTableId(providers[provider_key]["transponder"]["sdt_current_table_id"]) scanner.setSdtOtherTableId(providers[provider_key]["transponder"]["sdt_other_table_id"]) diff --git a/AutoBouquetsMaker/src/scanner/providers.py b/AutoBouquetsMaker/src/scanner/providers.py index 5cdbc3ef..e3f101c2 100644 --- a/AutoBouquetsMaker/src/scanner/providers.py +++ b/AutoBouquetsMaker/src/scanner/providers.py @@ -18,7 +18,7 @@ class Providers(): - VALID_PROTOCOLS = ("fastscan", "freesat", "lcn", "lcn2", "lcnbat", "lcnbat2", "nolcn", "sky", "vmuk", "vmuk2") + VALID_PROTOCOLS = ("fastscan", "freesat", "lcn", "lcn2", "lcnbat", "lcnbat2", "nolcn", "sky", "skyde", "vmuk", "vmuk2") PROVIDERS_DIR = os.path.dirname(__file__) + "/../providers" USER_PROVIDERS_DIR = os.path.realpath(resolveFilename(SCOPE_CONFIG)) + "/AutoBouquetsMaker/providers" @@ -100,6 +100,7 @@ def read(self): provider["ignore_visible_service_flag"] = 0 provider["custom_list"] = False provider["show_fta_options"] = True + provider["lcn_overrides"] = [] if dom.documentElement.nodeType == dom.documentElement.ELEMENT_NODE and dom.documentElement.tagName == "provider": for node in dom.documentElement.childNodes: if node.nodeType != node.ELEMENT_NODE: @@ -135,6 +136,9 @@ def read(self): transponder["bat_table_id"] = 0x4a transponder["fastscan_pid"] = 0x00 # no default value transponder["fastscan_table_id"] = 0x00 # no default value + transponder["skyq_pid"] = 0x08ae # Sky Q channel-list carousel (PID on 19.2E Sky DE) + transponder["skyq_table_id"] = 0x9e + transponder["skyq_variable_id"] = 0x0055 transponder["system"] = eDVBFrontendParametersSatellite.System_DVB_S transponder["polarization"] = eDVBFrontendParametersSatellite.Polarisation_Horizontal transponder["fec_inner"] = eDVBFrontendParametersSatellite.FEC_Auto @@ -197,12 +201,18 @@ def read(self): transponder["fastscan_pid"] = int(node.attributes.item(i).value, 16) elif node.attributes.item(i).name == "fastscan_table_id": transponder["fastscan_table_id"] = int(node.attributes.item(i).value, 16) + elif node.attributes.item(i).name == "skyq_pid": + transponder["skyq_pid"] = int(node.attributes.item(i).value, 16) + elif node.attributes.item(i).name == "skyq_table_id": + transponder["skyq_table_id"] = int(node.attributes.item(i).value, 16) + elif node.attributes.item(i).name == "skyq_variable_id": + transponder["skyq_variable_id"] = int(node.attributes.item(i).value, 16) elif node.attributes.item(i).name == "onid": transponder["onid"] = int(node.attributes.item(i).value) elif node.attributes.item(i).name == "tsid": transponder["tsid"] = int(node.attributes.item(i).value) - if len(list(transponder.keys())) in (22, 18): + if len(list(transponder.keys())) in (25, 18): provider["transponder"] = transponder elif node.tagName == "bouquettype": @@ -227,12 +237,15 @@ def read(self): elif node2.attributes.item(i).name == "region": # allow region to be a list of values (e.g. so we can accept both SD and HD descriptors) configuration["region"] = list(map(lambda x: int(x.strip(), 16), node2.attributes.item(i).value.split(","))) + elif node2.attributes.item(i).name == "bouquet_file": + # skyde protocol: filename inside the Sky Q NDS-ARCHIVE + configuration["bouquet_file"] = self.encodeNODE(node2.attributes.item(i).value) node2.normalize() if len(node2.childNodes) == 1 and node2.childNodes[0].nodeType == node2.TEXT_NODE: configuration["name"] = self.encodeNODE(node2.childNodes[0].data) - if len(list(configuration.keys())) == 4: + if len(list(configuration.keys())) in (4, 5): provider["bouquets"][configuration["key"]] = configuration elif node.tagName == "dvbcconfigs": @@ -357,6 +370,21 @@ def read(self): if len(list(transponder.keys())) == 8: provider["transponder"] = transponder + elif node.tagName == "lcn_overrides": + provider["lcn_overrides"] = [] + for node2 in node.childNodes: + if node2.nodeType == node2.ELEMENT_NODE and node2.tagName == "override": + ov = {} + for i in list(range(0, node2.attributes.length)): + n = node2.attributes.item(i).name + v = node2.attributes.item(i).value + if n in ("onid", "tsid", "sid"): + ov[n] = int(v, 16 if v.lower().startswith("0x") else 10) + elif n == "lcn": + ov[n] = int(v) + if set(ov.keys()) >= {"onid", "tsid", "sid", "lcn"}: + provider["lcn_overrides"].append(ov) + elif node.tagName == "sections": provider["sections"] = {} for node2 in node.childNodes: