hwdb/ids_parser.py

   1 #!/usr/bin/env python3
   2
   3 import re
   4 import sys
   5 from pyparsing import (Word, White, Literal, Regex,
   6                        LineEnd, SkipTo,
   7                        ZeroOrMore, OneOrMore, Combine, Optional, Suppress,
   8                        stringEnd, pythonStyleComment)
   9
  10 EOL = LineEnd().suppress()
  11 NUM1 = Word('0123456789abcdefABCDEF', exact=1)
  12 NUM2 = Word('0123456789abcdefABCDEF', exact=2)
  13 NUM3 = Word('0123456789abcdefABCDEF', exact=3)
  14 NUM4 = Word('0123456789abcdefABCDEF', exact=4)
  15 NUM6 = Word('0123456789abcdefABCDEF', exact=6)
  16 TAB = White('\t', exact=1).suppress()
  17 COMMENTLINE = pythonStyleComment + EOL
  18 EMPTYLINE = LineEnd()
  19 text_eol = lambda name: Regex(r'[^\n]+')(name) + EOL
  20
  21 def klass_grammar():
  22     klass_line = Literal('C ').suppress() + NUM2('klass') + text_eol('text')
  23     subclass_line = TAB + NUM2('subclass') + text_eol('text')
  24     protocol_line = TAB + TAB + NUM2('protocol') + text_eol('name')
  25     subclass = (subclass_line('SUBCLASS') -
  26                 ZeroOrMore(protocol_line('PROTOCOLS*')
  27                            ^ COMMENTLINE.suppress()))
  28     klass = (klass_line('KLASS') -
  29              ZeroOrMore(subclass('SUBCLASSES*')
  30                         ^ COMMENTLINE.suppress()))
  31     return klass
  32
  33 def usb_ids_grammar():
  34     vendor_line = NUM4('vendor') + text_eol('text')
  35     device_line = TAB + NUM4('device') + text_eol('text')
  36     vendor = (vendor_line('VENDOR') +
  37              ZeroOrMore(device_line('VENDOR_DEV*') ^ COMMENTLINE.suppress()))
  38
  39     klass = klass_grammar()
  40
  41     other_line = (Literal('AT ') ^ Literal('HID ') ^ Literal('R ')
  42                   ^ Literal('PHY ') ^ Literal('BIAS ') ^ Literal('HUT ')
  43                   ^ Literal('L ') ^ Literal('VT ') ^ Literal('HCC ')) + text_eol('text')
  44     other_group = (other_line - ZeroOrMore(TAB + text_eol('text')))
  45
  46     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  47     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  48                         ^ other_group.suppress() ^ commentgroup) + stringEnd()
  49
  50     grammar.parseWithTabs()
  51     return grammar
  52
  53 def pci_ids_grammar():
  54     vendor_line = NUM4('vendor') + text_eol('text')
  55     device_line = TAB + NUM4('device') + text_eol('text')
  56     subvendor_line = TAB + TAB + NUM4('a') + White(' ') + NUM4('b') + text_eol('name')
  57
  58     device = (device_line('DEVICE') +
  59               ZeroOrMore(subvendor_line('SUBVENDORS*') ^ COMMENTLINE.suppress()))
  60     vendor = (vendor_line('VENDOR') +
  61               ZeroOrMore(device('DEVICES*') ^ COMMENTLINE.suppress()))
  62
  63     klass = klass_grammar()
  64
  65     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  66     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  67                         ^ commentgroup) + stringEnd()
  68
  69     grammar.parseWithTabs()
  70     return grammar
  71
  72 def sdio_ids_grammar():
  73     vendor_line = NUM4('vendor') + text_eol('text')
  74     device_line = TAB + NUM4('device') + text_eol('text')
  75     vendor = (vendor_line('VENDOR') +
  76               ZeroOrMore(device_line('DEVICES*') ^ COMMENTLINE.suppress()))
  77
  78     klass = klass_grammar()
  79
  80     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  81     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*') ^ commentgroup) + stringEnd()
  82
  83     grammar.parseWithTabs()
  84     return grammar
  85
  86 def oui_grammar(type):
  87     prefix_line = (Combine(NUM2 - Suppress('-') - NUM2 - Suppress('-') - NUM2)('prefix')
  88                    - Literal('(hex)') -  text_eol('text'))
  89     if type == 'small':
  90         vendor_line = (NUM3('start') - '000-' - NUM3('end') - 'FFF'
  91                        - Literal('(base 16)') - text_eol('text2'))
  92     elif type == 'medium':
  93         vendor_line = (NUM1('start') - '00000-' - NUM1('end') - 'FFFFF'
  94                        - Literal('(base 16)') - text_eol('text2'))
  95     else:
  96         assert type == 'large'
  97         vendor_line = (NUM6('start')
  98                        - Literal('(base 16)') - text_eol('text2'))
  99
 100     extra_line = TAB - TAB - TAB - TAB - SkipTo(EOL)
 101     vendor = prefix_line + vendor_line + ZeroOrMore(extra_line) + Optional(EMPTYLINE)
 102
 103     grammar = (Literal('OUI') + text_eol('header')
 104                + text_eol('header') + text_eol('header') + EMPTYLINE
 105                + OneOrMore(vendor('VENDORS*')) + stringEnd())
 106
 107     grammar.parseWithTabs()
 108     return grammar
 109
 110
 111 def header(file, *sources):
 112     print('''\
 113 # This file is part of systemd.
 114 #
 115 # Data imported from:{}{}'''.format(' ' if len(sources) == 1 else '\n#   ',
 116                                     '\n#   '.join(sources)),
 117           file=file)
 118
 119 def add_item(items, key, value):
 120     if key in items:
 121         print(f'Ignoring duplicate entry: {key} = "{items[key]}", "{value}"')
 122     else:
 123         items[key] = value
 124
 125 def usb_vendor_model(p):
 126     items = {}
 127
 128     for vendor_group in p.VENDORS:
 129         vendor = vendor_group.VENDOR.vendor.upper()
 130         text = vendor_group.VENDOR.text.strip()
 131         add_item(items, (vendor,), text)
 132
 133         for vendor_dev in vendor_group.VENDOR_DEV:
 134             device = vendor_dev.device.upper()
 135             text = vendor_dev.text.strip()
 136             add_item(items, (vendor, device), text)
 137
 138     with open('20-usb-vendor-model.hwdb', 'wt') as out:
 139         header(out, 'http://www.linux-usb.org/usb.ids')
 140
 141         for key in sorted(items):
 142             if len(key) == 1:
 143                 p, n = 'usb:v{}*', 'VENDOR'
 144             else:
 145                 p, n = 'usb:v{}p{}*', 'MODEL',
 146             print('', p.format(*key),
 147                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 148
 149     print(f'Wrote {out.name}')
 150
 151 def usb_classes(p):
 152     items = {}
 153
 154     for klass_group in p.CLASSES:
 155         klass = klass_group.KLASS.klass.upper()
 156         text = klass_group.KLASS.text.strip()
 157
 158         if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 159             add_item(items, (klass,), text)
 160
 161         for subclass_group in klass_group.SUBCLASSES:
 162             subclass = subclass_group.subclass.upper()
 163             text = subclass_group.text.strip()
 164             if subclass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 165                 add_item(items, (klass, subclass), text)
 166
 167             for protocol_group in subclass_group.PROTOCOLS:
 168                 protocol = protocol_group.protocol.upper()
 169                 text = protocol_group.name.strip()
 170                 if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 171                     add_item(items, (klass, subclass, protocol), text)
 172
 173     with open('20-usb-classes.hwdb', 'wt') as out:
 174         header(out, 'http://www.linux-usb.org/usb.ids')
 175
 176         for key in sorted(items):
 177             if len(key) == 1:
 178                 p, n = 'usb:v*p*d*dc{}*', 'CLASS'
 179             elif len(key) == 2:
 180                 p, n = 'usb:v*p*d*dc{}dsc{}*', 'SUBCLASS'
 181             else:
 182                 p, n = 'usb:v*p*d*dc{}dsc{}dp{}*', 'PROTOCOL'
 183             print('', p.format(*key),
 184                   f' ID_USB_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 185
 186     print(f'Wrote {out.name}')
 187
 188 def pci_vendor_model(p):
 189     items = {}
 190
 191     for vendor_group in p.VENDORS:
 192         vendor = vendor_group.VENDOR.vendor.upper()
 193         text = vendor_group.VENDOR.text.strip()
 194         add_item(items, (vendor,), text)
 195
 196         for device_group in vendor_group.DEVICES:
 197             device = device_group.device.upper()
 198             text = device_group.text.strip()
 199             add_item(items, (vendor, device), text)
 200
 201             for subvendor_group in device_group.SUBVENDORS:
 202                 sub_vendor = subvendor_group.a.upper()
 203                 sub_model = subvendor_group.b.upper()
 204                 sub_text = subvendor_group.name.strip()
 205                 if sub_text.startswith(text):
 206                     sub_text = sub_text[len(text):].lstrip()
 207                 if sub_text:
 208                     sub_text = f' ({sub_text})'
 209                 add_item(items, (vendor, device, sub_vendor, sub_model), text + sub_text)
 210
 211     with open('20-pci-vendor-model.hwdb', 'wt') as out:
 212         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 213
 214         for key in sorted(items):
 215             if len(key) == 1:
 216                 p, n = 'pci:v0000{}*', 'VENDOR'
 217             elif len(key) == 2:
 218                 p, n = 'pci:v0000{}d0000{}*', 'MODEL'
 219             else:
 220                 p, n = 'pci:v0000{}d0000{}sv0000{}sd0000{}*', 'MODEL'
 221             print('', p.format(*key),
 222                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 223
 224     print(f'Wrote {out.name}')
 225
 226 def pci_classes(p):
 227     items = {}
 228
 229     for klass_group in p.CLASSES:
 230         klass = klass_group.KLASS.klass.upper()
 231         text = klass_group.KLASS.text.strip()
 232         add_item(items, (klass,), text)
 233
 234         for subclass_group in klass_group.SUBCLASSES:
 235             subclass = subclass_group.subclass.upper()
 236             text = subclass_group.text.strip()
 237             add_item(items, (klass, subclass), text)
 238
 239             for protocol_group in subclass_group.PROTOCOLS:
 240                 protocol = protocol_group.protocol.upper()
 241                 text = protocol_group.name.strip()
 242                 add_item(items, (klass, subclass, protocol), text)
 243
 244     with open('20-pci-classes.hwdb', 'wt') as out:
 245         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 246
 247         for key in sorted(items):
 248             if len(key) == 1:
 249                 p, n = 'pci:v*d*sv*sd*bc{}*', 'CLASS'
 250             elif len(key) == 2:
 251                 p, n = 'pci:v*d*sv*sd*bc{}sc{}*', 'SUBCLASS'
 252             else:
 253                 p, n = 'pci:v*d*sv*sd*bc{}sc{}i{}*', 'INTERFACE'
 254             print('', p.format(*key),
 255                   f' ID_PCI_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 256
 257     print(f'Wrote {out.name}')
 258
 259 def sdio_vendor_model(p):
 260     items = {}
 261
 262     for vendor_group in p.VENDORS:
 263         vendor = vendor_group.VENDOR.vendor.upper()
 264         text = vendor_group.VENDOR.text.strip()
 265         add_item(items, (vendor,), text)
 266
 267         for device_group in vendor_group.DEVICES:
 268             device = device_group.device.upper()
 269             text = device_group.text.strip()
 270             add_item(items, (vendor, device), text)
 271
 272     with open('20-sdio-vendor-model.hwdb', 'wt') as out:
 273         header(out, 'hwdb/sdio.ids')
 274
 275         for key in sorted(items):
 276             if len(key) == 1:
 277                 p, n = 'sdio:c*v{}*', 'VENDOR'
 278             else:
 279                 p, n = 'sdio:c*v{}d{}*', 'MODEL'
 280             print('', p.format(*key),
 281                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 282
 283     print(f'Wrote {out.name}')
 284
 285 def sdio_classes(p):
 286     items = {}
 287
 288     for klass_group in p.CLASSES:
 289         klass = klass_group.KLASS.klass.upper()
 290         text = klass_group.KLASS.text.strip()
 291         add_item(items, klass, text)
 292
 293     with open('20-sdio-classes.hwdb', 'wt') as out:
 294         header(out, 'hwdb/sdio.ids')
 295
 296         for klass in sorted(items):
 297             print(f'',
 298                   f'sdio:c{klass}v*d*',
 299                   f' ID_SDIO_CLASS_FROM_DATABASE={items[klass]}', sep='\n', file=out)
 300
 301     print(f'Wrote {out.name}')
 302
 303 # MAC Address Block Large/Medium/Small
 304 # Large  MA-L 24/24 bit (OUI)
 305 # Medium MA-M 28/20 bit (OUI prefix owned by IEEE)
 306 # Small  MA-S 36/12 bit (OUI prefix owned by IEEE)
 307 def oui(p1, p2, p3):
 308     prefixes = set()
 309     items = {}
 310
 311     for p, check in ((p1, False), (p2, False), (p3, True)):
 312         for vendor_group in p.VENDORS:
 313             prefix = vendor_group.prefix.upper()
 314             if check:
 315                 if prefix in prefixes:
 316                     continue
 317             else:
 318                 prefixes.add(prefix)
 319             start = vendor_group.start.upper()
 320             end = vendor_group.end.upper()
 321
 322             if end and start != end:
 323                 print(f'{prefix:} {start} != {end}', file=sys.stderr)
 324             text = vendor_group.text.strip()
 325
 326             key = prefix + start if end else prefix
 327             add_item(items, key, text)
 328
 329     with open('20-OUI.hwdb', 'wt') as out:
 330         header(out,
 331                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
 332                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
 333                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt')
 334
 335         for pattern in sorted(items):
 336             print(f'',
 337                   f'OUI:{pattern}*',
 338                   f' ID_OUI_FROM_DATABASE={items[pattern]}', sep='\n', file=out)
 339
 340     print(f'Wrote {out.name}')
 341
 342 if __name__ == '__main__':
 343     args = sys.argv[1:]
 344
 345     if not args or 'usb' in args:
 346         p = usb_ids_grammar().parseFile(open('usb.ids', errors='replace'))
 347         usb_vendor_model(p)
 348         usb_classes(p)
 349
 350     if not args or 'pci' in args:
 351         p = pci_ids_grammar().parseFile(open('pci.ids', errors='replace'))
 352         pci_vendor_model(p)
 353         pci_classes(p)
 354
 355     if not args or 'sdio' in args:
 356         p = pci_ids_grammar().parseFile(open('sdio.ids', errors='replace'))
 357         sdio_vendor_model(p)
 358         sdio_classes(p)
 359
 360     if not args or 'oui' in args:
 361         p = oui_grammar('small').parseFile(open('ma-small.txt'))
 362         p2 = oui_grammar('medium').parseFile(open('ma-medium.txt'))
 363         p3 = oui_grammar('large').parseFile(open('ma-large.txt'))
 364         oui(p, p2, p3)