hwdb/ids_parser.py

   1 #!/usr/bin/env python3
   2
   3 import re
   4 import sys
   5 from pyparsing import (Word, White, Literal, Regex,
   6                        LineEnd, SkipTo,
   7                        ZeroOrMore, OneOrMore, Combine, Optional, Suppress,
   8                        stringEnd, pythonStyleComment)
   9
  10 EOL = LineEnd().suppress()
  11 NUM1 = Word('0123456789abcdefABCDEF', exact=1)
  12 NUM2 = Word('0123456789abcdefABCDEF', exact=2)
  13 NUM3 = Word('0123456789abcdefABCDEF', exact=3)
  14 NUM4 = Word('0123456789abcdefABCDEF', exact=4)
  15 NUM6 = Word('0123456789abcdefABCDEF', exact=6)
  16 TAB = White('\t', exact=1).suppress()
  17 COMMENTLINE = pythonStyleComment + EOL
  18 EMPTYLINE = LineEnd()
  19 text_eol = lambda name: Regex(r'[^\n]+')(name) + EOL
  20 # text_eol = lambda name: Word(printables + ' ' + '®üäßçõãİó ×²⁶´‐“\u200E\u200B')(name) + EOL
  21
  22 def klass_grammar():
  23     klass_line = Literal('C ').suppress() + NUM2('klass') + text_eol('text')
  24     subclass_line = TAB + NUM2('subclass') + text_eol('text')
  25     protocol_line = TAB + TAB + NUM2('protocol') + text_eol('name')
  26     subclass = (subclass_line('SUBCLASS') -
  27                 ZeroOrMore(protocol_line('PROTOCOLS*')
  28                            ^ COMMENTLINE.suppress()))
  29     klass = (klass_line('KLASS') -
  30              ZeroOrMore(subclass('SUBCLASSES*')
  31                         ^ COMMENTLINE.suppress()))
  32     return klass
  33
  34 def usb_ids_grammar():
  35     vendor_line = NUM4('vendor') + text_eol('text')
  36     device_line = TAB + NUM4('device') + text_eol('text')
  37     vendor = (vendor_line('VENDOR') +
  38              ZeroOrMore(device_line('VENDOR_DEV*') ^ COMMENTLINE.suppress()))
  39
  40     klass = klass_grammar()
  41
  42     other_line = (Literal('AT ') ^ Literal('HID ') ^ Literal('R ')
  43                   ^ Literal('PHY ') ^ Literal('BIAS ') ^ Literal('HUT ')
  44                   ^ Literal('L ') ^ Literal('VT ') ^ Literal('HCC ')) + text_eol('text')
  45     other_group = (other_line - ZeroOrMore(TAB + text_eol('text')))
  46
  47     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  48     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  49                         ^ other_group.suppress() ^ commentgroup) + stringEnd()
  50
  51     grammar.parseWithTabs()
  52     return grammar
  53
  54 def pci_ids_grammar():
  55     vendor_line = NUM4('vendor') + text_eol('text')
  56     device_line = TAB + NUM4('device') + text_eol('text')
  57     subvendor_line = TAB + TAB + NUM4('a') + White(' ') + NUM4('b') + text_eol('name')
  58
  59     device = (device_line('DEVICE') +
  60               ZeroOrMore(subvendor_line('SUBVENDORS*') ^ COMMENTLINE.suppress()))
  61     vendor = (vendor_line('VENDOR') +
  62               ZeroOrMore(device('DEVICES*') ^ COMMENTLINE.suppress()))
  63
  64     klass = klass_grammar()
  65
  66     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  67     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  68                         ^ commentgroup) + stringEnd()
  69
  70     grammar.parseWithTabs()
  71     return grammar
  72
  73 def sdio_ids_grammar():
  74     vendor_line = NUM4('vendor') + text_eol('text')
  75     device_line = TAB + NUM4('device') + text_eol('text')
  76     vendor = (vendor_line('VENDOR') +
  77               ZeroOrMore(device_line('DEVICES*') ^ COMMENTLINE.suppress()))
  78
  79     klass = klass_grammar()
  80
  81     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  82     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*') ^ commentgroup) + stringEnd()
  83
  84     grammar.parseWithTabs()
  85     return grammar
  86
  87 def oui_grammar(type):
  88     prefix_line = (Combine(NUM2 - Suppress('-') - NUM2 - Suppress('-') - NUM2)('prefix')
  89                    - Literal('(hex)') -  text_eol('text'))
  90     if type == 'small':
  91         vendor_line = (NUM3('start') - '000-' - NUM3('end') - 'FFF'
  92                        - Literal('(base 16)') - text_eol('text2'))
  93     elif type == 'medium':
  94         vendor_line = (NUM1('start') - '00000-' - NUM1('end') - 'FFFFF'
  95                        - Literal('(base 16)') - text_eol('text2'))
  96     else:
  97         assert type == 'large'
  98         vendor_line = (NUM6('start')
  99                        - Literal('(base 16)') - text_eol('text2'))
 100
 101     extra_line = TAB - TAB - TAB - TAB - SkipTo(EOL)
 102     vendor = prefix_line + vendor_line + ZeroOrMore(extra_line) + Optional(EMPTYLINE)
 103
 104     grammar = (Literal('OUI') + text_eol('header')
 105                + text_eol('header') + text_eol('header') + EMPTYLINE
 106                + OneOrMore(vendor('VENDORS*')) + stringEnd())
 107
 108     grammar.parseWithTabs()
 109     return grammar
 110
 111
 112 def header(file, *sources):
 113     print('''\
 114 # This file is part of systemd.
 115 #
 116 # Data imported from:{}{}'''.format(' ' if len(sources) == 1 else '\n#   ',
 117                                     '\n#   '.join(sources)),
 118           file=file)
 119
 120 def add_item(items, key, value):
 121     if key in items:
 122         print(f'Ignoring duplicate entry: {key} = "{items[key]}", "{value}"')
 123     else:
 124         items[key] = value
 125
 126 def usb_vendor_model(p):
 127     items = {}
 128
 129     for vendor_group in p.VENDORS:
 130         vendor = vendor_group.VENDOR.vendor.upper()
 131         text = vendor_group.VENDOR.text.strip()
 132         add_item(items, (vendor,), text)
 133
 134         for vendor_dev in vendor_group.VENDOR_DEV:
 135             device = vendor_dev.device.upper()
 136             text = vendor_dev.text.strip()
 137             add_item(items, (vendor, device), text)
 138
 139     with open('20-usb-vendor-model.hwdb', 'wt') as out:
 140         header(out, 'http://www.linux-usb.org/usb.ids')
 141
 142         for key in sorted(items):
 143             if len(key) == 1:
 144                 p, n = 'usb:v{}*', 'VENDOR'
 145             else:
 146                 p, n = 'usb:v{}p{}*', 'MODEL',
 147             print('', p.format(*key),
 148                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 149
 150     print(f'Wrote {out.name}')
 151
 152 def usb_classes(p):
 153     items = {}
 154
 155     for klass_group in p.CLASSES:
 156         klass = klass_group.KLASS.klass.upper()
 157         text = klass_group.KLASS.text.strip()
 158
 159         if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 160             add_item(items, (klass,), text)
 161
 162         for subclass_group in klass_group.SUBCLASSES:
 163             subclass = subclass_group.subclass.upper()
 164             text = subclass_group.text.strip()
 165             if subclass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 166                 add_item(items, (klass, subclass), text)
 167
 168             for protocol_group in subclass_group.PROTOCOLS:
 169                 protocol = protocol_group.protocol.upper()
 170                 text = protocol_group.name.strip()
 171                 if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 172                     add_item(items, (klass, subclass, protocol), text)
 173
 174     with open('20-usb-classes.hwdb', 'wt') as out:
 175         header(out, 'http://www.linux-usb.org/usb.ids')
 176
 177         for key in sorted(items):
 178             if len(key) == 1:
 179                 p, n = 'usb:v*p*d*dc{}*', 'CLASS'
 180             elif len(key) == 2:
 181                 p, n = 'usb:v*p*d*dc{}dsc{}*', 'SUBCLASS'
 182             else:
 183                 p, n = 'usb:v*p*d*dc{}dsc{}dp{}*', 'PROTOCOL'
 184             print('', p.format(*key),
 185                   f' ID_USB_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 186
 187     print(f'Wrote {out.name}')
 188
 189 def pci_vendor_model(p):
 190     items = {}
 191
 192     for vendor_group in p.VENDORS:
 193         vendor = vendor_group.VENDOR.vendor.upper()
 194         text = vendor_group.VENDOR.text.strip()
 195         add_item(items, (vendor,), text)
 196
 197         for device_group in vendor_group.DEVICES:
 198             device = device_group.device.upper()
 199             text = device_group.text.strip()
 200             add_item(items, (vendor, device), text)
 201
 202             for subvendor_group in device_group.SUBVENDORS:
 203                 sub_vendor = subvendor_group.a.upper()
 204                 sub_model = subvendor_group.b.upper()
 205                 sub_text = subvendor_group.name.strip()
 206                 if sub_text.startswith(text):
 207                     sub_text = sub_text[len(text):].lstrip()
 208                 if sub_text:
 209                     sub_text = f' ({sub_text})'
 210                 add_item(items, (vendor, device, sub_vendor, sub_model), text + sub_text)
 211
 212     with open('20-pci-vendor-model.hwdb', 'wt') as out:
 213         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 214
 215         for key in sorted(items):
 216             if len(key) == 1:
 217                 p, n = 'pci:v0000{}*', 'VENDOR'
 218             elif len(key) == 2:
 219                 p, n = 'pci:v0000{}d0000{}*', 'MODEL'
 220             else:
 221                 p, n = 'pci:v0000{}d0000{}sv0000{}sd0000{}*', 'MODEL'
 222             print('', p.format(*key),
 223                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 224
 225     print(f'Wrote {out.name}')
 226
 227 def pci_classes(p):
 228     items = {}
 229
 230     for klass_group in p.CLASSES:
 231         klass = klass_group.KLASS.klass.upper()
 232         text = klass_group.KLASS.text.strip()
 233         add_item(items, (klass,), text)
 234
 235         for subclass_group in klass_group.SUBCLASSES:
 236             subclass = subclass_group.subclass.upper()
 237             text = subclass_group.text.strip()
 238             add_item(items, (klass, subclass), text)
 239
 240             for protocol_group in subclass_group.PROTOCOLS:
 241                 protocol = protocol_group.protocol.upper()
 242                 text = protocol_group.name.strip()
 243                 add_item(items, (klass, subclass, protocol), text)
 244
 245     with open('20-pci-classes.hwdb', 'wt') as out:
 246         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 247
 248         for key in sorted(items):
 249             if len(key) == 1:
 250                 p, n = 'pci:v*d*sv*sd*bc{}*', 'CLASS'
 251             elif len(key) == 2:
 252                 p, n = 'pci:v*d*sv*sd*bc{}sc{}*', 'SUBCLASS'
 253             else:
 254                 p, n = 'pci:v*d*sv*sd*bc{}sc{}i{}*', 'INTERFACE'
 255             print('', p.format(*key),
 256                   f' ID_PCI_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 257
 258     print(f'Wrote {out.name}')
 259
 260 def sdio_vendor_model(p):
 261     items = {}
 262
 263     for vendor_group in p.VENDORS:
 264         vendor = vendor_group.VENDOR.vendor.upper()
 265         text = vendor_group.VENDOR.text.strip()
 266         add_item(items, (vendor,), text)
 267
 268         for device_group in vendor_group.DEVICES:
 269             device = device_group.device.upper()
 270             text = device_group.text.strip()
 271             add_item(items, (vendor, device), text)
 272
 273     with open('20-sdio-vendor-model.hwdb', 'wt') as out:
 274         header(out, 'hwdb/sdio.ids')
 275
 276         for key in sorted(items):
 277             if len(key) == 1:
 278                 p, n = 'sdio:c*v{}*', 'VENDOR'
 279             else:
 280                 p, n = 'sdio:c*v{}d{}*', 'MODEL'
 281             print('', p.format(*key),
 282                   f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
 283
 284     print(f'Wrote {out.name}')
 285
 286 def sdio_classes(p):
 287     items = {}
 288
 289     for klass_group in p.CLASSES:
 290         klass = klass_group.KLASS.klass.upper()
 291         text = klass_group.KLASS.text.strip()
 292         add_item(items, klass, text)
 293
 294     with open('20-sdio-classes.hwdb', 'wt') as out:
 295         header(out, 'hwdb/sdio.ids')
 296
 297         for klass in sorted(items):
 298             print(f'',
 299                   f'sdio:c{klass}v*d*',
 300                   f' ID_SDIO_CLASS_FROM_DATABASE={items[klass]}', sep='\n', file=out)
 301
 302     print(f'Wrote {out.name}')
 303
 304 # MAC Address Block Large/Medium/Small
 305 # Large  MA-L 24/24 bit (OUI)
 306 # Medium MA-M 28/20 bit (OUI prefix owned by IEEE)
 307 # Small  MA-S 36/12 bit (OUI prefix owned by IEEE)
 308 def oui(p1, p2, p3):
 309     prefixes = set()
 310     items = {}
 311
 312     for p, check in ((p1, False), (p2, False), (p3, True)):
 313         for vendor_group in p.VENDORS:
 314             prefix = vendor_group.prefix.upper()
 315             if check:
 316                 if prefix in prefixes:
 317                     continue
 318             else:
 319                 prefixes.add(prefix)
 320             start = vendor_group.start.upper()
 321             end = vendor_group.end.upper()
 322
 323             if end and start != end:
 324                 print(f'{prefix:} {start} != {end}', file=sys.stderr)
 325             text = vendor_group.text.strip()
 326
 327             key = prefix + start if end else prefix
 328             add_item(items, key, text)
 329
 330     with open('20-OUI.hwdb', 'wt') as out:
 331         header(out,
 332                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
 333                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
 334                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt')
 335
 336         for pattern in sorted(items):
 337             print(f'',
 338                   f'OUI:{pattern}*',
 339                   f' ID_OUI_FROM_DATABASE={items[pattern]}', sep='\n', file=out)
 340
 341     print(f'Wrote {out.name}')
 342
 343 if __name__ == '__main__':
 344     args = sys.argv[1:]
 345
 346     if not args or 'usb' in args:
 347         p = usb_ids_grammar().parseFile(open('usb.ids', errors='replace'))
 348         usb_vendor_model(p)
 349         usb_classes(p)
 350
 351     if not args or 'pci' in args:
 352         p = pci_ids_grammar().parseFile(open('pci.ids', errors='replace'))
 353         pci_vendor_model(p)
 354         pci_classes(p)
 355
 356     if not args or 'sdio' in args:
 357         p = pci_ids_grammar().parseFile(open('sdio.ids', errors='replace'))
 358         sdio_vendor_model(p)
 359         sdio_classes(p)
 360
 361     if not args or 'oui' in args:
 362         p = oui_grammar('small').parseFile(open('ma-small.txt'))
 363         p2 = oui_grammar('medium').parseFile(open('ma-medium.txt'))
 364         p3 = oui_grammar('large').parseFile(open('ma-large.txt'))
 365         oui(p, p2, p3)