hwdb/ids_parser.py

   1 #!/usr/bin/env python3
   2
   3 import re
   4 import sys
   5 from pyparsing import (Word, White, Literal, Regex,
   6                        LineEnd, SkipTo,
   7                        ZeroOrMore, OneOrMore, Combine, Optional, Suppress,
   8                        stringEnd, pythonStyleComment)
   9
  10 EOL = LineEnd().suppress()
  11 NUM1 = Word('0123456789abcdefABCDEF', exact=1)
  12 NUM2 = Word('0123456789abcdefABCDEF', exact=2)
  13 NUM3 = Word('0123456789abcdefABCDEF', exact=3)
  14 NUM4 = Word('0123456789abcdefABCDEF', exact=4)
  15 NUM6 = Word('0123456789abcdefABCDEF', exact=6)
  16 TAB = White('\t', exact=1).suppress()
  17 COMMENTLINE = pythonStyleComment + EOL
  18 EMPTYLINE = LineEnd()
  19 text_eol = lambda name: Regex(r'[^\n]+')(name) + EOL
  20 # text_eol = lambda name: Word(printables + ' ' + '®üäßçõãİó ×²⁶´‐“\u200E\u200B')(name) + EOL
  21
  22 def klass_grammar():
  23     klass_line = Literal('C ').suppress() + NUM2('klass') + text_eol('text')
  24     subclass_line = TAB + NUM2('subclass') + text_eol('text')
  25     protocol_line = TAB + TAB + NUM2('protocol') + text_eol('name')
  26     subclass = (subclass_line('SUBCLASS') -
  27                 ZeroOrMore(protocol_line('PROTOCOLS*')
  28                            ^ COMMENTLINE.suppress()))
  29     klass = (klass_line('KLASS') -
  30              ZeroOrMore(subclass('SUBCLASSES*')
  31                         ^ COMMENTLINE.suppress()))
  32     return klass
  33
  34 def usb_ids_grammar():
  35     vendor_line = NUM4('vendor') + text_eol('text')
  36     device_line = TAB + NUM4('device') + text_eol('text')
  37     vendor = (vendor_line('VENDOR') +
  38              ZeroOrMore(device_line('VENDOR_DEV*') ^ COMMENTLINE.suppress()))
  39
  40     klass = klass_grammar()
  41
  42     other_line = (Literal('AT ') ^ Literal('HID ') ^ Literal('R ')
  43                   ^ Literal('PHY ') ^ Literal('BIAS ') ^ Literal('HUT ')
  44                   ^ Literal('L ') ^ Literal('VT ') ^ Literal('HCC ')) + text_eol('text')
  45     other_group = (other_line - ZeroOrMore(TAB + text_eol('text')))
  46
  47     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  48     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  49                         ^ other_group.suppress() ^ commentgroup) + stringEnd()
  50
  51     grammar.parseWithTabs()
  52     return grammar
  53
  54 def pci_ids_grammar():
  55     vendor_line = NUM4('vendor') + text_eol('text')
  56     device_line = TAB + NUM4('device') + text_eol('text')
  57     subvendor_line = TAB + TAB + NUM4('a') + White(' ') + NUM4('b') + text_eol('name')
  58
  59     device = (device_line('DEVICE') +
  60               ZeroOrMore(subvendor_line('SUBVENDORS*') ^ COMMENTLINE.suppress()))
  61     vendor = (vendor_line('VENDOR') +
  62               ZeroOrMore(device('DEVICES*') ^ COMMENTLINE.suppress()))
  63
  64     klass = klass_grammar()
  65
  66     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  67     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*')
  68                         ^ commentgroup) + stringEnd()
  69
  70     grammar.parseWithTabs()
  71     return grammar
  72
  73 def sdio_ids_grammar():
  74     vendor_line = NUM4('vendor') + text_eol('text')
  75     device_line = TAB + NUM4('device') + text_eol('text')
  76     vendor = (vendor_line('VENDOR') +
  77               ZeroOrMore(device_line('DEVICES*') ^ COMMENTLINE.suppress()))
  78
  79     klass = klass_grammar()
  80
  81     commentgroup = OneOrMore(COMMENTLINE).suppress() ^ EMPTYLINE.suppress()
  82     grammar = OneOrMore(vendor('VENDORS*') ^ klass('CLASSES*') ^ commentgroup) + stringEnd()
  83
  84     grammar.parseWithTabs()
  85     return grammar
  86
  87 def oui_grammar(type):
  88     prefix_line = (Combine(NUM2 - Suppress('-') - NUM2 - Suppress('-') - NUM2)('prefix')
  89                    - Literal('(hex)') -  text_eol('text'))
  90     if type == 'small':
  91         vendor_line = (NUM3('start') - '000-' - NUM3('end') - 'FFF'
  92                        - Literal('(base 16)') - text_eol('text2'))
  93     elif type == 'medium':
  94         vendor_line = (NUM1('start') - '00000-' - NUM1('end') - 'FFFFF'
  95                        - Literal('(base 16)') - text_eol('text2'))
  96     else:
  97         assert type == 'large'
  98         vendor_line = (NUM6('start')
  99                        - Literal('(base 16)') - text_eol('text2'))
 100
 101     extra_line = TAB - TAB - TAB - TAB - SkipTo(EOL)
 102     vendor = prefix_line + vendor_line + ZeroOrMore(extra_line) + Optional(EMPTYLINE)
 103
 104     grammar = (Literal('OUI') + text_eol('header')
 105                + text_eol('header') + text_eol('header') + EMPTYLINE
 106                + OneOrMore(vendor('VENDORS*')) + stringEnd())
 107
 108     grammar.parseWithTabs()
 109     return grammar
 110
 111
 112 def header(file, *sources):
 113     print('''\
 114 # This file is part of systemd.
 115 #
 116 # Data imported from:{}{}'''.format(' ' if len(sources) == 1 else '\n#   ',
 117                                     '\n#   '.join(sources)),
 118           file=file)
 119
 120 def usb_vendor_model(p):
 121     with open('20-usb-vendor-model.hwdb', 'wt') as out:
 122         header(out, 'http://www.linux-usb.org/usb.ids')
 123
 124         for vendor_group in p.VENDORS:
 125             vendor = vendor_group.VENDOR.vendor.upper()
 126             text = vendor_group.VENDOR.text.strip()
 127             print(f'',
 128                   f'usb:v{vendor}*',
 129                   f' ID_VENDOR_FROM_DATABASE={text}', sep='\n', file=out)
 130
 131             for vendor_dev in vendor_group.VENDOR_DEV:
 132                 device = vendor_dev.device.upper()
 133                 text = vendor_dev.text.strip()
 134                 print(f'',
 135                       f'usb:v{vendor}p{device}*',
 136                       f' ID_MODEL_FROM_DATABASE={text}', sep='\n', file=out)
 137     print(f'Wrote {out.name}')
 138
 139 def usb_classes(p):
 140     with open('20-usb-classes.hwdb', 'wt') as out:
 141         header(out, 'http://www.linux-usb.org/usb.ids')
 142
 143         for klass_group in p.CLASSES:
 144             klass = klass_group.KLASS.klass.upper()
 145             text = klass_group.KLASS.text.strip()
 146
 147             if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 148                 print(f'',
 149                       f'usb:v*p*d*dc{klass}*',
 150                       f' ID_USB_CLASS_FROM_DATABASE={text}', sep='\n', file=out)
 151
 152             for subclass_group in klass_group.SUBCLASSES:
 153                 subclass = subclass_group.subclass.upper()
 154                 text = subclass_group.text.strip()
 155                 if subclass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 156                     print(f'',
 157                           f'usb:v*p*d*dc{klass}dsc{subclass}*',
 158                           f' ID_USB_SUBCLASS_FROM_DATABASE={text}', sep='\n', file=out)
 159
 160                 for protocol_group in subclass_group.PROTOCOLS:
 161                     protocol = protocol_group.protocol.upper()
 162                     text = protocol_group.name.strip()
 163                     if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
 164                         print(f'',
 165                               f'usb:v*p*d*dc{klass}dsc{subclass}dp{protocol}*',
 166                               f' ID_USB_PROTOCOL_FROM_DATABASE={text}', sep='\n', file=out)
 167     print(f'Wrote {out.name}')
 168
 169 def pci_vendor_model(p):
 170     with open('20-pci-vendor-model.hwdb', 'wt') as out:
 171         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 172
 173         for vendor_group in p.VENDORS:
 174             vendor = vendor_group.VENDOR.vendor.upper()
 175             text = vendor_group.VENDOR.text.strip()
 176             print(f'',
 177                   f'pci:v0000{vendor}*',
 178                   f' ID_VENDOR_FROM_DATABASE={text}', sep='\n', file=out)
 179
 180             for device_group in vendor_group.DEVICES:
 181                 device = device_group.device.upper()
 182                 text = device_group.text.strip()
 183                 print(f'',
 184                       f'pci:v0000{vendor}d0000{device}*',
 185                       f' ID_MODEL_FROM_DATABASE={text}', sep='\n', file=out)
 186
 187                 for subvendor_group in device_group.SUBVENDORS:
 188                     sub_vendor = subvendor_group.a.upper()
 189                     sub_model = subvendor_group.b.upper()
 190                     sub_text = subvendor_group.name.strip()
 191                     if sub_text.startswith(text):
 192                         sub_text = sub_text[len(text):].lstrip()
 193                     if sub_text:
 194                         sub_text = f' ({sub_text})'
 195                     print(f'',
 196                           f'pci:v0000{vendor}d0000{device}sv0000{sub_vendor}sd0000{sub_model}*',
 197                           f' ID_MODEL_FROM_DATABASE={text}{sub_text}', sep='\n', file=out)
 198     print(f'Wrote {out.name}')
 199
 200 def pci_classes(p):
 201     with open('20-pci-classes.hwdb', 'wt') as out:
 202         header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
 203
 204         for klass_group in p.CLASSES:
 205             klass = klass_group.KLASS.klass.upper()
 206             text = klass_group.KLASS.text.strip()
 207
 208             print(f'',
 209                   f'pci:v*d*sv*sd*bc{klass}*',
 210                   f' ID_PCI_CLASS_FROM_DATABASE={text}', sep='\n', file=out)
 211
 212             for subclass_group in klass_group.SUBCLASSES:
 213                 subclass = subclass_group.subclass.upper()
 214                 text = subclass_group.text.strip()
 215                 print(f'',
 216                       f'pci:v*d*sv*sd*bc{klass}sc{subclass}*',
 217                       f' ID_PCI_SUBCLASS_FROM_DATABASE={text}', sep='\n', file=out)
 218
 219                 for protocol_group in subclass_group.PROTOCOLS:
 220                     protocol = protocol_group.protocol.upper()
 221                     text = protocol_group.name.strip()
 222                     print(f'',
 223                           f'pci:v*d*sv*sd*bc{klass}sc{subclass}i{protocol}*',
 224                           f' ID_PCI_INTERFACE_FROM_DATABASE={text}', sep='\n', file=out)
 225     print(f'Wrote {out.name}')
 226
 227 def sdio_vendor_model(p):
 228     with open('20-sdio-vendor-model.hwdb', 'wt') as out:
 229         header(out, 'hwdb/sdio.ids')
 230
 231         for vendor_group in p.VENDORS:
 232             vendor = vendor_group.VENDOR.vendor.upper()
 233             text = vendor_group.VENDOR.text.strip()
 234             print(f'',
 235                   f'sdio:c*v{vendor}*',
 236                   f' ID_VENDOR_FROM_DATABASE={text}', sep='\n', file=out)
 237
 238             for device_group in vendor_group.DEVICES:
 239                 device = device_group.device.upper()
 240                 text = device_group.text.strip()
 241                 print(f'',
 242                       f'sdio:c*v{vendor}d{device}*',
 243                       f' ID_MODEL_FROM_DATABASE={text}', sep='\n', file=out)
 244     print(f'Wrote {out.name}')
 245
 246 def sdio_classes(p):
 247     with open('20-sdio-classes.hwdb', 'wt') as out:
 248         header(out, 'hwdb/sdio.ids')
 249
 250         for klass_group in p.CLASSES:
 251             klass = klass_group.KLASS.klass.upper()
 252             text = klass_group.KLASS.text.strip()
 253
 254             print(f'',
 255                   f'sdio:c{klass}v*d*',
 256                   f' ID_SDIO_CLASS_FROM_DATABASE={text}', sep='\n', file=out)
 257     print(f'Wrote {out.name}')
 258
 259 # MAC Address Block Large/Medium/Small
 260 # Large  MA-L 24/24 bit (OUI)
 261 # Medium MA-M 28/20 bit (OUI prefix owned by IEEE)
 262 # Small  MA-S 36/12 bit (OUI prefix owned by IEEE)
 263 def oui(p1, p2, p3):
 264     with open('20-OUI.hwdb', 'wt') as out:
 265         header(out,
 266                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
 267                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
 268                'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt')
 269
 270         prefixes = set()
 271
 272         for p, check in ((p1, False), (p2, False), (p3, True)):
 273             for vendor_group in p.VENDORS:
 274                 prefix = vendor_group.prefix.upper()
 275                 if check:
 276                     if prefix in prefixes:
 277                         continue
 278                 else:
 279                     prefixes.add(prefix)
 280                 start = vendor_group.start.upper()
 281                 end = vendor_group.end.upper()
 282
 283                 if end and start != end:
 284                     print(f'{prefix:} {start} != {end}', file=sys.stderr)
 285                 text = vendor_group.text.strip()
 286
 287                 print(f'',
 288                       f'OUI:{prefix}{start if end else ""}*',
 289                       f' ID_OUI_FROM_DATABASE={text}', sep='\n', file=out)
 290     print(f'Wrote {out.name}')
 291
 292 if __name__ == '__main__':
 293     p = usb_ids_grammar().parseFile(open('usb.ids'))
 294     usb_vendor_model(p)
 295     usb_classes(p)
 296
 297     p = pci_ids_grammar().parseFile(open('pci.ids'))
 298     pci_vendor_model(p)
 299     pci_classes(p)
 300
 301     p = pci_ids_grammar().parseFile(open('sdio.ids'))
 302     sdio_vendor_model(p)
 303     sdio_classes(p)
 304
 305     p = oui_grammar('small').parseFile(open('ma-small.txt'))
 306     p2 = oui_grammar('medium').parseFile(open('ma-medium.txt'))
 307     p3 = oui_grammar('large').parseFile(open('ma-large.txt'))
 308
 309     oui(p, p2, p3)