pakfire/packages/lexer.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os
   5 import re
   6
   7 from pakfire.constants import *
   8
   9 class LexerError(Exception):
  10         pass
  11
  12
  13 class LexerUnhandledLine(LexerError):
  14         pass
  15
  16
  17 class EndOfFileError(LexerError):
  18         pass
  19
  20
  21 class LexerUndefinedVariableError(LexerError):
  22         pass
  23
  24
  25 LEXER_VALID_PACKAGE_NAME    = re.compile(r"[A-Za-z][A-Za-z0-9\_\-\+]")
  26
  27 # XXX need to build check
  28 LEXER_VALID_SCRIPTLET_NAME  = re.compile(r"((pre|post|posttrans)(in|un|up))")
  29
  30 LEXER_COMMENT_CHAR    = "#"
  31 LEXER_COMMENT         = re.compile(r"^\s*#")
  32 LEXER_QUOTES          = "\"'"
  33 LEXER_EMPTY_LINE      = re.compile(r"^\s*$")
  34
  35 LEXER_DEFINITION      = re.compile(r"^([A-Za-z0-9_\-]+)\s*(\+)?=\s*(.+)?")
  36
  37 LEXER_BLOCK_LINE_INDENT = "\t"
  38 LEXER_BLOCK_LINE      = re.compile(r"^\t(.*)$")
  39 LEXER_BLOCK_END       = re.compile(r"^end$")
  40
  41 LEXER_DEFINE_BEGIN    = re.compile(r"^def ([A-Za-z0-9_\-]+)$")
  42 LEXER_DEFINE_LINE     = LEXER_BLOCK_LINE
  43 LEXER_DEFINE_END      = LEXER_BLOCK_END
  44
  45 LEXER_PACKAGE_BEGIN   = re.compile(r"^package ([A-Za-z0-9_\-\+\%\{\}]+)$")
  46 LEXER_PACKAGE_LINE    = LEXER_BLOCK_LINE
  47 LEXER_PACKAGE_END     = LEXER_BLOCK_END
  48 LEXER_PACKAGE_INHERIT = re.compile(r"^template ([A-Z]+)$")
  49
  50 LEXER_SCRIPTLET_BEGIN = re.compile(r"^script ([a-z]+)\s?(/[A-Za-z0-9\-\_/]+)?$")
  51 LEXER_SCRIPTLET_LINE  = LEXER_BLOCK_LINE
  52 LEXER_SCRIPTLET_END   = LEXER_BLOCK_END
  53
  54 LEXER_TEMPLATE_BEGIN  = re.compile(r"^template ([A-Z]+)$")
  55 LEXER_TEMPLATE_LINE   = LEXER_BLOCK_LINE
  56 LEXER_TEMPLATE_END    = LEXER_BLOCK_END
  57
  58 LEXER_BUILD_BEGIN     = re.compile(r"^build$")
  59 LEXER_BUILD_LINE      = LEXER_BLOCK_LINE
  60 LEXER_BUILD_END       = LEXER_BLOCK_END
  61
  62 LEXER_DEPS_BEGIN      = re.compile(r"^dependencies$")
  63 LEXER_DEPS_LINE       = LEXER_BLOCK_LINE
  64 LEXER_DEPS_END        = LEXER_BLOCK_END
  65
  66 LEXER_DISTRO_BEGIN    = re.compile(r"^distribution$")
  67 LEXER_DISTRO_LINE     = LEXER_BLOCK_LINE
  68 LEXER_DISTRO_END      = LEXER_BLOCK_END
  69
  70 LEXER_PACKAGE2_BEGIN  = re.compile(r"^package$")
  71 LEXER_PACKAGE2_LINE   = LEXER_BLOCK_LINE
  72 LEXER_PACKAGE2_END    = LEXER_BLOCK_END
  73
  74 # Statements:
  75 LEXER_EXPORT          = re.compile(r"^export ([A-Za-z0-9_\-])\s*(\+)?=\s*(.+)$")
  76 LEXER_UNEXPORT        = re.compile(r"^unexport ([A-Za-z0-9_\-]+)$")
  77 LEXER_INCLUDE         = re.compile(r"^include (.+)$")
  78
  79 LEXER_VARIABLE        = re.compile(r"\%\{([A-Za-z0-9_\-]+)\}")
  80
  81
  82 class Lexer(object):
  83         def __init__(self, lines=[], parent=None, environ=None):
  84                 self.lines = lines
  85                 self.parent = parent
  86
  87                 self._lineno = 0
  88
  89                 # A place to store all definitions.
  90                 self._definitions = {}
  91
  92                 # Init function that can be overwritten by child classes.
  93                 self.init(environ)
  94
  95                 # Run the parser.
  96                 self.run()
  97
  98         def inherit(self, other):
  99                 self._definitions.update(other._definitions)
 100
 101         @property
 102         def definitions(self):
 103                 return self._definitions
 104
 105         @classmethod
 106         def open(cls, filename, *args, **kwargs):
 107                 f = open(filename)
 108                 lines = f.readlines()
 109                 f.close()
 110
 111                 return cls(lines, *args, **kwargs)
 112
 113         @property
 114         def lineno(self):
 115                 return self._lineno + 1
 116
 117         @property
 118         def root(self):
 119                 if self.parent:
 120                         return self.parent.root
 121
 122                 return self
 123
 124         def get_line(self, no, raw=False):
 125                 try:
 126                         line = self.lines[no]
 127                 except KeyError:
 128                         raise EndOfFileError
 129
 130                 # Strip newline.
 131                 line = line.rstrip("\n")
 132
 133                 # DEBUG
 134                 #print line
 135
 136                 if raw:
 137                         return line
 138
 139                 # strip comments - caution: quotations
 140
 141                 if line.startswith(LEXER_COMMENT_CHAR):
 142                         return ""
 143
 144                 # XXX fix removing of comments in lines
 145                 #i = -1
 146                 #length = len(line)
 147                 #quote = None
 148
 149                 #for i in range(length):
 150                 #       s = line[i]
 151
 152                 #       if s in LEXER_QUOTES:
 153                 #               if quote == s:
 154                 #                       quote = None
 155                 #               else:
 156                 #                       quote = s
 157
 158                 #       if s == LEXER_COMMENT_CHAR:
 159                 #               return line[:i+1]
 160
 161                 return line
 162
 163         def line_is_empty(self):
 164                 line = self.get_line(self._lineno)
 165
 166                 m = re.match(LEXER_EMPTY_LINE, line)
 167                 if m:
 168                         return True
 169
 170                 return False
 171
 172         def expand_string(self, s):
 173                 if s is None:
 174                         return ""
 175
 176                 while s:
 177                         m = re.search(LEXER_VARIABLE, s)
 178                         if not m:
 179                                 break
 180
 181                         var = m.group(1)
 182                         s = s.replace("%%{%s}" % var, self.get_var(var))
 183
 184                 return s
 185
 186         def get_var(self, key, default=None):
 187                 definitions = {}
 188                 definitions.update(self.root.definitions)
 189                 definitions.update(self.definitions)
 190
 191                 val = None
 192                 try:
 193                         val = definitions[key]
 194                 except KeyError:
 195                         logging.warning("Undefined variable: %s" % key)
 196                         #if default is None:
 197                         #       logging.warning("Undefined variable: %s" % key)
 198                         #       raise LexerUndefinedVariableError, key
 199
 200                 if val is None:
 201                         val = default
 202
 203                 return self.expand_string(val)
 204
 205         def init(self, environ):
 206                 pass
 207
 208         def get_default_parsers(self):
 209                 return [
 210                         (LEXER_COMMENT,                 self.parse_comment),
 211                         (LEXER_DEFINITION,              self.parse_definition),
 212                         (LEXER_DEFINE_BEGIN,    self.parse_define),
 213                 ]
 214
 215         def get_parsers(self):
 216                 return []
 217
 218         def parse_line(self):
 219                 # Skip empty lines.
 220                 if self.line_is_empty():
 221                         self._lineno += 1
 222                         return
 223
 224                 line = self.get_line(self._lineno)
 225
 226                 parsers = self.get_default_parsers() + self.get_parsers()
 227
 228                 found = False
 229                 for pattern, func in parsers:
 230                         m = re.match(pattern, line)
 231                         if m:
 232                                 # Hey, I found a match, we parse it with the subparser function.
 233                                 found = True
 234                                 func()
 235
 236                                 break
 237
 238                 if not found:
 239                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 240
 241         def read_block(self, pattern_start=None, pattern_line=None, pattern_end=None,
 242                         raw=False):
 243                 assert pattern_start
 244                 assert pattern_line
 245                 assert pattern_end
 246
 247                 line = self.get_line(self._lineno)
 248
 249                 m = re.match(pattern_start, line)
 250                 if not m:
 251                         raise LexerError
 252
 253                 # Go in to next line.
 254                 self._lineno += 1
 255
 256                 groups = m.groups()
 257
 258                 lines = []
 259                 while True:
 260                         line = self.get_line(self._lineno, raw=raw)
 261
 262                         m = re.match(pattern_end, line)
 263                         if m:
 264                                 self._lineno += 1
 265                                 break
 266
 267                         m = re.match(pattern_line, line)
 268                         if m:
 269                                 lines.append(m.group(1))
 270                                 self._lineno += 1
 271                                 continue
 272
 273                         m = re.match(LEXER_EMPTY_LINE, line)
 274                         if m:
 275                                 lines.append("")
 276                                 self._lineno += 1
 277                                 continue
 278
 279                         if not line.startswith(LEXER_BLOCK_LINE_INDENT):
 280                                 raise LexerError, "Line has not the right indentation: %d: %s" \
 281                                         % (self.lineno, line)
 282
 283                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 284
 285                 return (groups, lines)
 286
 287         def run(self):
 288                 while self._lineno < len(self.lines):
 289                         self.parse_line()
 290
 291         def parse_comment(self):
 292                 line = self.get_line(self._lineno)
 293
 294                 if not line:
 295                         return
 296
 297                 raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 298
 299         def parse_definition(self, pattern=LEXER_DEFINITION):
 300                 line = self.get_line(self._lineno)
 301
 302                 m = re.match(pattern, line)
 303                 if not m:
 304                         raise LexerError, "Not a definition: %s" % line
 305
 306                 # Line was correctly parsed, can go on.
 307                 self._lineno += 1
 308
 309                 k, o, v = m.groups()
 310
 311                 if o == "+":
 312                         prev = self.definitions.get(k, None)
 313                         if prev:
 314                                 v = " ".join((prev, v))
 315
 316                 # Handle backslash.
 317                 while v and v.endswith("\\"):
 318                         line = self.get_line(self._lineno)
 319                         self._lineno += 1
 320
 321                         v = v[:-1] + line
 322
 323                 self._definitions[k] = v
 324
 325                 return k, v
 326
 327         def parse_define(self):
 328                 line = self.get_line(self._lineno)
 329
 330                 m = re.match(LEXER_DEFINE_BEGIN, line)
 331                 if not m:
 332                         raise Exception, "XXX not a define"
 333
 334                 # Go in to next line.
 335                 self._lineno += 1
 336
 337                 key = m.group(1)
 338                 assert key
 339
 340                 value = []
 341                 while True:
 342                         line = self.get_line(self._lineno)
 343
 344                         m = re.match(LEXER_DEFINE_END, line)
 345                         if m:
 346                                 self._lineno += 1
 347                                 break
 348
 349                         m = re.match(LEXER_DEFINE_LINE, line)
 350                         if m:
 351                                 self._lineno += 1
 352                                 value.append(m.group(1))
 353                                 continue
 354
 355                         m = re.match(LEXER_EMPTY_LINE, line)
 356                         if m:
 357                                 self._lineno += 1
 358                                 value.append("")
 359                                 continue
 360
 361                         raise LexerError, "Unhandled line: %s" % line
 362
 363                 self._definitions[key] = "\n".join(value)
 364
 365
 366 class DefaultLexer(Lexer):
 367         """
 368                 A lexer which only knows about about simple definitions and def.
 369         """
 370         pass
 371
 372
 373 class TemplateLexer(DefaultLexer):
 374         def init(self, environ):
 375                 # A place to store the scriptlets.
 376                 self.scriptlets = {}
 377
 378         @property
 379         def definitions(self):
 380                 definitions = {}
 381
 382                 assert self.parent
 383                 definitions.update(self.parent.definitions)
 384                 definitions.update(self._definitions)
 385
 386                 return definitions
 387
 388         def get_parsers(self):
 389                 return [
 390                         (LEXER_SCRIPTLET_BEGIN, self.parse_scriptlet),
 391                 ]
 392
 393         def parse_scriptlet(self):
 394                 line = self.get_line(self._lineno)
 395
 396                 m = re.match(LEXER_SCRIPTLET_BEGIN, line)
 397                 if not m:
 398                         raise Exception, "Not a scriptlet"
 399
 400                 self._lineno += 1
 401
 402                 name = m.group(1)
 403
 404                 # check if scriptlet was already defined.
 405                 if self.scriptlets.has_key(name):
 406                         raise Exception, "Scriptlet %s is already defined" % name
 407
 408                 path = m.group(2)
 409                 if path:
 410                         self.scriptlets[name] = {
 411                                 "lang" : "bin",
 412                                 "path" : self.expand_string(path),
 413                         }
 414                         return
 415
 416                 lines = []
 417                 while True:
 418                         line = self.get_line(self._lineno, raw=True)
 419
 420                         m = re.match(LEXER_SCRIPTLET_END, line)
 421                         if m:
 422                                 self._lineno += 1
 423                                 break
 424
 425                         m = re.match(LEXER_SCRIPTLET_LINE, line)
 426                         if m:
 427                                 lines.append(m.group(1))
 428                                 self._lineno += 1
 429                                 continue
 430
 431                         m = re.match(LEXER_EMPTY_LINE, line)
 432                         if m:
 433                                 lines.append("")
 434                                 self._lineno += 1
 435                                 continue
 436
 437                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 438
 439                 self.scriptlets[name] = {
 440                         "lang"      : "shell",
 441                         "scriptlet" : "\n".join(lines),
 442                 }
 443
 444
 445 class PackageLexer(TemplateLexer):
 446         def init(self, environ):
 447                 TemplateLexer.init(self, environ)
 448
 449                 self._template = "MAIN"
 450
 451         @property
 452         def definitions(self):
 453                 definitions = {}
 454
 455                 if self.template:
 456                         definitions.update(self.template.definitions)
 457
 458                 definitions.update(self._definitions)
 459
 460                 return definitions
 461
 462         @property
 463         def template(self):
 464                 if not self._template:
 465                         return None
 466
 467                 # Get templates from root.
 468                 assert self.root
 469                 templates = self.root.templates
 470
 471                 try:
 472                         return templates[self._template]
 473                 except KeyError:
 474                         raise LexerError, "Template does not exist: %s" % self._template
 475
 476         def get_parsers(self):
 477                 parsers = TemplateLexer.get_parsers(self)
 478
 479                 parsers += [
 480                         (LEXER_PACKAGE_INHERIT,         self.parse_inherit),
 481                 ]
 482
 483                 return parsers
 484
 485         def parse_inherit(self):
 486                 line = self.get_line(self._lineno)
 487
 488                 m = re.match(LEXER_PACKAGE_INHERIT, line)
 489                 if not m:
 490                         raise LexerError, "Not a template inheritance: %s" % line
 491
 492                 self._lineno += 1
 493
 494                 self._template = m.group(1)
 495
 496                 # Check if template exists.
 497                 assert self.template
 498
 499
 500 class BuildLexer(DefaultLexer):
 501         @property
 502         def definitions(self):
 503                 return self._definitions
 504
 505         @property
 506         def stages(self):
 507                 return self.definitions
 508
 509         def inherit(self, other):
 510                 """
 511                         Inherit everything from other lexer.
 512                 """
 513                 self._definitions.update(other._definitions)
 514
 515
 516 class RootLexer(DefaultLexer):
 517         def init(self, environ):
 518                 # A list of variables that should be exported in the build
 519                 # environment.
 520                 self.exports = []
 521
 522                 # Import all environment variables.
 523                 if environ:
 524                         for k, v in environ.items():
 525                                 self._definitions[k] = v
 526
 527                                 self.exports.append(k)
 528
 529                 # A place to store all packages.
 530                 self.packages = []
 531
 532                 # A place to store all templates.
 533                 self.templates = {}
 534
 535                 # Place for build instructions
 536                 self.build = BuildLexer([], parent=self)
 537
 538                 # Include all macros.
 539                 if not self.parent:
 540                         for macro in MACRO_FILES:
 541                                 self.include(macro)
 542
 543         def include(self, file):
 544                 # Create a new lexer, and parse the whole file.
 545                 include = RootLexer.open(file, parent=self)
 546
 547                 # Copy all data from the included file.
 548                 self.inherit(include)
 549
 550         def inherit(self, other):
 551                 """
 552                         Inherit everything from other lexer.
 553                 """
 554                 self._definitions.update(other._definitions)
 555
 556                 self.build.inherit(other.build)
 557                 self.templates.update(other.templates)
 558                 self.packages += other.packages
 559
 560                 for export in other.exports:
 561                         if not export in self.exports:
 562                                 self.exports.append(export)
 563
 564         def get_parsers(self):
 565                 return [
 566                         (LEXER_INCLUDE,                 self.parse_include),
 567                         (LEXER_TEMPLATE_BEGIN,  self.parse_template),
 568                         (LEXER_PACKAGE_BEGIN,   self.parse_package),
 569                         (LEXER_BUILD_BEGIN,             self.parse_build),
 570                 ]
 571
 572         def parse_build(self):
 573                 line = self.get_line(self._lineno)
 574
 575                 m = re.match(LEXER_BUILD_BEGIN, line)
 576                 if not m:
 577                         raise LexerError, "Not a build statement: %s" % line
 578
 579                 self._lineno += 1
 580
 581                 lines = []
 582
 583                 while True:
 584                         line = self.get_line(self._lineno)
 585
 586                         m = re.match(LEXER_BUILD_END, line)
 587                         if m:
 588                                 self._lineno += 1
 589                                 break
 590
 591                         m = re.match(LEXER_BUILD_LINE, line)
 592                         if m:
 593                                 lines.append(m.group(1))
 594                                 self._lineno += 1
 595
 596                         # Accept empty lines.
 597                         m = re.match(LEXER_EMPTY_LINE, line)
 598                         if m:
 599                                 lines.append(line)
 600                                 self._lineno += 1
 601                                 continue
 602
 603                 build = BuildLexer(lines, parent=self)
 604                 self.build.inherit(build)
 605
 606         def parse_include(self):
 607                 line = self.get_line(self._lineno)
 608
 609                 m = re.match(LEXER_INCLUDE, line)
 610                 if not m:
 611                         raise LexerError, "Not an include statement: %s" % line
 612
 613                 # Get the filename from the line.
 614                 file = m.group(1)
 615                 file = self.expand_string(file)
 616
 617                 # Include the content of the file.
 618                 self.include(file)
 619
 620                 # Go on to next line.
 621                 self._lineno += 1
 622
 623         def parse_export(self):
 624                 k, v = self.parse_definition(pattern, LEXER_EXPORT)
 625
 626                 if k and not k in self.exports:
 627                         self.exports.append(k)
 628
 629         def parse_unexport(self):
 630                 line = self.get_line(self._lineno)
 631                 self._lineno += 1
 632
 633                 m = re.match(LEXER_UNEXPORT, line)
 634                 if m:
 635                         k = m.group(1)
 636                         if k and k in self.exports:
 637                                 self.exports.remove(k)
 638
 639         def parse_template(self):
 640                 line = self.get_line(self._lineno)
 641
 642                 m = re.match(LEXER_TEMPLATE_BEGIN, line)
 643                 if not m:
 644                         raise Exception, "Not a template"
 645
 646                 # Line was correctly parsed, can go on.
 647                 self._lineno += 1
 648
 649                 name = m.group(1)
 650                 lines = []
 651
 652                 while True:
 653                         line = self.get_line(self._lineno)
 654
 655                         m = re.match(LEXER_TEMPLATE_END, line)
 656                         if m:
 657                                 self._lineno += 1
 658                                 break
 659
 660                         m = re.match(LEXER_TEMPLATE_LINE, line)
 661                         if m:
 662                                 lines.append(m.group(1))
 663                                 self._lineno += 1
 664
 665                         # Accept empty lines.
 666                         m = re.match(LEXER_EMPTY_LINE, line)
 667                         if m:
 668                                 lines.append(line)
 669                                 self._lineno += 1
 670                                 continue
 671
 672                 template = TemplateLexer(lines, parent=self)
 673                 self.templates[name] = template
 674
 675         def parse_package(self):
 676                 line = self.get_line(self._lineno)
 677
 678                 m = re.match(LEXER_PACKAGE_BEGIN, line)
 679                 if not m:
 680                         raise Exception, "Not a package: %s" %line
 681
 682                 self._lineno += 1
 683
 684                 name = m.group(1)
 685                 name = self.expand_string(name)
 686
 687                 m = re.match(LEXER_VALID_PACKAGE_NAME, name)
 688                 if not m:
 689                         raise LexerError, "Invalid package name: %s" % name
 690
 691                 lines = ["name = %s" % name]
 692
 693                 while True:
 694                         line = self.get_line(self._lineno)
 695
 696                         m = re.match(LEXER_PACKAGE_END, line)
 697                         if m:
 698                                 self._lineno += 1
 699                                 break
 700
 701                         m = re.match(LEXER_PACKAGE_LINE, line)
 702                         if m:
 703                                 self._lineno += 1
 704                                 lines.append(m.group(1))
 705                                 continue
 706
 707                         # Accept empty lines.
 708                         m = re.match(LEXER_EMPTY_LINE, line)
 709                         if m:
 710                                 self._lineno += 1
 711                                 lines.append(line)
 712                                 continue
 713
 714                         raise Exception, "XXX unhandled line in package block: %s" % line
 715
 716                 package = PackageLexer(lines, parent=self)
 717                 self.packages.append(package)
 718
 719
 720 class FileLexer(DefaultLexer):
 721         def init(self, environ):
 722                 self.build = DefaultLexer()
 723                 self.deps = DefaultLexer()
 724                 self.distro = DefaultLexer()
 725                 self.package = DefaultLexer()
 726
 727         def get_parsers(self):
 728                 return [
 729                         (LEXER_BUILD_BEGIN,             self.parse_build),
 730                         (LEXER_DISTRO_BEGIN,    self.parse_distro),
 731                         (LEXER_PACKAGE2_BEGIN,  self.parse_package),
 732                         (LEXER_DEPS_BEGIN,              self.parse_deps),
 733                 ]
 734
 735         def parse_build(self):
 736                 keys, lines = self.read_block(
 737                         pattern_start=LEXER_BUILD_BEGIN,
 738                         pattern_line=LEXER_BUILD_LINE,
 739                         pattern_end=LEXER_BUILD_END,
 740                         raw=True,
 741                 )
 742
 743                 build = DefaultLexer(lines)
 744                 self.build.inherit(build)
 745
 746         def parse_distro(self):
 747                 keys, lines = self.read_block(
 748                         pattern_start=LEXER_DISTRO_BEGIN,
 749                         pattern_line=LEXER_DISTRO_LINE,
 750                         pattern_end=LEXER_DISTRO_END,
 751                         raw=True,
 752                 )
 753
 754                 distro = DefaultLexer(lines)
 755                 self.distro.inherit(distro)
 756
 757         def parse_package(self):
 758                 keys, lines = self.read_block(
 759                         pattern_start=LEXER_PACKAGE2_BEGIN,
 760                         pattern_line=LEXER_PACKAGE2_LINE,
 761                         pattern_end=LEXER_PACKAGE2_END,
 762                         raw=True,
 763                 )
 764
 765                 pkg = DefaultLexer(lines)
 766                 self.package.inherit(pkg)
 767
 768         def parse_deps(self):
 769                 keys, lines = self.read_block(
 770                         pattern_start=LEXER_DEPS_BEGIN,
 771                         pattern_line=LEXER_DEPS_LINE,
 772                         pattern_end=LEXER_DEPS_END,
 773                         raw=True,
 774                 )
 775
 776                 deps = DefaultLexer(lines)
 777                 self.deps.inherit(deps)