python/pakfire/packages/lexer.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import re
   5
   6 from pakfire.constants import *
   7 from pakfire.i18n import _
   8
   9 import pakfire.shell
  10
  11 import logging
  12 #log = logging.getLogger("pakfire.lexer")
  13 log = logging.getLogger("pakfire")
  14
  15 class LexerError(Exception):
  16         pass
  17
  18
  19 class LexerUnhandledLine(LexerError):
  20         pass
  21
  22
  23 class EndOfFileError(LexerError):
  24         pass
  25
  26
  27 class LexerUndefinedVariableError(LexerError):
  28         pass
  29
  30
  31 LEXER_VALID_PACKAGE_NAME    = re.compile(r"[A-Za-z][A-Za-z0-9\_\-\+]")
  32
  33 # XXX need to build check
  34 LEXER_VALID_SCRIPTLET_NAME  = re.compile(r"((pre|post|pretrans|posttrans)(in|un|up))")
  35
  36 LEXER_COMMENT_CHAR    = "#"
  37 LEXER_COMMENT         = re.compile(r"^\s*#")
  38 LEXER_QUOTES          = "\"'"
  39 LEXER_EMPTY_LINE      = re.compile(r"^\s*$")
  40
  41 LEXER_DEFINITION      = re.compile(r"^([A-Za-z0-9_\-]+)\s*([\+\:])?=\s*(.+)?$")
  42
  43 LEXER_BLOCK_LINE_INDENT = "\t"
  44 LEXER_BLOCK_LINE      = re.compile(r"^\t(.*)$")
  45 LEXER_BLOCK_END       = re.compile(r"^end$")
  46
  47 LEXER_DEFINE_BEGIN    = re.compile(r"^(def)?\s?([A-Za-z0-9_\-]+)$")
  48 LEXER_DEFINE_LINE     = LEXER_BLOCK_LINE
  49 LEXER_DEFINE_END      = LEXER_BLOCK_END
  50
  51 LEXER_PACKAGE_BEGIN   = re.compile(r"^package ([A-Za-z0-9_\-\+\%\{\}]+)$")
  52 LEXER_PACKAGE_LINE    = LEXER_BLOCK_LINE
  53 LEXER_PACKAGE_END     = LEXER_BLOCK_END
  54 LEXER_PACKAGE_INHERIT = re.compile(r"^template ([A-Z0-9]+)$")
  55
  56 LEXER_SCRIPTLET_BEGIN = re.compile(r"^script ([a-z]+)\s?(shell|python)?$")
  57 LEXER_SCRIPTLET_LINE  = LEXER_BLOCK_LINE
  58 LEXER_SCRIPTLET_END   = LEXER_BLOCK_END
  59
  60 LEXER_TEMPLATE_BEGIN  = re.compile(r"^template ([A-Z0-9]+)$")
  61 LEXER_TEMPLATE_LINE   = LEXER_BLOCK_LINE
  62 LEXER_TEMPLATE_END    = LEXER_BLOCK_END
  63
  64 LEXER_BUILD_BEGIN     = re.compile(r"^build$")
  65 LEXER_BUILD_LINE      = LEXER_BLOCK_LINE
  66 LEXER_BUILD_END       = LEXER_BLOCK_END
  67
  68 LEXER_DEPS_BEGIN      = re.compile(r"^dependencies$")
  69 LEXER_DEPS_LINE       = LEXER_BLOCK_LINE
  70 LEXER_DEPS_END        = LEXER_BLOCK_END
  71
  72 LEXER_DISTRO_BEGIN    = re.compile(r"^distribution$")
  73 LEXER_DISTRO_LINE     = LEXER_BLOCK_LINE
  74 LEXER_DISTRO_END      = LEXER_BLOCK_END
  75
  76 LEXER_PACKAGES_BEGIN  = re.compile(r"^packages$")
  77 LEXER_PACKAGES_LINE   = LEXER_BLOCK_LINE
  78 LEXER_PACKAGES_END    = LEXER_BLOCK_END
  79
  80 LEXER_PACKAGE2_BEGIN  = re.compile(r"^package$")
  81 LEXER_PACKAGE2_LINE   = LEXER_BLOCK_LINE
  82 LEXER_PACKAGE2_END    = LEXER_BLOCK_END
  83
  84 LEXER_QUALITY_AGENT_BEGIN = re.compile(r"^quality-agent$")
  85 LEXER_QUALITY_AGENT_LINE  = LEXER_BLOCK_LINE
  86 LEXER_QUALITY_AGENT_END   = LEXER_BLOCK_END
  87
  88 # Statements:
  89 LEXER_EXPORT          = re.compile(r"^export\s+([A-Za-z0-9_\-]+)\s*(\+)?=\s*(.+)?$")
  90 LEXER_EXPORT2         = re.compile(r"^export\s+([A-Za-z0-9_\-]+)$")
  91 LEXER_UNEXPORT        = re.compile(r"^unexport\s+([A-Za-z0-9_\-]+)$")
  92 LEXER_INCLUDE         = re.compile(r"^include\s+(.+)$")
  93
  94 LEXER_VARIABLE        = re.compile(r"\%\{([A-Za-z0-9_\-]+)\}")
  95 LEXER_SHELL           = re.compile(r"\%\((.*)\)")
  96
  97 LEXER_IF_IF           = re.compile(r"^if\s+(.*)\s+(==|!=)\s+(.*)\s*")
  98 LEXER_IF_ELIF         = re.compile(r"^elif\s+(.*)\s*(==|!=)\s*(.*)\s*")
  99 LEXER_IF_ELSE         = re.compile(r"^else")
 100 LEXER_IF_LINE         = LEXER_BLOCK_LINE
 101 LEXER_IF_END          = LEXER_BLOCK_END
 102
 103 class Lexer(object):
 104         def __init__(self, lines=[], parent=None, environ=None):
 105                 self.lines = lines
 106                 self.parent = parent
 107
 108                 self._lineno = 0
 109
 110                 # A place to store all definitions.
 111                 self._definitions = {}
 112
 113                 # Init function that can be overwritten by child classes.
 114                 self.init(environ)
 115
 116                 # Run the parser.
 117                 self.run()
 118
 119         def inherit(self, other):
 120                 """
 121                         Inherit everything from other lexer.
 122                 """
 123                 self._definitions.update(other._definitions)
 124
 125         @property
 126         def definitions(self):
 127                 definitions = {}
 128
 129                 if self.parent:
 130                         definitions.update(self.parent.definitions)
 131
 132                 definitions.update(self._definitions)
 133
 134                 return definitions
 135
 136         @classmethod
 137         def open(cls, filename, *args, **kwargs):
 138                 log.debug("Opening file for parsing: %s" % filename)
 139
 140                 f = open(filename)
 141                 lines = f.readlines()
 142                 f.close()
 143
 144                 return cls(lines, *args, **kwargs)
 145
 146         @property
 147         def lineno(self):
 148                 return self._lineno + 1
 149
 150         @property
 151         def root(self):
 152                 if self.parent:
 153                         return self.parent.root
 154
 155                 return self
 156
 157         def get_line(self, no, raw=False):
 158                 try:
 159                         line = self.lines[no]
 160                 except KeyError:
 161                         raise EndOfFileError
 162
 163                 # Strip newline.
 164                 line = line.rstrip("\n")
 165
 166                 # DEBUG
 167                 #print line
 168
 169                 if raw:
 170                         return line
 171
 172                 # strip comments - caution: quotations
 173
 174                 if line.startswith(LEXER_COMMENT_CHAR):
 175                         return ""
 176
 177                 # XXX fix removing of comments in lines
 178                 #i = -1
 179                 #length = len(line)
 180                 #quote = None
 181
 182                 #for i in range(length):
 183                 #       s = line[i]
 184
 185                 #       if s in LEXER_QUOTES:
 186                 #               if quote == s:
 187                 #                       quote = None
 188                 #               else:
 189                 #                       quote = s
 190
 191                 #       if s == LEXER_COMMENT_CHAR:
 192                 #               return line[:i+1]
 193
 194                 return line
 195
 196         def line_is_empty(self):
 197                 line = self.get_line(self._lineno)
 198
 199                 m = re.match(LEXER_EMPTY_LINE, line)
 200                 if m:
 201                         return True
 202
 203                 return False
 204
 205         def expand_string(self, s):
 206                 if s is None:
 207                         return ""
 208
 209                 # First run all embedded commands.
 210                 while s:
 211                         m = re.search(LEXER_SHELL, s)
 212                         if not m:
 213                                 break
 214
 215                         command = m.group(1)
 216                         result = self.exec_command(command)
 217
 218                         s = s.replace("%%(%s)" % command, result or "")
 219
 220                 # Then expand the variables.
 221                 while s:
 222                         m = re.search(LEXER_VARIABLE, s)
 223                         if not m:
 224                                 break
 225
 226                         var = m.group(1)
 227                         s = s.replace("%%{%s}" % var, self.get_var(var))
 228
 229                 return s
 230
 231         def exec_command(self, command):
 232                 # Expand all variables in the command.
 233                 command = self.expand_string(command)
 234
 235                 # If the command is empty, we don't do anything.
 236                 if not command:
 237                         return
 238
 239                 # Do we need to chroot and change personality?
 240                 shellenv = pakfire.shell.ShellExecuteEnvironment(command,
 241                         shell=True, record_output=True, log_output=False, record_stderr=False)
 242
 243                 try:
 244                         shellenv.execute()
 245
 246                 except ShellEnvironmentError:
 247                         return
 248
 249                 # Strip newline.
 250                 if shellenv.output:
 251                         return shellenv.output.rstrip("\n")
 252
 253                 return shellenv.output
 254
 255         def get_var(self, key, default=None, raw=False):
 256                 definitions = {}
 257                 definitions.update(self.root.definitions)
 258                 if self.parent:
 259                         definitions.update(self.parent.definitions)
 260                 definitions.update(self.definitions)
 261
 262                 val = None
 263                 try:
 264                         val = definitions[key]
 265                 except KeyError:
 266                         pass
 267
 268                 if val is None:
 269                         val = default
 270
 271                 if raw:
 272                         return val
 273
 274                 return self.expand_string(val)
 275
 276         def init(self, environ):
 277                 pass
 278
 279         def get_default_parsers(self):
 280                 return [
 281                         (LEXER_COMMENT,                 self.parse_comment),
 282                         (LEXER_DEFINITION,              self.parse_definition),
 283                         (LEXER_DEFINE_BEGIN,    self.parse_define),
 284                         (LEXER_IF_IF,                   self.parse_if),
 285                 ]
 286
 287         def get_parsers(self):
 288                 return []
 289
 290         def parse_line(self):
 291                 # Skip empty lines.
 292                 if self.line_is_empty():
 293                         self._lineno += 1
 294                         return
 295
 296                 line = self.get_line(self._lineno)
 297
 298                 parsers = self.get_parsers() + self.get_default_parsers()
 299
 300                 found = False
 301                 for pattern, func in parsers:
 302                         m = re.match(pattern, line)
 303                         if m:
 304                                 # Hey, I found a match, we parse it with the subparser function.
 305                                 found = True
 306                                 func()
 307
 308                                 break
 309
 310                 if not found:
 311                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 312
 313         def read_block(self, pattern_start=None, pattern_line=None, pattern_end=None,
 314                         raw=False):
 315                 assert pattern_start
 316                 assert pattern_line
 317                 assert pattern_end
 318
 319                 line = self.get_line(self._lineno)
 320
 321                 m = re.match(pattern_start, line)
 322                 if not m:
 323                         raise LexerError
 324
 325                 # Go in to next line.
 326                 self._lineno += 1
 327
 328                 groups = m.groups()
 329
 330                 lines = []
 331                 while True:
 332                         line = self.get_line(self._lineno, raw=raw)
 333
 334                         m = re.match(pattern_end, line)
 335                         if m:
 336                                 self._lineno += 1
 337                                 break
 338
 339                         m = re.match(pattern_line, line)
 340                         if m:
 341                                 lines.append(m.group(1))
 342                                 self._lineno += 1
 343                                 continue
 344
 345                         m = re.match(LEXER_EMPTY_LINE, line)
 346                         if m:
 347                                 lines.append("")
 348                                 self._lineno += 1
 349                                 continue
 350
 351                         if not line.startswith(LEXER_BLOCK_LINE_INDENT):
 352                                 raise LexerError, "Line has not the right indentation: %d: %s" \
 353                                         % (self.lineno, line)
 354
 355                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 356
 357                 return (groups, lines)
 358
 359         def run(self):
 360                 while self._lineno < len(self.lines):
 361                         self.parse_line()
 362
 363         def parse_comment(self):
 364                 line = self.get_line(self._lineno)
 365
 366                 if not line:
 367                         return
 368
 369                 raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 370
 371         DEP_DEFINITIONS = ("prerequires", "requires", "provides", "conflicts", "obsoletes", "recommends", "suggests")
 372
 373         def parse_definition(self, pattern=LEXER_DEFINITION):
 374                 line = self.get_line(self._lineno)
 375
 376                 m = re.match(pattern, line)
 377                 if not m:
 378                         raise LexerError, "Not a definition: %s" % line
 379
 380                 # Line was correctly parsed, can go on.
 381                 self._lineno += 1
 382
 383                 k, o, v = m.groups()
 384
 385                 if o == "+":
 386                         prev = self.get_var(k, default=None, raw=True)
 387                         if prev:
 388                                 # Add a newline for all dependencies.
 389                                 if k in self.DEP_DEFINITIONS:
 390                                         prev += "\n"
 391
 392                                 v = " ".join((prev or "", v.strip()))
 393
 394                 elif o == ":":
 395                         # Expand the value immediately and save it.
 396                         v = self.expand_string(v)
 397
 398                 # Handle backslash.
 399                 while v and v.endswith("\\"):
 400                         line = self.get_line(self._lineno)
 401                         self._lineno += 1
 402
 403                         v = v[:-1] + line
 404
 405                 self._definitions[k] = v
 406
 407                 return k, v
 408
 409         def parse_define(self):
 410                 line = self.get_line(self._lineno)
 411
 412                 m = re.match(LEXER_DEFINE_BEGIN, line)
 413                 if not m:
 414                         raise Exception, "XXX not a define"
 415
 416                 # Check content of next line.
 417                 found = None
 418                 i = 1
 419                 while True:
 420                         line = self.get_line(self._lineno + i)
 421
 422                         # Skip empty lines.
 423                         empty = re.match(LEXER_EMPTY_LINE, line)
 424                         if empty:
 425                                 i += 1
 426                                 continue
 427
 428                         for pattern in (LEXER_DEFINE_LINE, LEXER_DEFINE_END):
 429                                 found = re.match(pattern, line)
 430                                 if found:
 431                                         break
 432
 433                         if found:
 434                                 break
 435
 436                 if found is None:
 437                         line = self.get_line(self._lineno)
 438                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 439
 440                 # Go in to next line.
 441                 self._lineno += 1
 442
 443                 key = m.group(2)
 444                 assert key
 445
 446                 value = []
 447                 while True:
 448                         line = self.get_line(self._lineno)
 449
 450                         m = re.match(LEXER_DEFINE_END, line)
 451                         if m:
 452                                 self._lineno += 1
 453                                 break
 454
 455                         m = re.match(LEXER_DEFINE_LINE, line)
 456                         if m:
 457                                 self._lineno += 1
 458                                 value.append(m.group(1))
 459                                 continue
 460
 461                         m = re.match(LEXER_EMPTY_LINE, line)
 462                         if m:
 463                                 self._lineno += 1
 464                                 value.append("")
 465                                 continue
 466
 467                         raise LexerError, "Unhandled line: %s" % line
 468
 469                 self._definitions[key] = "\n".join(value)
 470
 471         def _parse_if_block(self, first=True):
 472                 line = self.get_line(self._lineno)
 473
 474                 found = False
 475
 476                 if first:
 477                         m = re.match(LEXER_IF_IF, line)
 478                         if m:
 479                                 found = True
 480                 else:
 481                         m = re.match(LEXER_IF_ELIF, line)
 482                         if m:
 483                                 found = True
 484                         else:
 485                                 m = re.match(LEXER_IF_ELSE, line)
 486                                 if m:
 487                                         found = True
 488
 489                 if not found:
 490                         raise LexerError, "No valid begin of if statement: %d: %s" \
 491                                         % (self.lineno, line)
 492
 493                 self._lineno += 1
 494                 clause = m.groups()
 495                 lines = []
 496
 497                 block_closed = False
 498                 while len(self.lines) >= self._lineno:
 499                         line = self.get_line(self._lineno)
 500
 501                         for pattern in (LEXER_IF_END, LEXER_IF_ELIF, LEXER_IF_ELSE):
 502                                 m = re.match(pattern, line)
 503                                 if m:
 504                                         block_closed = True
 505                                         break
 506
 507                         if block_closed:
 508                                 break
 509
 510                         m = re.match(LEXER_IF_LINE, line)
 511                         if m:
 512                                 self._lineno += 1
 513                                 lines.append("%s" % m.groups())
 514                                 continue
 515
 516                         m = re.match(LEXER_EMPTY_LINE, line)
 517                         if m:
 518                                 self._lineno += 1
 519                                 lines.append("")
 520                                 continue
 521
 522                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 523
 524                 if not block_closed:
 525                         raise LexerError, "Unclosed if block"
 526
 527                 return (clause, lines)
 528
 529         def parse_if(self):
 530                 blocks = []
 531                 blocks.append(self._parse_if_block(first=True))
 532
 533                 while len(self.lines) >= self._lineno:
 534                         line = self.get_line(self._lineno)
 535
 536                         found = False
 537                         for pattern in (LEXER_IF_ELIF, LEXER_IF_ELSE,):
 538                                 m = re.match(pattern, line)
 539                                 if m:
 540                                         # Found another block.
 541                                         found = True
 542                                         blocks.append(self._parse_if_block(first=False))
 543                                         break
 544
 545                         if not found:
 546                                 break
 547
 548                 # Check for end.
 549                 line = self.get_line(self._lineno)
 550                 m = re.match(LEXER_IF_END, line)
 551                 if not m:
 552                         raise LexerError, "Unclosed if clause"
 553
 554                 self._lineno += 1
 555
 556                 # Read in all blocks, now we evaluate each clause.
 557                 for clause, lines in blocks:
 558                         val = False
 559
 560                         if len(clause) == 3:
 561                                 a, op, b = clause
 562
 563                                 # Remove leading and trailing "s and 's.
 564                                 a = a.lstrip("\"'").rstrip("\"'")
 565                                 b = b.lstrip("\"'").rstrip("\"'")
 566
 567                                 a = self.expand_string(a)
 568                                 b = self.expand_string(b)
 569
 570                                 if op == "==":
 571                                         val = a == b
 572                                 elif op == "!=":
 573                                         val = not a == b
 574                                 else:
 575                                         raise LexerError, "Unknown operator: %s" % op
 576
 577                         else:
 578                                 # Else is always true.
 579                                 val = True
 580
 581                         # If any clause is true, we can parse the block.
 582                         if val:
 583                                 lexer = self.__class__(lines, parent=self)
 584                                 self.inherit(lexer)
 585                                 break
 586
 587
 588 class DefaultLexer(Lexer):
 589         """
 590                 A lexer which only knows about simple definitions.
 591         """
 592         pass
 593
 594
 595 class QualityAgentLexer(DefaultLexer):
 596         """
 597                 A lexer to read quality agent exceptions.
 598         """
 599         @property
 600         def exports(self):
 601                 exports = {}
 602
 603                 # Check if we permit full relro.
 604                 if self.get_var("permit_not_full_relro"):
 605                         exports["QUALITY_AGENT_PERMIT_NOT_FULL_RELRO"] = \
 606                                 self.get_var("permit_not_full_relro")
 607
 608                 # Check if we permit $ORIGIN in rpath.
 609                 if self.get_var("rpath_allow_origin"):
 610                         exports["QUALITY_AGENT_RPATH_ALLOW_ORIGIN"] = \
 611                                 self.get_var("rpath_allow_origin")
 612
 613                 # Load execstack whitelist.
 614                 if self.get_var("whitelist_execstack"):
 615                         exports["QUALITY_AGENT_WHITELIST_EXECSTACK"] = \
 616                                 self.get_var("whitelist_execstack")
 617
 618                 # Load nx whitelist.
 619                 if self.get_var("whitelist_nx"):
 620                         exports["QUALITY_AGENT_WHITELIST_NX"] = \
 621                                 self.get_var("whitelist_nx")
 622
 623                 # Load rpath whitelist.
 624                 if self.get_var("whitelist_rpath"):
 625                         exports["QUALITY_AGENT_WHITELIST_RPATH"] = \
 626                                 self.get_var("whitelist_rpath")
 627
 628                 # Load symlink whitelist
 629                 if self.get_var("whitelist_symlink"):
 630                         exports["QUALITY_AGENT_WHITELIST_SYMLINK"] = \
 631                                 self.get_var("whitelist_symlink")
 632
 633                 return exports
 634
 635
 636 class TemplateLexer(DefaultLexer):
 637         def init(self, environ):
 638                 # A place to store the scriptlets.
 639                 self.scriptlets = {}
 640
 641         def inherit(self, other):
 642                 DefaultLexer.inherit(self, other)
 643
 644                 # Inherit all scriptlets.
 645                 self.scriptlets.update(other.scriptlets)
 646
 647         def get_parsers(self):
 648                 return [
 649                         (LEXER_SCRIPTLET_BEGIN, self.parse_scriptlet),
 650                 ]
 651
 652         def parse_scriptlet(self):
 653                 line = self.get_line(self._lineno)
 654
 655                 m = re.match(LEXER_SCRIPTLET_BEGIN, line)
 656                 if not m:
 657                         raise Exception, "Not a scriptlet"
 658
 659                 self._lineno += 1
 660
 661                 name = m.group(1)
 662
 663                 # check if scriptlet was already defined.
 664                 if self.scriptlets.has_key(name):
 665                         raise Exception, "Scriptlet %s is already defined" % name
 666
 667                 lang = m.group(2) or "shell"
 668                 lines = [
 669                         "#<lang: %s>" % lang,
 670                 ]
 671
 672                 while True:
 673                         line = self.get_line(self._lineno, raw=True)
 674
 675                         m = re.match(LEXER_SCRIPTLET_END, line)
 676                         if m:
 677                                 self._lineno += 1
 678                                 break
 679
 680                         m = re.match(LEXER_SCRIPTLET_LINE, line)
 681                         if m:
 682                                 lines.append(m.group(1))
 683                                 self._lineno += 1
 684                                 continue
 685
 686                         m = re.match(LEXER_EMPTY_LINE, line)
 687                         if m:
 688                                 lines.append("")
 689                                 self._lineno += 1
 690                                 continue
 691
 692                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 693
 694                 self.scriptlets[name] = {
 695                         "lang"      : lang,
 696                         "scriptlet" : self.expand_string("\n".join(lines)),
 697                 }
 698
 699         def get_scriptlet(self, name):
 700                 return self.scriptlets.get(name, None)
 701
 702
 703 class PackageLexer(TemplateLexer):
 704         def init(self, environ):
 705                 TemplateLexer.init(self, environ)
 706
 707                 self._template = "MAIN"
 708
 709                 assert isinstance(self.parent, PackagesLexer) or \
 710                         isinstance(self.parent, PackageLexer), self.parent
 711
 712         @property
 713         def definitions(self):
 714                 definitions = {}
 715
 716                 if self.parent:
 717                         definitions.update(self.parent.definitions)
 718
 719                 if self.template:
 720                         definitions.update(self.template.definitions)
 721
 722                 definitions.update(self._definitions)
 723
 724                 return definitions
 725
 726         @property
 727         def template(self):
 728                 if not self._template:
 729                         return None
 730
 731                 # Get template from parent (if exists).
 732                 templates = getattr(self.parent, "templates", None)
 733                 if templates:
 734                         return templates.get(self._template, None)
 735
 736         def get_parsers(self):
 737                 parsers = [
 738                         (LEXER_PACKAGE_INHERIT,         self.parse_inherit),
 739                 ] + TemplateLexer.get_parsers(self)
 740
 741                 return parsers
 742
 743         def parse_inherit(self):
 744                 line = self.get_line(self._lineno)
 745
 746                 m = re.match(LEXER_PACKAGE_INHERIT, line)
 747                 if not m:
 748                         raise LexerError, "Not a template inheritance: %s" % line
 749
 750                 self._lineno += 1
 751
 752                 self._template = m.group(1)
 753
 754                 # Check if template exists.
 755                 if not self.template:
 756                         log.warning(_("Template does not exist: %s") % self._template)
 757
 758         def get_scriptlet(self, name):
 759                 scriptlet = self.scriptlets.get(name, None)
 760
 761                 if scriptlet is None and self.template:
 762                         scriptlet = self.template.get_scriptlet(name)
 763
 764                 if scriptlet and scriptlet["lang"] == "shell":
 765                         scriptlet["scriptlet"] = \
 766                                 self.expand_string(scriptlet["scriptlet"])
 767
 768                 return scriptlet
 769
 770
 771 class ExportLexer(DefaultLexer):
 772         @property
 773         def exports(self):
 774                 if not hasattr(self.parent, "exports"):
 775                         return self._exports
 776
 777                 exports = []
 778                 for export in self._exports + self.parent.exports:
 779                         exports.append(export)
 780
 781                 return exports
 782
 783         def init(self, environ):
 784                 # A list of variables that should be exported in the build
 785                 # environment.
 786                 self._exports = []
 787                 self._unexports = []
 788
 789         def get_parsers(self):
 790                 return [
 791                         (LEXER_EXPORT,                  self.parse_export),
 792                         (LEXER_EXPORT2,                 self.parse_export2),
 793                         (LEXER_UNEXPORT,                self.parse_unexport),
 794                 ]
 795
 796         def inherit(self, other):
 797                 DefaultLexer.inherit(self, other)
 798
 799                 # Try to remove all unexported variables.
 800                 for unexport in other._unexports:
 801                         try:
 802                                 self._exports.remove(unexport)
 803                         except:
 804                                 pass
 805
 806                 for export in other._exports:
 807                         if not export in self._exports:
 808                                 self._exports.append(export)
 809
 810         def parse_export(self):
 811                 k, v = self.parse_definition(pattern=LEXER_EXPORT)
 812
 813                 if k and not k in self.exports:
 814                         self._exports.append(k)
 815
 816         def parse_export2(self):
 817                 line = self.get_line(self._lineno)
 818                 self._lineno += 1
 819
 820                 m = re.match(LEXER_EXPORT2, line)
 821                 if m:
 822                         k = m.group(1)
 823                         if k and k in self.exports:
 824                                 self._exports.append(k)
 825
 826         def parse_unexport(self):
 827                 line = self.get_line(self._lineno)
 828                 self._lineno += 1
 829
 830                 m = re.match(LEXER_UNEXPORT, line)
 831                 if m:
 832                         k = m.group(1)
 833                         if k and k in self.exports:
 834                                 self._exports.remove(k)
 835                                 self._unexports.append(k)
 836
 837
 838 class BuildLexer(ExportLexer):
 839         @property
 840         def stages(self):
 841                 return self.definitions
 842
 843
 844 class RootLexer(ExportLexer):
 845         def init(self, environ):
 846                 ExportLexer.init(self, environ)
 847
 848                 # Import all environment variables.
 849                 if environ:
 850                         for k, v in environ.items():
 851                                 self._definitions[k] = v
 852
 853                                 self.exports.append(k)
 854
 855                 # Place for build instructions
 856                 self.build = BuildLexer([], parent=self)
 857
 858                 # A place to store all packages and templates.
 859                 # The parent of this is the build block because a lot
 860                 # of relevant variables are set there and need to be used
 861                 # later. That keeps us the root lexer a bit more clean.
 862                 self.packages = PackagesLexer([], parent=self.build)
 863
 864                 # Place for quality-agent exceptions
 865                 self.quality_agent = QualityAgentLexer([], parent=self)
 866
 867                 # Include all macros.
 868                 if not self.parent:
 869                         self.include_macros()
 870
 871         def include_macros(self):
 872                 log.debug("Including all macros...")
 873
 874                 for file in sorted(os.listdir(MACRO_FILE_DIR)):
 875                         if not file.endswith(MACRO_EXTENSION):
 876                                 continue
 877
 878                         file = os.path.join(MACRO_FILE_DIR, file)
 879                         self.include(file)
 880
 881         def include(self, filename):
 882                 log.debug("Including file: %s" % filename)
 883
 884                 # Create a new lexer, and parse the whole file.
 885                 include = RootLexer.open(filename, parent=self)
 886
 887                 # Copy all data from the included file.
 888                 self.inherit(include)
 889
 890         def inherit(self, other):
 891                 """
 892                         Inherit everything from other lexer.
 893                 """
 894                 ExportLexer.inherit(self, other)
 895
 896                 self._definitions.update(other._definitions)
 897
 898                 self.build.inherit(other.build)
 899                 self.packages.inherit(other.packages)
 900                 self.quality_agent.inherit(other.quality_agent)
 901
 902         @property
 903         def templates(self):
 904                 return self.packages.templates
 905
 906         def get_parsers(self):
 907                 parsers = ExportLexer.get_parsers(self)
 908                 parsers += [
 909                         (LEXER_INCLUDE,                 self.parse_include),
 910                         (LEXER_PACKAGES_BEGIN,  self.parse_packages),
 911                         (LEXER_BUILD_BEGIN,             self.parse_build),
 912                         (LEXER_QUALITY_AGENT_BEGIN, self.parse_quality_agent),
 913                 ]
 914
 915                 return parsers
 916
 917         def parse_build(self):
 918                 line = self.get_line(self._lineno)
 919
 920                 m = re.match(LEXER_BUILD_BEGIN, line)
 921                 if not m:
 922                         raise LexerError, "Not a build statement: %s" % line
 923
 924                 self._lineno += 1
 925
 926                 lines = []
 927
 928                 while True:
 929                         line = self.get_line(self._lineno)
 930
 931                         m = re.match(LEXER_BUILD_END, line)
 932                         if m:
 933                                 self._lineno += 1
 934                                 break
 935
 936                         m = re.match(LEXER_BUILD_LINE, line)
 937                         if m:
 938                                 lines.append(m.group(1))
 939                                 self._lineno += 1
 940
 941                         # Accept empty lines.
 942                         m = re.match(LEXER_EMPTY_LINE, line)
 943                         if m:
 944                                 lines.append(line)
 945                                 self._lineno += 1
 946                                 continue
 947
 948                 build = BuildLexer(lines, parent=self.build)
 949                 self.build.inherit(build)
 950
 951         def parse_include(self):
 952                 line = self.get_line(self._lineno)
 953
 954                 m = re.match(LEXER_INCLUDE, line)
 955                 if not m:
 956                         raise LexerError, "Not an include statement: %s" % line
 957
 958                 # Get the filename from the line.
 959                 file = m.group(1)
 960                 file = self.expand_string(file)
 961
 962                 # Include the content of the file.
 963                 self.include(file)
 964
 965                 # Go on to next line.
 966                 self._lineno += 1
 967
 968         def parse_packages(self):
 969                 keys, lines = self.read_block(
 970                         pattern_start=LEXER_PACKAGES_BEGIN,
 971                         pattern_line=LEXER_PACKAGES_LINE,
 972                         pattern_end=LEXER_PACKAGES_END,
 973                         raw=True,
 974                 )
 975
 976                 pkgs = PackagesLexer(lines, parent=self.packages)
 977                 self.packages.inherit(pkgs)
 978
 979         def parse_quality_agent(self):
 980                 keys, lines = self.read_block(
 981                         pattern_start=LEXER_QUALITY_AGENT_BEGIN,
 982                         pattern_line=LEXER_QUALITY_AGENT_LINE,
 983                         pattern_end=LEXER_QUALITY_AGENT_END,
 984                         raw = True,
 985                 )
 986
 987                 qa = QualityAgentLexer(lines, parent=self.quality_agent)
 988                 self.quality_agent.inherit(qa)
 989
 990
 991 class PackagesLexer(DefaultLexer):
 992         def init(self, environ):
 993                 # A place to store all templates.
 994                 self._templates = {}
 995
 996                 # A place to store all packages.
 997                 self.packages = []
 998
 999         @property
1000         def templates(self):
1001                 templates = {}
1002
1003                 if self.parent and hasattr(self.parent, "templates"):
1004                         templates.update(self.parent.templates)
1005
1006                 templates.update(self._templates)
1007
1008                 return templates
1009
1010         def inherit(self, other):
1011                 DefaultLexer.inherit(self, other)
1012
1013                 # Copy all templates and packages but make sure
1014                 # to update the parent lexer (for accessing each other).
1015                 for name, template in other.templates.items():
1016                         template.parent = self
1017                         self._templates[name] = template
1018
1019                 for pkg in other.packages:
1020                         pkg.parent = self
1021                         self.packages.append(pkg)
1022
1023         def __iter__(self):
1024                 return iter(self.packages)
1025
1026         def get_parsers(self):
1027                 return [
1028                         (LEXER_TEMPLATE_BEGIN,  self.parse_template),
1029                         (LEXER_PACKAGE_BEGIN,   self.parse_package),
1030                 ]
1031
1032         def parse_template(self):
1033                 line = self.get_line(self._lineno)
1034
1035                 m = re.match(LEXER_TEMPLATE_BEGIN, line)
1036                 if not m:
1037                         raise Exception, "Not a template"
1038
1039                 # Line was correctly parsed, can go on.
1040                 self._lineno += 1
1041
1042                 name = m.group(1)
1043                 lines = []
1044
1045                 while True:
1046                         line = self.get_line(self._lineno)
1047
1048                         m = re.match(LEXER_TEMPLATE_END, line)
1049                         if m:
1050                                 self._lineno += 1
1051                                 break
1052
1053                         m = re.match(LEXER_TEMPLATE_LINE, line)
1054                         if m:
1055                                 lines.append(m.group(1))
1056                                 self._lineno += 1
1057
1058                         # Accept empty lines.
1059                         m = re.match(LEXER_EMPTY_LINE, line)
1060                         if m:
1061                                 lines.append(line)
1062                                 self._lineno += 1
1063                                 continue
1064
1065                 self._templates[name] = TemplateLexer(lines, parent=self)
1066
1067         def parse_package(self):
1068                 line = self.get_line(self._lineno)
1069
1070                 m = re.match(LEXER_PACKAGE_BEGIN, line)
1071                 if not m:
1072                         raise Exception, "Not a package: %s" %line
1073
1074                 self._lineno += 1
1075
1076                 name = m.group(1)
1077                 name = self.expand_string(name)
1078
1079                 m = re.match(LEXER_VALID_PACKAGE_NAME, name)
1080                 if not m:
1081                         raise LexerError, "Invalid package name: %s" % name
1082
1083                 lines = ["_name = %s" % name]
1084
1085                 opened = False
1086                 while len(self.lines) > self._lineno:
1087                         line = self.get_line(self._lineno)
1088
1089                         m = re.match(LEXER_PACKAGE_END, line)
1090                         if m:
1091                                 opened = False
1092                                 self._lineno += 1
1093                                 break
1094
1095                         m = re.match(LEXER_PACKAGE_LINE, line)
1096                         if m:
1097                                 self._lineno += 1
1098                                 lines.append(m.group(1))
1099                                 opened = True
1100                                 continue
1101
1102                         # Accept empty lines.
1103                         m = re.match(LEXER_EMPTY_LINE, line)
1104                         if m:
1105                                 self._lineno += 1
1106                                 lines.append(line)
1107                                 continue
1108
1109                         # If there is an unhandled line in a block, we raise an error.
1110                         if opened:
1111                                 raise Exception, "XXX unhandled line in package block: %s" % line
1112
1113                         # If the block was never opened, we just go on.
1114                         else:
1115                                 break
1116
1117                 if opened:
1118                         raise LexerError, "Unclosed package block '%s'." % name
1119
1120                 package = PackageLexer(lines, parent=self)
1121                 self.packages.append(package)
1122
1123
1124 class FileLexer(DefaultLexer):
1125         def init(self, environ):
1126                 self.build = DefaultLexer()
1127                 self.deps = DefaultLexer()
1128                 self.distro = DefaultLexer()
1129                 self.package = DefaultLexer()
1130
1131         def get_parsers(self):
1132                 return [
1133                         (LEXER_BUILD_BEGIN,             self.parse_build),
1134                         (LEXER_DISTRO_BEGIN,    self.parse_distro),
1135                         (LEXER_PACKAGE2_BEGIN,  self.parse_package),
1136                         (LEXER_DEPS_BEGIN,              self.parse_deps),
1137                 ]
1138
1139         def parse_build(self):
1140                 keys, lines = self.read_block(
1141                         pattern_start=LEXER_BUILD_BEGIN,
1142                         pattern_line=LEXER_BUILD_LINE,
1143                         pattern_end=LEXER_BUILD_END,
1144                         raw=True,
1145                 )
1146
1147                 build = DefaultLexer(lines)
1148                 self.build.inherit(build)
1149
1150         def parse_distro(self):
1151                 keys, lines = self.read_block(
1152                         pattern_start=LEXER_DISTRO_BEGIN,
1153                         pattern_line=LEXER_DISTRO_LINE,
1154                         pattern_end=LEXER_DISTRO_END,
1155                         raw=True,
1156                 )
1157
1158                 distro = DefaultLexer(lines)
1159                 self.distro.inherit(distro)
1160
1161         def parse_package(self):
1162                 keys, lines = self.read_block(
1163                         pattern_start=LEXER_PACKAGE2_BEGIN,
1164                         pattern_line=LEXER_PACKAGE2_LINE,
1165                         pattern_end=LEXER_PACKAGE2_END,
1166                         raw=True,
1167                 )
1168
1169                 pkg = DefaultLexer(lines)
1170                 self.package.inherit(pkg)
1171
1172         def parse_deps(self):
1173                 keys, lines = self.read_block(
1174                         pattern_start=LEXER_DEPS_BEGIN,
1175                         pattern_line=LEXER_DEPS_LINE,
1176                         pattern_end=LEXER_DEPS_END,
1177                         raw=True,
1178                 )
1179
1180                 deps = DefaultLexer(lines)
1181                 self.deps.inherit(deps)