pakfire/packages/lexer.py

   1 #!/usr/bin/python
   2
   3 import logging
   4 import os
   5 import re
   6
   7 from pakfire.constants import *
   8
   9 class LexerError(Exception):
  10         pass
  11
  12
  13 class LexerUnhandledLine(LexerError):
  14         pass
  15
  16
  17 class EndOfFileError(LexerError):
  18         pass
  19
  20
  21 class LexerUndefinedVariableError(LexerError):
  22         pass
  23
  24
  25 LEXER_VALID_PACKAGE_NAME    = re.compile(r"[A-Za-z][A-Za-z0-9\_\-\+]")
  26
  27 # XXX need to build check
  28 LEXER_VALID_SCRIPTLET_NAME  = re.compile(r"((pre|post|posttrans)(in|un|up))")
  29
  30 LEXER_COMMENT_CHAR    = "#"
  31 LEXER_COMMENT         = re.compile(r"^\s*#")
  32 LEXER_QUOTES          = "\"'"
  33 LEXER_EMPTY_LINE      = re.compile(r"^\s*$")
  34
  35 LEXER_DEFINITION      = re.compile(r"^([A-Za-z0-9_\-]+)\s*(\+)?=\s*(.+)?$")
  36
  37 LEXER_BLOCK_LINE_INDENT = "\t"
  38 LEXER_BLOCK_LINE      = re.compile(r"^\t(.*)$")
  39 LEXER_BLOCK_END       = re.compile(r"^end$")
  40
  41 LEXER_DEFINE_BEGIN    = re.compile(r"^(def)?\s?([A-Za-z0-9_\-]+)$")
  42 LEXER_DEFINE_LINE     = LEXER_BLOCK_LINE
  43 LEXER_DEFINE_END      = LEXER_BLOCK_END
  44
  45 LEXER_PACKAGE_BEGIN   = re.compile(r"^package ([A-Za-z0-9_\-\+\%\{\}]+)$")
  46 LEXER_PACKAGE_LINE    = LEXER_BLOCK_LINE
  47 LEXER_PACKAGE_END     = LEXER_BLOCK_END
  48 LEXER_PACKAGE_INHERIT = re.compile(r"^template ([A-Z]+)$")
  49
  50 LEXER_SCRIPTLET_BEGIN = re.compile(r"^script ([a-z]+)\s?(/[A-Za-z0-9\-\_/]+)?$")
  51 LEXER_SCRIPTLET_LINE  = LEXER_BLOCK_LINE
  52 LEXER_SCRIPTLET_END   = LEXER_BLOCK_END
  53
  54 LEXER_TEMPLATE_BEGIN  = re.compile(r"^template ([A-Z]+)$")
  55 LEXER_TEMPLATE_LINE   = LEXER_BLOCK_LINE
  56 LEXER_TEMPLATE_END    = LEXER_BLOCK_END
  57
  58 LEXER_BUILD_BEGIN     = re.compile(r"^build$")
  59 LEXER_BUILD_LINE      = LEXER_BLOCK_LINE
  60 LEXER_BUILD_END       = LEXER_BLOCK_END
  61
  62 LEXER_DEPS_BEGIN      = re.compile(r"^dependencies$")
  63 LEXER_DEPS_LINE       = LEXER_BLOCK_LINE
  64 LEXER_DEPS_END        = LEXER_BLOCK_END
  65
  66 LEXER_DISTRO_BEGIN    = re.compile(r"^distribution$")
  67 LEXER_DISTRO_LINE     = LEXER_BLOCK_LINE
  68 LEXER_DISTRO_END      = LEXER_BLOCK_END
  69
  70 LEXER_PACKAGES_BEGIN  = re.compile(r"^packages$")
  71 LEXER_PACKAGES_LINE   = LEXER_BLOCK_LINE
  72 LEXER_PACKAGES_END    = LEXER_BLOCK_END
  73
  74 LEXER_PACKAGE2_BEGIN  = re.compile(r"^package$")
  75 LEXER_PACKAGE2_LINE   = LEXER_BLOCK_LINE
  76 LEXER_PACKAGE2_END    = LEXER_BLOCK_END
  77
  78 # Statements:
  79 LEXER_EXPORT          = re.compile(r"^export\s+([A-Za-z0-9_\-]+)\s*(\+)?=\s*(.+)?$")
  80 LEXER_EXPORT2         = re.compile(r"^export\s+([A-Za-z0-9_\-]+)$")
  81 LEXER_UNEXPORT        = re.compile(r"^unexport\s+([A-Za-z0-9_\-]+)$")
  82 LEXER_INCLUDE         = re.compile(r"^include\s+(.+)$")
  83
  84 LEXER_VARIABLE        = re.compile(r"\%\{([A-Za-z0-9_\-]+)\}")
  85 LEXER_SHELL           = re.compile(r"\%\(.*\)")
  86
  87 LEXER_IF_IF           = re.compile(r"^if\s+(.*)\s+(==|!=)\s+(.*)\s*")
  88 LEXER_IF_ELIF         = re.compile(r"^elif\s+(.*)\s*(==|!=)\s*(.*)\s*")
  89 LEXER_IF_ELSE         = re.compile(r"^else")
  90 LEXER_IF_LINE         = LEXER_BLOCK_LINE
  91 LEXER_IF_END          = LEXER_BLOCK_END
  92
  93 class Lexer(object):
  94         def __init__(self, lines=[], parent=None, environ=None):
  95                 self.lines = lines
  96                 self.parent = parent
  97
  98                 self._lineno = 0
  99
 100                 # A place to store all definitions.
 101                 self._definitions = {}
 102
 103                 # Init function that can be overwritten by child classes.
 104                 self.init(environ)
 105
 106                 # Run the parser.
 107                 self.run()
 108
 109         def inherit(self, other):
 110                 """
 111                         Inherit everything from other lexer.
 112                 """
 113                 self._definitions.update(other._definitions)
 114
 115         @property
 116         def definitions(self):
 117                 return self._definitions
 118
 119         @classmethod
 120         def open(cls, filename, *args, **kwargs):
 121                 f = open(filename)
 122                 lines = f.readlines()
 123                 f.close()
 124
 125                 return cls(lines, *args, **kwargs)
 126
 127         @property
 128         def lineno(self):
 129                 return self._lineno + 1
 130
 131         @property
 132         def root(self):
 133                 if self.parent:
 134                         return self.parent.root
 135
 136                 return self
 137
 138         def get_line(self, no, raw=False):
 139                 try:
 140                         line = self.lines[no]
 141                 except KeyError:
 142                         raise EndOfFileError
 143
 144                 # Strip newline.
 145                 line = line.rstrip("\n")
 146
 147                 # DEBUG
 148                 #print line
 149
 150                 if raw:
 151                         return line
 152
 153                 # strip comments - caution: quotations
 154
 155                 if line.startswith(LEXER_COMMENT_CHAR):
 156                         return ""
 157
 158                 # XXX fix removing of comments in lines
 159                 #i = -1
 160                 #length = len(line)
 161                 #quote = None
 162
 163                 #for i in range(length):
 164                 #       s = line[i]
 165
 166                 #       if s in LEXER_QUOTES:
 167                 #               if quote == s:
 168                 #                       quote = None
 169                 #               else:
 170                 #                       quote = s
 171
 172                 #       if s == LEXER_COMMENT_CHAR:
 173                 #               return line[:i+1]
 174
 175                 return line
 176
 177         def line_is_empty(self):
 178                 line = self.get_line(self._lineno)
 179
 180                 m = re.match(LEXER_EMPTY_LINE, line)
 181                 if m:
 182                         return True
 183
 184                 return False
 185
 186         def expand_string(self, s):
 187                 if s is None:
 188                         return ""
 189
 190                 while s:
 191                         m = re.search(LEXER_VARIABLE, s)
 192                         if not m:
 193                                 break
 194
 195                         var = m.group(1)
 196                         s = s.replace("%%{%s}" % var, self.get_var(var))
 197
 198                 return s
 199
 200         def get_var(self, key, default=None):
 201                 definitions = {}
 202                 definitions.update(self.root.definitions)
 203                 definitions.update(self.definitions)
 204
 205                 val = None
 206                 try:
 207                         val = definitions[key]
 208                 except KeyError:
 209                         pass
 210
 211                 if val is None:
 212                         val = default
 213
 214                 return self.expand_string(val)
 215
 216         def init(self, environ):
 217                 pass
 218
 219         def get_default_parsers(self):
 220                 return [
 221                         (LEXER_COMMENT,                 self.parse_comment),
 222                         (LEXER_DEFINITION,              self.parse_definition),
 223                         (LEXER_DEFINE_BEGIN,    self.parse_define),
 224                         (LEXER_IF_IF,                   self.parse_if),
 225                 ]
 226
 227         def get_parsers(self):
 228                 return []
 229
 230         def parse_line(self):
 231                 # Skip empty lines.
 232                 if self.line_is_empty():
 233                         self._lineno += 1
 234                         return
 235
 236                 line = self.get_line(self._lineno)
 237
 238                 parsers = self.get_parsers() + self.get_default_parsers()
 239
 240                 found = False
 241                 for pattern, func in parsers:
 242                         m = re.match(pattern, line)
 243                         if m:
 244                                 # Hey, I found a match, we parse it with the subparser function.
 245                                 found = True
 246                                 func()
 247
 248                                 break
 249
 250                 if not found:
 251                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 252
 253         def read_block(self, pattern_start=None, pattern_line=None, pattern_end=None,
 254                         raw=False):
 255                 assert pattern_start
 256                 assert pattern_line
 257                 assert pattern_end
 258
 259                 line = self.get_line(self._lineno)
 260
 261                 m = re.match(pattern_start, line)
 262                 if not m:
 263                         raise LexerError
 264
 265                 # Go in to next line.
 266                 self._lineno += 1
 267
 268                 groups = m.groups()
 269
 270                 lines = []
 271                 while True:
 272                         line = self.get_line(self._lineno, raw=raw)
 273
 274                         m = re.match(pattern_end, line)
 275                         if m:
 276                                 self._lineno += 1
 277                                 break
 278
 279                         m = re.match(pattern_line, line)
 280                         if m:
 281                                 lines.append(m.group(1))
 282                                 self._lineno += 1
 283                                 continue
 284
 285                         m = re.match(LEXER_EMPTY_LINE, line)
 286                         if m:
 287                                 lines.append("")
 288                                 self._lineno += 1
 289                                 continue
 290
 291                         if not line.startswith(LEXER_BLOCK_LINE_INDENT):
 292                                 raise LexerError, "Line has not the right indentation: %d: %s" \
 293                                         % (self.lineno, line)
 294
 295                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 296
 297                 return (groups, lines)
 298
 299         def run(self):
 300                 while self._lineno < len(self.lines):
 301                         self.parse_line()
 302
 303         def parse_comment(self):
 304                 line = self.get_line(self._lineno)
 305
 306                 if not line:
 307                         return
 308
 309                 raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 310
 311         def parse_definition(self, pattern=LEXER_DEFINITION):
 312                 line = self.get_line(self._lineno)
 313
 314                 m = re.match(pattern, line)
 315                 if not m:
 316                         raise LexerError, "Not a definition: %s" % line
 317
 318                 # Line was correctly parsed, can go on.
 319                 self._lineno += 1
 320
 321                 k, o, v = m.groups()
 322
 323                 if o == "+":
 324                         prev = self.definitions.get(k, None)
 325                         if prev is None and self.parent:
 326                                 prev = self.parent.definitions.get(k, None)
 327                         if prev:
 328                                 v = " ".join((prev or "", v))
 329
 330                 # Handle backslash.
 331                 while v and v.endswith("\\"):
 332                         line = self.get_line(self._lineno)
 333                         self._lineno += 1
 334
 335                         v = v[:-1] + line
 336
 337                 self._definitions[k] = v
 338
 339                 return k, v
 340
 341         def parse_define(self):
 342                 line = self.get_line(self._lineno)
 343
 344                 m = re.match(LEXER_DEFINE_BEGIN, line)
 345                 if not m:
 346                         raise Exception, "XXX not a define"
 347
 348                 # Check content of next line.
 349                 found = None
 350                 i = 1
 351                 while True:
 352                         line = self.get_line(self._lineno + i)
 353
 354                         # Skip empty lines.
 355                         empty = re.match(LEXER_EMPTY_LINE, line)
 356                         if empty:
 357                                 i += 1
 358                                 continue
 359
 360                         for pattern in (LEXER_DEFINE_LINE, LEXER_DEFINE_END):
 361                                 found = re.match(pattern, line)
 362                                 if found:
 363                                         break
 364
 365                         if found:
 366                                 break
 367
 368                 if found is None:
 369                         line = self.get_line(self._lineno)
 370                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 371
 372                 # Go in to next line.
 373                 self._lineno += 1
 374
 375                 key = m.group(2)
 376                 assert key
 377
 378                 value = []
 379                 while True:
 380                         line = self.get_line(self._lineno)
 381
 382                         m = re.match(LEXER_DEFINE_END, line)
 383                         if m:
 384                                 self._lineno += 1
 385                                 break
 386
 387                         m = re.match(LEXER_DEFINE_LINE, line)
 388                         if m:
 389                                 self._lineno += 1
 390                                 value.append(m.group(1))
 391                                 continue
 392
 393                         m = re.match(LEXER_EMPTY_LINE, line)
 394                         if m:
 395                                 self._lineno += 1
 396                                 value.append("")
 397                                 continue
 398
 399                         raise LexerError, "Unhandled line: %s" % line
 400
 401                 self._definitions[key] = "\n".join(value)
 402
 403         def _parse_if_block(self, first=True):
 404                 line = self.get_line(self._lineno)
 405
 406                 found = False
 407
 408                 if first:
 409                         m = re.match(LEXER_IF_IF, line)
 410                         if m:
 411                                 found = True
 412                 else:
 413                         m = re.match(LEXER_IF_ELIF, line)
 414                         if m:
 415                                 found = True
 416                         else:
 417                                 m = re.match(LEXER_IF_ELSE, line)
 418                                 if m:
 419                                         found = True
 420
 421                 if not found:
 422                         raise LexerError, "No valid begin of if statement: %d: %s" \
 423                                         % (self.lineno, line)
 424
 425                 self._lineno += 1
 426                 clause = m.groups()
 427                 lines = []
 428
 429                 block_closed = False
 430                 while len(self.lines) >= self._lineno:
 431                         line = self.get_line(self._lineno)
 432
 433                         for pattern in (LEXER_IF_END, LEXER_IF_ELIF, LEXER_IF_ELSE):
 434                                 m = re.match(pattern, line)
 435                                 if m:
 436                                         block_closed = True
 437                                         break
 438
 439                         if block_closed:
 440                                 break
 441
 442                         m = re.match(LEXER_IF_LINE, line)
 443                         if m:
 444                                 self._lineno += 1
 445                                 lines.append("%s" % m.groups())
 446                                 continue
 447
 448                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 449
 450                 if not block_closed:
 451                         raise LexerError, "Unclosed if block"
 452
 453                 return (clause, lines)
 454
 455         def parse_if(self):
 456                 blocks = []
 457                 blocks.append(self._parse_if_block(first=True))
 458
 459                 while len(self.lines) >= self._lineno:
 460                         line = self.get_line(self._lineno)
 461
 462                         found = False
 463                         for pattern in (LEXER_IF_ELIF, LEXER_IF_ELSE,):
 464                                 m = re.match(pattern, line)
 465                                 if m:
 466                                         # Found another block.
 467                                         found = True
 468                                         blocks.append(self._parse_if_block(first=False))
 469                                         break
 470
 471                         if not found:
 472                                 break
 473
 474                 # Check for end.
 475                 line = self.get_line(self._lineno)
 476                 m = re.match(LEXER_IF_END, line)
 477                 if not m:
 478                         raise LexerError, "Unclosed if clause"
 479
 480                 self._lineno += 1
 481
 482                 # Read in all blocks, now we evaluate each clause.
 483                 for clause, lines in blocks:
 484                         val = False
 485
 486                         if len(clause) == 3:
 487                                 a, op, b = clause
 488
 489                                 # Remove leading and trailing "s and 's.
 490                                 a = a.lstrip("\"'").rstrip("\"'")
 491                                 b = b.lstrip("\"'").rstrip("\"'")
 492
 493                                 a = self.expand_string(a)
 494                                 b = self.expand_string(b)
 495
 496                                 if op == "==":
 497                                         val = a == b
 498                                 elif op == "!=":
 499                                         val = not a == b
 500                                 else:
 501                                         raise LexerError, "Unknown operator: %s" % op
 502
 503                         else:
 504                                 # Else is always true.
 505                                 val = True
 506
 507                         # If any clause is true, we can parse the block.
 508                         if val:
 509                                 lexer = self.__class__(lines, parent=self)
 510                                 self.inherit(lexer)
 511                                 break
 512
 513
 514 class DefaultLexer(Lexer):
 515         """
 516                 A lexer which only knows about simple definitions.
 517         """
 518         pass
 519
 520
 521 class TemplateLexer(DefaultLexer):
 522         def init(self, environ):
 523                 # A place to store the scriptlets.
 524                 self.scriptlets = {}
 525
 526         @property
 527         def definitions(self):
 528                 definitions = {}
 529
 530                 assert self.parent
 531                 definitions.update(self.parent.definitions)
 532                 definitions.update(self._definitions)
 533
 534                 return definitions
 535
 536         def get_parsers(self):
 537                 return [
 538                         (LEXER_SCRIPTLET_BEGIN, self.parse_scriptlet),
 539                 ]
 540
 541         def parse_scriptlet(self):
 542                 line = self.get_line(self._lineno)
 543
 544                 m = re.match(LEXER_SCRIPTLET_BEGIN, line)
 545                 if not m:
 546                         raise Exception, "Not a scriptlet"
 547
 548                 self._lineno += 1
 549
 550                 name = m.group(1)
 551
 552                 # check if scriptlet was already defined.
 553                 if self.scriptlets.has_key(name):
 554                         raise Exception, "Scriptlet %s is already defined" % name
 555
 556                 path = m.group(2)
 557                 if path:
 558                         self.scriptlets[name] = {
 559                                 "lang" : "bin",
 560                                 "path" : self.expand_string(path),
 561                         }
 562                         return
 563
 564                 lines = []
 565                 while True:
 566                         line = self.get_line(self._lineno, raw=True)
 567
 568                         m = re.match(LEXER_SCRIPTLET_END, line)
 569                         if m:
 570                                 self._lineno += 1
 571                                 break
 572
 573                         m = re.match(LEXER_SCRIPTLET_LINE, line)
 574                         if m:
 575                                 lines.append(m.group(1))
 576                                 self._lineno += 1
 577                                 continue
 578
 579                         m = re.match(LEXER_EMPTY_LINE, line)
 580                         if m:
 581                                 lines.append("")
 582                                 self._lineno += 1
 583                                 continue
 584
 585                         raise LexerUnhandledLine, "%d: %s" % (self.lineno, line)
 586
 587                 self.scriptlets[name] = {
 588                         "lang"      : "shell",
 589                         "scriptlet" : "\n".join(lines),
 590                 }
 591
 592
 593 class PackageLexer(TemplateLexer):
 594         def init(self, environ):
 595                 TemplateLexer.init(self, environ)
 596
 597                 self._template = "MAIN"
 598
 599                 assert isinstance(self.parent, PackagesLexer) or \
 600                         isinstance(self.parent, PackageLexer), self.parent
 601
 602         @property
 603         def definitions(self):
 604                 definitions = {}
 605
 606                 if self.template:
 607                         definitions.update(self.template.definitions)
 608
 609                 definitions.update(self._definitions)
 610
 611                 return definitions
 612
 613         @property
 614         def template(self):
 615                 if not self._template:
 616                         return None
 617
 618                 # Get template from parent.
 619                 try:
 620                         return self.root.templates[self._template]
 621                 except KeyError:
 622                         raise LexerError, "Template does not exist: %s" % self._template
 623
 624         def get_parsers(self):
 625                 parsers = [
 626                         (LEXER_PACKAGE_INHERIT,         self.parse_inherit),
 627                 ] + TemplateLexer.get_parsers(self)
 628
 629                 return parsers
 630
 631         def parse_inherit(self):
 632                 line = self.get_line(self._lineno)
 633
 634                 m = re.match(LEXER_PACKAGE_INHERIT, line)
 635                 if not m:
 636                         raise LexerError, "Not a template inheritance: %s" % line
 637
 638                 self._lineno += 1
 639
 640                 self._template = m.group(1)
 641
 642                 # Check if template exists.
 643                 assert self.template
 644
 645
 646 class ExportLexer(DefaultLexer):
 647         @property
 648         def exports(self):
 649                 if not hasattr(self.parent, "exports"):
 650                         return self._exports
 651
 652                 exports = []
 653                 for export in self._exports + self.parent.exports:
 654                         exports.append(export)
 655
 656                 return exports
 657
 658         def init(self, environ):
 659                 # A list of variables that should be exported in the build
 660                 # environment.
 661                 self._exports = []
 662                 self._unexports = []
 663
 664         def get_parsers(self):
 665                 return [
 666                         (LEXER_EXPORT,                  self.parse_export),
 667                         (LEXER_EXPORT2,                 self.parse_export2),
 668                         (LEXER_UNEXPORT,                self.parse_unexport),
 669                 ]
 670
 671         def inherit(self, other):
 672                 DefaultLexer.inherit(self, other)
 673
 674                 # Try to remove all unexported variables.
 675                 for unexport in other._unexports:
 676                         try:
 677                                 self._exports.remove(unexport)
 678                         except:
 679                                 pass
 680
 681                 for export in other._exports:
 682                         if not export in self._exports:
 683                                 self._exports.append(export)
 684
 685         def parse_export(self):
 686                 k, v = self.parse_definition(pattern=LEXER_EXPORT)
 687
 688                 if k and not k in self.exports:
 689                         self._exports.append(k)
 690
 691         def parse_export2(self):
 692                 line = self.get_line(self._lineno)
 693                 self._lineno += 1
 694
 695                 m = re.match(LEXER_EXPORT2, line)
 696                 if m:
 697                         k = m.group(1)
 698                         if k and k in self.exports:
 699                                 self._exports.append(k)
 700
 701         def parse_unexport(self):
 702                 line = self.get_line(self._lineno)
 703                 self._lineno += 1
 704
 705                 m = re.match(LEXER_UNEXPORT, line)
 706                 if m:
 707                         k = m.group(1)
 708                         if k and k in self.exports:
 709                                 self._exports.remove(k)
 710                                 self._unexports.append(k)
 711
 712
 713 class BuildLexer(ExportLexer):
 714         @property
 715         def stages(self):
 716                 return self.definitions
 717
 718
 719 class RootLexer(ExportLexer):
 720         def init(self, environ):
 721                 ExportLexer.init(self, environ)
 722
 723                 # A place to store all packages and templates.
 724                 self.packages = PackagesLexer([], parent=self)
 725
 726                 # Import all environment variables.
 727                 if environ:
 728                         for k, v in environ.items():
 729                                 self._definitions[k] = v
 730
 731                                 self.exports.append(k)
 732
 733                 # Place for build instructions
 734                 self.build = BuildLexer([], parent=self)
 735
 736                 # Include all macros.
 737                 if not self.parent:
 738                         for macro in MACRO_FILES:
 739                                 self.include(macro)
 740
 741         def include(self, file):
 742                 # Create a new lexer, and parse the whole file.
 743                 include = RootLexer.open(file, parent=self)
 744
 745                 # Copy all data from the included file.
 746                 self.inherit(include)
 747
 748         def inherit(self, other):
 749                 """
 750                         Inherit everything from other lexer.
 751                 """
 752                 ExportLexer.inherit(self, other)
 753
 754                 self._definitions.update(other._definitions)
 755
 756                 self.build.inherit(other.build)
 757                 self.packages.inherit(other.packages)
 758
 759         @property
 760         def templates(self):
 761                 return self.packages.templates
 762
 763         def get_parsers(self):
 764                 parsers = ExportLexer.get_parsers(self)
 765                 parsers += [
 766                         (LEXER_INCLUDE,                 self.parse_include),
 767                         (LEXER_PACKAGES_BEGIN,  self.parse_packages),
 768                         (LEXER_BUILD_BEGIN,             self.parse_build),
 769                 ]
 770
 771                 return parsers
 772
 773         def parse_build(self):
 774                 line = self.get_line(self._lineno)
 775
 776                 m = re.match(LEXER_BUILD_BEGIN, line)
 777                 if not m:
 778                         raise LexerError, "Not a build statement: %s" % line
 779
 780                 self._lineno += 1
 781
 782                 lines = []
 783
 784                 while True:
 785                         line = self.get_line(self._lineno)
 786
 787                         m = re.match(LEXER_BUILD_END, line)
 788                         if m:
 789                                 self._lineno += 1
 790                                 break
 791
 792                         m = re.match(LEXER_BUILD_LINE, line)
 793                         if m:
 794                                 lines.append(m.group(1))
 795                                 self._lineno += 1
 796
 797                         # Accept empty lines.
 798                         m = re.match(LEXER_EMPTY_LINE, line)
 799                         if m:
 800                                 lines.append(line)
 801                                 self._lineno += 1
 802                                 continue
 803
 804                 build = BuildLexer(lines, parent=self)
 805                 self.build.inherit(build)
 806
 807         def parse_include(self):
 808                 line = self.get_line(self._lineno)
 809
 810                 m = re.match(LEXER_INCLUDE, line)
 811                 if not m:
 812                         raise LexerError, "Not an include statement: %s" % line
 813
 814                 # Get the filename from the line.
 815                 file = m.group(1)
 816                 file = self.expand_string(file)
 817
 818                 # Include the content of the file.
 819                 self.include(file)
 820
 821                 # Go on to next line.
 822                 self._lineno += 1
 823
 824         def parse_packages(self):
 825                 keys, lines = self.read_block(
 826                         pattern_start=LEXER_PACKAGES_BEGIN,
 827                         pattern_line=LEXER_PACKAGES_LINE,
 828                         pattern_end=LEXER_PACKAGES_END,
 829                         raw=True,
 830                 )
 831
 832                 pkgs = PackagesLexer(lines, parent=self)
 833                 self.packages.inherit(pkgs)
 834
 835
 836 class PackagesLexer(DefaultLexer):
 837         def init(self, environ):
 838                 # A place to store all templates.
 839                 self.templates = {}
 840
 841                 # A place to store all packages.
 842                 self.packages = []
 843
 844         def inherit(self, other):
 845                 # Copy all templates and packages but make sure
 846                 # to update the parent lexer (for accessing each other).
 847                 for name, template in other.templates.items():
 848                         template.parent = self
 849                         self.templates[name] = template
 850
 851                 for pkg in other.packages:
 852                         pkg.parent = self
 853                         self.packages.append(pkg)
 854
 855         def __iter__(self):
 856                 return iter(self.packages)
 857
 858         def get_parsers(self):
 859                 return [
 860                         (LEXER_TEMPLATE_BEGIN,  self.parse_template),
 861                         (LEXER_PACKAGE_BEGIN,   self.parse_package),
 862                 ]
 863
 864         def parse_template(self):
 865                 line = self.get_line(self._lineno)
 866
 867                 m = re.match(LEXER_TEMPLATE_BEGIN, line)
 868                 if not m:
 869                         raise Exception, "Not a template"
 870
 871                 # Line was correctly parsed, can go on.
 872                 self._lineno += 1
 873
 874                 name = m.group(1)
 875                 lines = []
 876
 877                 while True:
 878                         line = self.get_line(self._lineno)
 879
 880                         m = re.match(LEXER_TEMPLATE_END, line)
 881                         if m:
 882                                 self._lineno += 1
 883                                 break
 884
 885                         m = re.match(LEXER_TEMPLATE_LINE, line)
 886                         if m:
 887                                 lines.append(m.group(1))
 888                                 self._lineno += 1
 889
 890                         # Accept empty lines.
 891                         m = re.match(LEXER_EMPTY_LINE, line)
 892                         if m:
 893                                 lines.append(line)
 894                                 self._lineno += 1
 895                                 continue
 896
 897                 template = TemplateLexer(lines, parent=self)
 898                 self.templates[name] = template
 899
 900         def parse_package(self):
 901                 line = self.get_line(self._lineno)
 902
 903                 m = re.match(LEXER_PACKAGE_BEGIN, line)
 904                 if not m:
 905                         raise Exception, "Not a package: %s" %line
 906
 907                 self._lineno += 1
 908
 909                 name = m.group(1)
 910                 name = self.expand_string(name)
 911
 912                 m = re.match(LEXER_VALID_PACKAGE_NAME, name)
 913                 if not m:
 914                         raise LexerError, "Invalid package name: %s" % name
 915
 916                 lines = ["_name = %s" % name]
 917
 918                 opened = False
 919                 while len(self.lines) > self._lineno:
 920                         line = self.get_line(self._lineno)
 921
 922                         m = re.match(LEXER_PACKAGE_END, line)
 923                         if m:
 924                                 opened = False
 925                                 self._lineno += 1
 926                                 break
 927
 928                         m = re.match(LEXER_PACKAGE_LINE, line)
 929                         if m:
 930                                 self._lineno += 1
 931                                 lines.append(m.group(1))
 932                                 opened = True
 933                                 continue
 934
 935                         # Accept empty lines.
 936                         m = re.match(LEXER_EMPTY_LINE, line)
 937                         if m:
 938                                 self._lineno += 1
 939                                 lines.append(line)
 940                                 continue
 941
 942                         # If there is an unhandled line in a block, we raise an error.
 943                         if opened:
 944                                 raise Exception, "XXX unhandled line in package block: %s" % line
 945
 946                         # If the block was never opened, we just go on.
 947                         else:
 948                                 break
 949
 950                 if opened:
 951                         raise LexerError, "Unclosed package block '%s'." % name
 952
 953                 package = PackageLexer(lines, parent=self)
 954                 self.packages.append(package)
 955
 956
 957 class FileLexer(DefaultLexer):
 958         def init(self, environ):
 959                 self.build = DefaultLexer()
 960                 self.deps = DefaultLexer()
 961                 self.distro = DefaultLexer()
 962                 self.package = DefaultLexer()
 963
 964         def get_parsers(self):
 965                 return [
 966                         (LEXER_BUILD_BEGIN,             self.parse_build),
 967                         (LEXER_DISTRO_BEGIN,    self.parse_distro),
 968                         (LEXER_PACKAGE2_BEGIN,  self.parse_package),
 969                         (LEXER_DEPS_BEGIN,              self.parse_deps),
 970                 ]
 971
 972         def parse_build(self):
 973                 keys, lines = self.read_block(
 974                         pattern_start=LEXER_BUILD_BEGIN,
 975                         pattern_line=LEXER_BUILD_LINE,
 976                         pattern_end=LEXER_BUILD_END,
 977                         raw=True,
 978                 )
 979
 980                 build = DefaultLexer(lines)
 981                 self.build.inherit(build)
 982
 983         def parse_distro(self):
 984                 keys, lines = self.read_block(
 985                         pattern_start=LEXER_DISTRO_BEGIN,
 986                         pattern_line=LEXER_DISTRO_LINE,
 987                         pattern_end=LEXER_DISTRO_END,
 988                         raw=True,
 989                 )
 990
 991                 distro = DefaultLexer(lines)
 992                 self.distro.inherit(distro)
 993
 994         def parse_package(self):
 995                 keys, lines = self.read_block(
 996                         pattern_start=LEXER_PACKAGE2_BEGIN,
 997                         pattern_line=LEXER_PACKAGE2_LINE,
 998                         pattern_end=LEXER_PACKAGE2_END,
 999                         raw=True,
1000                 )
1001
1002                 pkg = DefaultLexer(lines)
1003                 self.package.inherit(pkg)
1004
1005         def parse_deps(self):
1006                 keys, lines = self.read_block(
1007                         pattern_start=LEXER_DEPS_BEGIN,
1008                         pattern_line=LEXER_DEPS_LINE,
1009                         pattern_end=LEXER_DEPS_END,
1010                         raw=True,
1011                 )
1012
1013                 deps = DefaultLexer(lines)
1014                 self.deps.inherit(deps)