Updated parser to tokenize and parse line by line.

Lexer tokenizes the current line whenever it is called by parser.
"end" token is returned when EOF is encountered.
Currently assumes that newline is their before EOF.

diff --git a/lib/Automake/Parser/Lexer.pm b/lib/Automake/Parser/Lexer.pm
index 1b8a2277756352da3513963c2c07c07b15681cda..ec514c50fffaad99a0eb062b7d30334901434c33 100644 (file)
--- a/lib/Automake/Parser/Lexer.pm
+++ b/lib/Automake/Parser/Lexer.pm
@@ -5,18 +5,25 @@ use Exporter;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(lex);
 
+# Store end token.
+my @end_tok = ["end"];
+
 # lex(string,multiline)
 # Takes as input a string of line and multiline variable deciding whether 
 # current line is related to the previous line. Divides it into tokens as 
 # specified by Regex and outputs an array of Tokens. Every Token is an 
 # array having two values: token name and its value. If its an operator, 
 # it has only one value.
-sub lex($$)
+sub lex($)
 {
-       my ( $curr_line , $multiline ) = @_;
+       my ( $multiline ) = @_;
        my @tokens;
        my $rhs = 0;
-       $_ = $curr_line;
+    $_ = scalar <>;
+       
+       # Send an end token when EOF is reached.
+       return ( \@end_tok, $multiline ) unless $_;
+       
        while( $_ )
        {
                if( $multiline )
@@ -130,6 +137,10 @@ sub lex($$)
                        die "Incorrect input $_";
                }
        }
+       
+       # Returns undef when no tokens.
+       return ( undef, $multiline ) if !@tokens;
+       
        return ( \@tokens , $multiline );
 }
 

diff --git a/lib/Automake/Parser/ParserTable.pm b/lib/Automake/Parser/ParserTable.pm
index 13b607821cd87a28e33ff0669d36a7b173b007ac..1647af076e01dc4ab8cf1c96e67285cbd1451db3 100644 (file)
--- a/lib/Automake/Parser/ParserTable.pm
+++ b/lib/Automake/Parser/ParserTable.pm
@@ -9,21 +9,21 @@ our @Export=qw(@table $accept);
 our $accept=17;
 
 our @table=(
-               {stmts => 5, makerule => 8, comment => 2, lhs => 11, input => 4, ifblock => 10, stmt => 6, optionlist => 13, automakerule => 7, conditional => 9, if => 3, commentlist => 12, value => 1},
-               {reduce => [1,  \&lhs], ':' => 14, '_' => 15},
+               {makerule => 8, stmt => 6, value => 1, input => 4, if => 3, automakerule => 7, ifblock => 10, comment => 2, optionlist => 13, lhs => 11, stmts => 5, conditional => 9, commentlist => 12},
+               {'_' => 15, reduce => [1,  \&lhs], ':' => 14},
                {reduce => [1,  \&commentlist]},
                {value => 16},
                {end => 17},
-               {makerule => 8, comment => 2, lhs => 11, ifblock => 10, stmt => 18, optionlist => 13, automakerule => 7, conditional => 9, if => 3, reduce => [1,  \&input], commentlist => 12, value => 1},
+               {optionlist => 13, comment => 2, commentlist => 12, lhs => 11, conditional => 9, automakerule => 7, ifblock => 10, reduce => [1,  \&input], if => 3, stmt => 18, makerule => 8, value => 1},
                {newline => 19},
                {reduce => [1,  \&stmt]},
                {reduce => [1,  \&stmt]},
                {reduce => [1,  \&stmt]},
-               {optionalelse => 21, reduce => [0,  \&optionalelse], else => 20},
+               {else => 20, reduce => [0,  \&optionalelse], optionalelse => 21},
                {'+' => 23, '=' => 22},
                {comment => 24, reduce => [1,  \&stmt]},
-               {PYTHON => 30, SCRIPTS => 32, JAVA => 31, LTLIBRARIES => 28, DATA => 33, value => 25, HEADERS => 34, TEXINFOS => 36, primaries => 37, LIBRARIES => 27, PROGRAMS => 26, LISP => 29, MASN => 35},
-               {rhs => 39, rhsval => 38},
+               {primaries => 37, value => 25, PROGRAMS => 26, MASN => 35, TEXINFOS => 36, LIBRARIES => 27, HEADERS => 34, SCRIPTS => 32, DATA => 33, LTLIBRARIES => 28, LISP => 29, JAVA => 31, PYTHON => 30},
+               {rhsval => 38, rhs => 39},
                {reduce => [2,  \&optionlist]},
                {newline => 40},
                {},
@@ -49,21 +49,21 @@ our @table=(
                {reduce => [2,  \&lhs]},
                {reduce => [1,  \&rhs]},
                {reduce => [3,  \&makerule], rhsval => 48},
-               {lhs => 11, ifblock => 10, stmts => 49, makerule => 8, comment => 2, conditional => 9, if => 3, commentlist => 12, value => 1, stmt => 6, optionlist => 13, automakerule => 7},
+               {automakerule => 7, ifblock => 10, comment => 2, optionlist => 13, commentlist => 12, lhs => 11, stmts => 49, conditional => 9, stmt => 6, makerule => 8, value => 1, if => 3},
                {reduce => [3,  \&stmts]},
-               {comment => 2, stmts => 50, makerule => 8, ifblock => 10, lhs => 11, automakerule => 7, optionlist => 13, stmt => 6, value => 1, commentlist => 12, if => 3, conditional => 9},
-               {reduce => [0,  \&optionalcond], value => 51, optionalcond => 52},
-               {optionalcomments => 53, comment => 2, commentlist => 54, reduce => [0,  \&optionalcomments]},
-               {reduce => [1,  \&optionalrhs], rhsval => 48},
-               {rhs => 45, optionalrhs => 55, reduce => [0,  \&optionalrhs], rhsval => 38},
+               {if => 3, value => 1, stmt => 6, makerule => 8, commentlist => 12, lhs => 11, stmts => 50, conditional => 9, comment => 2, optionlist => 13, ifblock => 10, automakerule => 7},
+               {value => 51, optionalcond => 52, reduce => [0,  \&optionalcond]},
+               {comment => 2, reduce => [0,  \&optionalcomments], optionalcomments => 53, commentlist => 54},
+               {rhsval => 48, reduce => [1,  \&optionalrhs]},
+               {optionalrhs => 55, rhsval => 38, reduce => [0,  \&optionalrhs], rhs => 45},
                {reduce => [3,  \&optionlist]},
                {reduce => [2,  \&rhs]},
-               {reduce => [4,  \&ifblock], value => 1, commentlist => 12, if => 3, conditional => 9, automakerule => 7, optionlist => 13, stmt => 18, ifblock => 10, lhs => 11, comment => 2, makerule => 8},
-               {comment => 2, makerule => 8, ifblock => 10, lhs => 11, automakerule => 7, optionlist => 13, stmt => 18, reduce => [3,  \&optionalelse], value => 1, commentlist => 12, if => 3, conditional => 9},
+               {if => 3, reduce => [4,  \&ifblock], value => 1, stmt => 18, makerule => 8, commentlist => 12, conditional => 9, lhs => 11, optionlist => 13, comment => 2, ifblock => 10, automakerule => 7},
+               {value => 1, stmt => 18, makerule => 8, if => 3, reduce => [3,  \&optionalelse], ifblock => 10, automakerule => 7, commentlist => 12, conditional => 9, lhs => 11, optionlist => 13, comment => 2},
                {reduce => [1,  \&optionalcond]},
                {reduce => [4,  \&conditional]},
                {reduce => [4,  \&automakerule]},
-               {reduce => [1,  \&optionalcomments], comment => 24},
-               {reduce => [0,  \&optionalcomments], commentlist => 54, comment => 2, optionalcomments => 56},
+               {comment => 24, reduce => [1,  \&optionalcomments]},
+               {comment => 2, reduce => [0,  \&optionalcomments], optionalcomments => 56, commentlist => 54},
                {reduce => [5,  \&automakerule]}
 );
\ No newline at end of file

diff --git a/lib/Automake/Parser/input.txt b/lib/Automake/Parser/input.txt
index 77e2ba539b0fc15f48b62e6f0e66e77c863b8c06..9324ebbca349b350d87b793fb06b2e105cd89aad 100644 (file)
--- a/lib/Automake/Parser/input.txt
+++ b/lib/Automake/Parser/input.txt
@@ -1,4 +1,4 @@
-## Process this file with automake to produce Makefile.in
+# Process this file with automake to produce Makefile.in
 dist_bin_PROGRAMS = server \
 client
 server_SOURCES = server.c db.c   ## Server Files \

diff --git a/lib/Automake/Parser/parser.pl b/lib/Automake/Parser/parser.pl
index 65eda46daec6759cdf8c82eb6e8075d83197cb8a..99344500429680c3137c3bd1253f9a1be96e469c 100644 (file)
--- a/lib/Automake/Parser/parser.pl
+++ b/lib/Automake/Parser/parser.pl
@@ -10,35 +10,12 @@ my $debug = 0;
 #Stores the list of tokens generated by lexer.
 my @tokens; 
 
-my $multiline = 0;
-my $curr_tokens;
-
-#Read input from file specified in Arguements or STDIN.
-while ( <> )
-{
-       ( $curr_tokens, $multiline ) = lex( $_ , $multiline );
-       push @tokens, @$curr_tokens;
-}
-
-#Prints to STDERR if Debug mode is on.
-if( $debug )
-{
-       print STDERR "Lexer Output\n";
-       foreach my $token ( @tokens )
-       {
-               print STDERR join(" ", @{$token}), "\n";
-       }
-}
-#Push a newline token if last token is not newline
-if( $tokens[-1][0] ne "newline" )
-{
-       push @tokens,["newline"];
-}
-
-push @tokens, [ "end" ];
 my @stack = (0);
 print STDERR "Parser Output\n" if $debug;
 
+my $multiline = 0;
+my $curr_tokens;
+
 while ( @stack )
 {
        if($stack[-1] == $ParserTable::accept)
@@ -47,6 +24,16 @@ while ( @stack )
                printgraph( $stack[-4] );
                last;
        }
+       while( !@tokens )
+       {
+               # Calls lexer to get next tokens.
+               ( $curr_tokens, $multiline ) = lex( $multiline );
+               
+               # Continue if their is no tokens
+               next unless $curr_tokens;
+               
+               push @tokens, @$curr_tokens;
+       }
        my @curr_token = @{ $tokens[0] };       
        if(my $val = $ParserTable::table[ $stack[-1] ]{ $curr_token[0] })
        {

diff --git a/lib/Automake/Parser/t/conditional.txt b/lib/Automake/Parser/t/conditional.txt
index 3708d80935861c669ee3d11045f0fca90b36f244..1db874a372c34e8d422b84a46f2258c290902fb5 100644 (file)
--- a/lib/Automake/Parser/t/conditional.txt
+++ b/lib/Automake/Parser/t/conditional.txt
@@ -3,4 +3,4 @@ if windows
 server_SOURCES = winserver.c
 else
 server_SOURCES = unixserver.c
-endif
\ No newline at end of file
+endif

diff --git a/lib/Automake/Parser/t/nestedcond.txt b/lib/Automake/Parser/t/nestedcond.txt
index 99d9ae9ca4882c9195b9d24a4a60f1edc34e0340..b7141a5e680cb3c8e2aa519c42c635ad150117e5 100644 (file)
--- a/lib/Automake/Parser/t/nestedcond.txt
+++ b/lib/Automake/Parser/t/nestedcond.txt
@@ -17,4 +17,4 @@ endif
 if CASE_B
 SUBDIRS += iXYZ
 SUBDIRS += jZYX
-endif
\ No newline at end of file
+endif