From: Jamie McCracken Date: Tue, 14 Apr 2009 00:30:29 +0000 (-0400) Subject: Updated genie scanner to handle all new/missing vala tokens and directives X-Git-Tag: 0.7.1~31 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cbc2127176905c19229c89ef9c28afc8ca81f41a;p=thirdparty%2Fvala.git Updated genie scanner to handle all new/missing vala tokens and directives --- diff --git a/vala/valageniescanner.vala b/vala/valageniescanner.vala index 0974306de..bf3156568 100644 --- a/vala/valageniescanner.vala +++ b/vala/valageniescanner.vala @@ -48,6 +48,14 @@ public class Vala.Genie.Scanner { string _comment; + Conditional[] conditional_stack; + + struct Conditional { + public bool matched; + public bool else_found; + public bool skip_section; + } + public Scanner (SourceFile source_file) { this.source_file = source_file; @@ -246,6 +254,9 @@ public class Vala.Genie.Scanner { break; } break; + case 'o': + if (matches (begin, "owned")) return TokenType.OWNED; + break; case 'p': if (matches (begin, "print")) return TokenType.PRINT; break; @@ -258,6 +269,9 @@ public class Vala.Genie.Scanner { case 'w': if (matches (begin, "while")) return TokenType.WHILE; break; + case 'y': + if (matches (begin, "yield")) return TokenType.YIELD; + break; } break; case 6: @@ -325,6 +339,9 @@ public class Vala.Genie.Scanner { case 't': if (matches (begin, "typeof")) return TokenType.TYPEOF; break; + case 'y': + if (matches (begin, "yields")) return TokenType.YIELDS; + break; } break; case 7: @@ -355,6 +372,9 @@ public class Vala.Genie.Scanner { case 'p': if (matches (begin, "private")) return TokenType.PRIVATE; break; + case 'u': + if (matches (begin, "unowned")) return TokenType.UNOWNED; + break; case 'v': if (matches (begin, "virtual")) return TokenType.VIRTUAL; break; @@ -371,6 +391,9 @@ public class Vala.Genie.Scanner { case 'd': if (matches (begin, "delegate")) return TokenType.DELEGATE; break; + case 'i': + if (matches (begin, "internal")) return TokenType.INTERNAL; + break; case 'o': if (matches (begin, "override")) return TokenType.OVERRIDE; break; @@ -817,6 +840,34 @@ public class Vala.Genie.Scanner { case '"': if (begin[0] == '\'') { type = TokenType.CHARACTER_LITERAL; + } else if (current < end - 6 && begin[1] == '"' && begin[2] == '"') { + type = TokenType.VERBATIM_STRING_LITERAL; + token_length_in_chars = 6; + current += 3; + while (current < end - 4) { + if (current[0] == '"' && current[1] == '"' && current[2] == '"') { + break; + } else if (current[0] == '\n') { + current++; + line++; + column = 1; + token_length_in_chars = 3; + } else { + unichar u = ((string) current).get_char_validated ((long) (end - current)); + if (u != (unichar) (-1)) { + current += u.to_utf8 (null); + token_length_in_chars++; + } else { + Report.error (new SourceReference (source_file, line, column + token_length_in_chars, line, column + token_length_in_chars), "invalid UTF-8 character"); + } + } + } + if (current[0] == '"' && current[1] == '"' && current[2] == '"') { + current += 3; + } else { + Report.error (new SourceReference (source_file, line, column + token_length_in_chars, line, column + token_length_in_chars), "syntax error, expected \"\"\""); + } + break; } else { type = TokenType.STRING_LITERAL; } @@ -938,6 +989,12 @@ public class Vala.Genie.Scanner { current++; column++; } + + if ((column == 1) && (current[0] == '#')) { + pp_directive (); + return true; + } + return found; } @@ -1071,5 +1128,258 @@ public class Vala.Genie.Scanner { return result_builder.str; } + + bool pp_whitespace () { + bool found = false; + while (current < end && current[0].isspace () && current[0] != '\n') { + found = true; + current++; + column++; + } + return found; + } + + void pp_directive () { + // hash sign + current++; + column++; + + pp_whitespace (); + + char* begin = current; + int len = 0; + while (current < end && current[0].isalnum ()) { + current++; + column++; + len++; + } + + if (len == 2 && matches (begin, "if")) { + parse_pp_if (); + } else if (len == 4 && matches (begin, "elif")) { + parse_pp_elif (); + } else if (len == 4 && matches (begin, "else")) { + parse_pp_else (); + } else if (len == 5 && matches (begin, "endif")) { + parse_pp_endif (); + } else { + Report.error (new SourceReference (source_file, line, column - len, line, column), "syntax error, invalid preprocessing directive"); + } + + if (conditional_stack.length > 0 + && conditional_stack[conditional_stack.length - 1].skip_section) { + // skip lines until next preprocessing directive + bool bol = false; + while (current < end) { + if (bol && current[0] == '#') { + // go back to begin of line + current -= (column - 1); + column = 1; + return; + } + if (current[0] == '\n') { + line++; + column = 0; + bol = true; + } else if (!current[0].isspace ()) { + bol = false; + } + current++; + column++; + } + } + } + + void pp_eol () { + pp_whitespace (); + if (current >= end || current[0] != '\n') { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected newline"); + } + } + + void parse_pp_if () { + pp_whitespace (); + + bool condition = parse_pp_expression (); + + pp_eol (); + + conditional_stack += Conditional (); + + if (condition && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) { + // condition true => process code within if + conditional_stack[conditional_stack.length - 1].matched = true; + } else { + // skip lines until next preprocessing directive + conditional_stack[conditional_stack.length - 1].skip_section = true; + } + } + + void parse_pp_elif () { + pp_whitespace (); + + bool condition = parse_pp_expression (); + + pp_eol (); + + if (conditional_stack.length == 0 || conditional_stack[conditional_stack.length - 1].else_found) { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, unexpected #elif"); + return; + } + + if (condition && !conditional_stack[conditional_stack.length - 1].matched + && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) { + // condition true => process code within if + conditional_stack[conditional_stack.length - 1].matched = true; + conditional_stack[conditional_stack.length - 1].skip_section = false; + } else { + // skip lines until next preprocessing directive + conditional_stack[conditional_stack.length - 1].skip_section = true; + } + } + + void parse_pp_else () { + pp_eol (); + + if (conditional_stack.length == 0 || conditional_stack[conditional_stack.length - 1].else_found) { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, unexpected #else"); + return; + } + + if (!conditional_stack[conditional_stack.length - 1].matched + && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) { + // condition true => process code within if + conditional_stack[conditional_stack.length - 1].matched = true; + conditional_stack[conditional_stack.length - 1].skip_section = false; + } else { + // skip lines until next preprocessing directive + conditional_stack[conditional_stack.length - 1].skip_section = true; + } + } + + void parse_pp_endif () { + pp_eol (); + + if (conditional_stack.length == 0) { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, unexpected #endif"); + return; + } + + conditional_stack.length--; + } + + bool parse_pp_symbol () { + int len = 0; + while (current < end && is_ident_char (current[0])) { + current++; + column++; + len++; + } + + if (len == 0) { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected identifier"); + return false; + } + + string identifier = ((string) (current - len)).ndup (len); + bool defined; + if (identifier == "true") { + defined = true; + } else if (identifier == "false") { + defined = false; + } else { + defined = source_file.context.is_defined (identifier); + } + + return defined; + } + + bool parse_pp_primary_expression () { + if (current >= end) { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected identifier"); + } else if (is_ident_char (current[0])) { + return parse_pp_symbol (); + } else if (current[0] == '(') { + current++; + column++; + pp_whitespace (); + bool result = parse_pp_expression (); + pp_whitespace (); + if (current < end && current[0] == ')') { + current++; + column++; + } else { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected `)'"); + } + return result; + } else { + Report.error (new SourceReference (source_file, line, column, line, column), "syntax error, expected identifier"); + } + return false; + } + + bool parse_pp_unary_expression () { + if (current < end && current[0] == '!') { + current++; + column++; + pp_whitespace (); + return !parse_pp_unary_expression (); + } + + return parse_pp_primary_expression (); + } + + bool parse_pp_equality_expression () { + bool left = parse_pp_unary_expression (); + pp_whitespace (); + while (true) { + if (current < end - 1 && current[0] == '=' && current[1] == '=') { + current += 2; + column += 2; + pp_whitespace (); + bool right = parse_pp_unary_expression (); + left = (left == right); + } else if (current < end - 1 && current[0] == '!' && current[1] == '=') { + current += 2; + column += 2; + pp_whitespace (); + bool right = parse_pp_unary_expression (); + left = (left != right); + } else { + break; + } + } + return left; + } + + bool parse_pp_and_expression () { + bool left = parse_pp_equality_expression (); + pp_whitespace (); + while (current < end - 1 && current[0] == '&' && current[1] == '&') { + current += 2; + column += 2; + pp_whitespace (); + bool right = parse_pp_equality_expression (); + left = left && right; + } + return left; + } + + bool parse_pp_or_expression () { + bool left = parse_pp_and_expression (); + pp_whitespace (); + while (current < end - 1 && current[0] == '|' && current[1] == '|') { + current += 2; + column += 2; + pp_whitespace (); + bool right = parse_pp_and_expression (); + left = left || right; + } + return left; + } + + bool parse_pp_expression () { + return parse_pp_or_expression (); + } } diff --git a/vala/valagenietokentype.vala b/vala/valagenietokentype.vala index a2fd0ca46..5193c61ec 100644 --- a/vala/valagenietokentype.vala +++ b/vala/valagenietokentype.vala @@ -91,6 +91,7 @@ public enum Vala.Genie.TokenType { INLINE, INTEGER_LITERAL, INTERFACE, + INTERNAL, INTERR, IS, ISA, @@ -120,6 +121,7 @@ public enum Vala.Genie.TokenType { OPEN_BRACKET, OPEN_PARENS, OVERRIDE, + OWNED, PASS, PERCENT, PLUS, @@ -149,15 +151,19 @@ public enum Vala.Genie.TokenType { TRUE, TRY, TYPEOF, + UNOWNED, USES, VAR, + VERBATIM_STRING_LITERAL, VIRTUAL, VOID, VOLATILE, WEAK, WHEN, WHILE, - WRITEONLY; + WRITEONLY, + YIELD, + YIELDS; public weak string to_string () { switch (this) { @@ -227,6 +233,7 @@ public enum Vala.Genie.TokenType { case INLINE: return "`inline'"; case INTEGER_LITERAL: return "integer literal"; case INTERFACE: return "`interface'"; + case INTERNAL: return "`internal'"; case INTERR: return "`?'"; case IS: return "`is'"; case ISA: return "`isa'"; @@ -256,6 +263,7 @@ public enum Vala.Genie.TokenType { case OPEN_BRACKET: return "`['"; case OPEN_PARENS: return "`('"; case OVERRIDE: return "`override'"; + case OWNED: return "`owned'"; case PASS: return "`pass'"; case PERCENT: return "`%'"; case PLUS: return "`+'"; @@ -285,6 +293,7 @@ public enum Vala.Genie.TokenType { case TRUE: return "`true'"; case TRY: return "`try'"; case TYPEOF: return "`typeof'"; + case UNOWNED: return "`unowned'"; case USES: return "`uses'"; case VAR: return "`var'"; case VIRTUAL: return "`virtual'"; @@ -294,6 +303,8 @@ public enum Vala.Genie.TokenType { case WHEN: return "`when'"; case WHILE: return "`while'"; case WRITEONLY: return "`writeonly'"; + case YIELD: return "`yield'"; + case YIELDS: return "`yields'"; default: return "unknown token"; } }