Change boundary of extracted comments.

diff --git a/gettext-tools/src/ChangeLog b/gettext-tools/src/ChangeLog
index 6bfb700784e26377786c6419c29e9a560f77cdd5..1db53f5343be75123bf5baf4b4c3b537c297b84b 100644 (file)
--- a/gettext-tools/src/ChangeLog
+++ b/gettext-tools/src/ChangeLog
@@ -1,3 +1,24 @@
+2007-06-25  Bruno Haible  <bruno@clisp.org>
+
+       Change boundary of extracted comments.
+       * x-perl.c (struct token_ty): Add comment field.
+       (free_token): Release tp->comment.
+       (extract_quotelike): Fill in tp->comment.
+       (x_perl_prelex): Set or release tp->comment when constructing tokens.
+       (x_perl_lex): Likewise.
+       (extract_balanced): Pass token's comment to remember_a_message.
+
+2007-06-18  Bruno Haible  <bruno@clisp.org>
+
+       Change boundary of extracted comments.
+       * x-ycp.c (struct token_ty): Add comment field.
+       (free_token): New function.
+       (phase5_get): Fill in tp->comment.
+       (phase8_get): Use free_token.
+       (extract_parenthesized): Pass token's comment to remember_a_message.
+       Use free_token.
+       Reported by Karl Eichwalder <ke@suse.de>.
+
 2007-06-09  Bruno Haible  <bruno@clisp.org>
 
        * format-java.c: Include xmalloca.h instead of xallocsa.h.

diff --git a/gettext-tools/src/x-perl.c b/gettext-tools/src/x-perl.c
index fc3d498cf8576094edb345a33b194b14737d763d..863155f632618645e3c73abbe81e3314e3418db0 100644 (file)
--- a/gettext-tools/src/x-perl.c
+++ b/gettext-tools/src/x-perl.c
@@ -555,6 +555,7 @@ struct token_ty
                                   token_type_symbol    ASCII
                                   token_type_variable  global_source_encoding
                                 */
+  refcounted_string_list_ty *comment; /* for token_type_string */
   int line_number;
 };
 
@@ -621,6 +622,8 @@ free_token (token_ty *tp)
     default:
       break;
     }
+  if (tp->type == token_type_string)
+    drop_reference (tp->comment);
   free (tp);
 }
 
@@ -843,6 +846,7 @@ extract_quotelike (token_ty *tp, int delim)
   string[len - 1] = '\0';
   tp->string = xstrdup (string + 1);
   free (string);
+  tp->comment = add_reference (savable_comment);
 }
 
 /* Extract the quotelike constructs with double delimiters, like
@@ -882,7 +886,8 @@ extract_triple_quotelike (message_list_ty *mlp, token_ty *tp, int delim,
 
 /* Perform pass 3 of quotelike extraction (interpolation).
    *tp is a token of type token_type_string.
-   This function replaces tp->string.  */
+   This function replaces tp->string.
+   This function does not access tp->comment.  */
 /* FIXME: Currently may writes null-bytes into the string.  */
 static void
 extract_quotelike_pass3 (token_ty *tp, int error_level)
@@ -1696,6 +1701,10 @@ interpolate_keywords (message_list_ty *mlp, const char *string, int lineno)
   token.type = token_type_string;
   token.sub_type = string_type_qq;
   token.line_number = line_number;
+  /* No need for  token.comment = add_reference (savable_comment);  here.
+     We can let token.comment uninitialized here, and use savable_comment
+     directly, because this function only parses the given string and does
+     not call phase2_getc.  */
   pos.file_name = logical_file_name;
   pos.line_number = lineno;
 
@@ -2208,6 +2217,7 @@ x_perl_prelex (message_list_ty *mlp, token_ty *tp)
              if (delim != '\'')
                interpolate_keywords (mlp, tp->string, line_number);
              free (tp->string);
+             drop_reference (tp->comment);
              tp->type = token_type_regex_op;
              prefer_division_over_regexp = true;
 
@@ -2265,9 +2275,11 @@ x_perl_prelex (message_list_ty *mlp, token_ty *tp)
                  interpolate_keywords (mlp, tp->string, line_number);
                  break;
                case 'r':
+                 drop_reference (tp->comment);
                  tp->type = token_type_regex_op;
                  break;
                case 'w':
+                 drop_reference (tp->comment);
                  tp->type = token_type_symbol;
                  tp->sub_type = symbol_type_none;
                  break;
@@ -2457,6 +2469,7 @@ x_perl_prelex (message_list_ty *mlp, token_ty *tp)
                      tp->string = string;
                      tp->type = token_type_string;
                      tp->sub_type = string_type_qq;
+                     tp->comment = add_reference (savable_comment);
                      tp->line_number = line_number + 1;
                      interpolate_keywords (mlp, tp->string, line_number + 1);
                      return;
@@ -2502,6 +2515,7 @@ x_perl_prelex (message_list_ty *mlp, token_ty *tp)
              extract_quotelike (tp, c);
              interpolate_keywords (mlp, tp->string, line_number);
              free (tp->string);
+             drop_reference (tp->comment);
              tp->type = token_type_other;
              prefer_division_over_regexp = true;
              /* Eat the following modifiers.  */
@@ -2669,6 +2683,7 @@ x_perl_lex (message_list_ty *mlp)
        {
          tp->type = token_type_string;
          tp->sub_type = string_type_q;
+         tp->comment = add_reference (savable_comment);
 #if DEBUG_PERL
          fprintf (stderr,
                   "%s:%d: token %s mutated to token_type_string\n",
@@ -3139,7 +3154,8 @@ extract_balanced (message_list_ty *mlp,
              pos.file_name = logical_file_name;
              pos.line_number = tp->line_number;
              xgettext_current_source_encoding = po_charset_utf8;
-             remember_a_message (mlp, NULL, string, inner_context, &pos, savable_comment);
+             remember_a_message (mlp, NULL, string, inner_context, &pos,
+                                 tp->comment);
              xgettext_current_source_encoding = xgettext_global_source_encoding;
            }
          else if (!skip_until_comma)
@@ -3169,7 +3185,7 @@ extract_balanced (message_list_ty *mlp,
                  arglist_parser_remember (argparser, arg,
                                           string, inner_context,
                                           logical_file_name, tp->line_number,
-                                          savable_comment);
+                                          tp->comment);
                  xgettext_current_source_encoding = xgettext_global_source_encoding;
                }
            }

diff --git a/gettext-tools/src/x-ycp.c b/gettext-tools/src/x-ycp.c
index 803ec98327e0b417a5369897bf393f77173f639f..0df746c8c57b0027becab2a7ef8db1db49be88cd 100644 (file)
--- a/gettext-tools/src/x-ycp.c
+++ b/gettext-tools/src/x-ycp.c
@@ -329,6 +329,7 @@ struct token_ty
 {
   token_type_ty type;
   char *string;                /* for token_type_string_literal, token_type_symbol */
+  refcounted_string_list_ty *comment;  /* for token_type_string_literal */
   int line_number;
 };
 
@@ -405,6 +406,17 @@ phase7_getc ()
 }
 
 
+/* Free the memory pointed to by a 'struct token_ty'.  */
+static inline void
+free_token (token_ty *tp)
+{
+  if (tp->type == token_type_string_literal || tp->type == token_type_symbol)
+    free (tp->string);
+  if (tp->type == token_type_string_literal)
+    drop_reference (tp->comment);
+}
+
+
 /* Combine characters into tokens.  Discard whitespace.  */
 
 static token_ty phase5_pushback[1];
@@ -532,6 +544,7 @@ phase5_get (token_ty *tp)
          buffer[bufpos] = '\0';
          tp->string = xstrdup (buffer);
          tp->type = token_type_string_literal;
+         tp->comment = add_reference (savable_comment);
          return;
 
        case '(':
@@ -592,7 +605,7 @@ phase8_get (token_ty *tp)
       len = strlen (tp->string);
       tp->string = xrealloc (tp->string, len + strlen (tmp.string) + 1);
       strcpy (tp->string + len, tmp.string);
-      free (tmp.string);
+      free_token (&tmp);
     }
 }
 
@@ -672,7 +685,7 @@ extract_parenthesized (message_list_ty *mlp,
                  /* Seen an msgid.  */
                  plural_mp = remember_a_message (mlp, NULL, token.string,
                                                  inner_context, &pos,
-                                                 savable_comment);
+                                                 token.comment);
                  state = 2;
                }
              else
@@ -680,13 +693,14 @@ extract_parenthesized (message_list_ty *mlp,
                  /* Seen an msgid_plural.  */
                  remember_a_message_plural (plural_mp, token.string,
                                             inner_context, &pos,
-                                            savable_comment);
+                                            token.comment);
                  state = 0;
                }
+             drop_reference (token.comment);
            }
          else
            {
-             free (token.string);
+             free_token (&token);
              state = 0;
            }
          next_context_iter = null_context_list_iterator;
@@ -698,7 +712,7 @@ extract_parenthesized (message_list_ty *mlp,
              flag_context_list_table_lookup (
                flag_context_list_table,
                token.string, strlen (token.string)));
-         free (token.string);
+         free_token (&token);
          state = 0;
          continue;