1 /* Extended regular expression matching and search library.
2 Copyright (C) 2002-2013 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Isamu Hasegawa <isamu@yamato.ibm.com>.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <http://www.gnu.org/licenses/>. */
20 #ifndef _REGEX_INTERNAL_H
21 #define _REGEX_INTERNAL_H 1
29 #if defined HAVE_LANGINFO_H || defined HAVE_LANGINFO_CODESET || defined _LIBC
30 # include <langinfo.h>
32 #if defined HAVE_LOCALE_H || defined _LIBC
35 #if defined HAVE_WCHAR_H || defined _LIBC
37 #endif /* HAVE_WCHAR_H || _LIBC */
38 #if defined HAVE_WCTYPE_H || defined _LIBC
40 #endif /* HAVE_WCTYPE_H || _LIBC */
41 #if defined HAVE_STDBOOL_H || defined _LIBC
43 #endif /* HAVE_STDBOOL_H || _LIBC */
44 #if defined HAVE_STDINT_H || defined _LIBC
46 #endif /* HAVE_STDINT_H || _LIBC */
48 # include <bits/libc-lock.h>
50 # define __libc_lock_define(CLASS,NAME)
51 # define __libc_lock_init(NAME) do { } while (0)
52 # define __libc_lock_lock(NAME) do { } while (0)
53 # define __libc_lock_unlock(NAME) do { } while (0)
56 /* In case that the system doesn't have isblank(). */
57 #if !defined _LIBC && !defined HAVE_ISBLANK && !defined isblank
58 # define isblank(ch) ((ch) == ' ' || (ch) == '\t')
62 # ifndef _RE_DEFINE_LOCALE_FUNCTIONS
63 # define _RE_DEFINE_LOCALE_FUNCTIONS 1
64 # include <locale/localeinfo.h>
65 # include <locale/elem-hash.h>
66 # include <locale/coll-lookup.h>
70 /* This is for other GNU distributions with internationalized messages. */
71 #if (HAVE_LIBINTL_H && ENABLE_NLS) || defined _LIBC
75 # define gettext(msgid) \
76 __dcgettext (_libc_intl_domainname, msgid, LC_MESSAGES)
79 # define gettext(msgid) (msgid)
83 /* This define is so xgettext can find the internationalizable
85 # define gettext_noop(String) String
88 /* For loser systems without the definition. */
90 # define SIZE_MAX ((size_t) -1)
93 #if (defined MB_CUR_MAX && HAVE_LOCALE_H && HAVE_WCTYPE_H && HAVE_WCHAR_H && HAVE_WCRTOMB && HAVE_MBRTOWC && HAVE_WCSCOLL) || _LIBC
94 # define RE_ENABLE_I18N
98 # define BE(expr, val) __builtin_expect (expr, val)
100 # define BE(expr, val) (expr)
103 /* Number of single byte character. */
106 #define COLL_ELEM_LEN_MAX 8
108 /* The character which represents newline. */
109 #define NEWLINE_CHAR '\n'
110 #define WIDE_NEWLINE_CHAR L'\n'
112 /* Rename to standard API for using out of glibc. */
114 # define __wctype wctype
115 # define __iswctype iswctype
116 # define __btowc btowc
117 # define __mbrtowc mbrtowc
118 # define __mempcpy mempcpy
119 # define __wcrtomb wcrtomb
120 # define __regfree regfree
121 # define attribute_hidden
122 #endif /* not _LIBC */
124 extern const char __re_error_msgid
[] attribute_hidden
;
125 extern const size_t __re_error_msgid_idx
[] attribute_hidden
;
127 /* An integer used to represent a set of bits. It must be unsigned,
128 and must be at least as wide as unsigned int. */
129 typedef unsigned long int bitset_word_t
;
130 /* All bits set in a bitset_word_t. */
131 #define BITSET_WORD_MAX ULONG_MAX
132 /* Number of bits in a bitset_word_t. */
133 #define BITSET_WORD_BITS (sizeof (bitset_word_t) * CHAR_BIT)
134 /* Number of bitset_word_t in a bit_set. */
135 #define BITSET_WORDS (SBC_MAX / BITSET_WORD_BITS)
136 typedef bitset_word_t bitset_t
[BITSET_WORDS
];
137 typedef bitset_word_t
*re_bitset_ptr_t
;
138 typedef const bitset_word_t
*re_const_bitset_ptr_t
;
140 #define bitset_set(set,i) \
141 (set[i / BITSET_WORD_BITS] |= (bitset_word_t) 1 << i % BITSET_WORD_BITS)
142 #define bitset_clear(set,i) \
143 (set[i / BITSET_WORD_BITS] &= ~((bitset_word_t) 1 << i % BITSET_WORD_BITS))
144 #define bitset_contain(set,i) \
145 (set[i / BITSET_WORD_BITS] & ((bitset_word_t) 1 << i % BITSET_WORD_BITS))
146 #define bitset_empty(set) memset (set, '\0', sizeof (bitset_t))
147 #define bitset_set_all(set) memset (set, '\xff', sizeof (bitset_t))
148 #define bitset_copy(dest,src) memcpy (dest, src, sizeof (bitset_t))
150 #define PREV_WORD_CONSTRAINT 0x0001
151 #define PREV_NOTWORD_CONSTRAINT 0x0002
152 #define NEXT_WORD_CONSTRAINT 0x0004
153 #define NEXT_NOTWORD_CONSTRAINT 0x0008
154 #define PREV_NEWLINE_CONSTRAINT 0x0010
155 #define NEXT_NEWLINE_CONSTRAINT 0x0020
156 #define PREV_BEGBUF_CONSTRAINT 0x0040
157 #define NEXT_ENDBUF_CONSTRAINT 0x0080
158 #define WORD_DELIM_CONSTRAINT 0x0100
159 #define NOT_WORD_DELIM_CONSTRAINT 0x0200
163 INSIDE_WORD
= PREV_WORD_CONSTRAINT
| NEXT_WORD_CONSTRAINT
,
164 WORD_FIRST
= PREV_NOTWORD_CONSTRAINT
| NEXT_WORD_CONSTRAINT
,
165 WORD_LAST
= PREV_WORD_CONSTRAINT
| NEXT_NOTWORD_CONSTRAINT
,
166 INSIDE_NOTWORD
= PREV_NOTWORD_CONSTRAINT
| NEXT_NOTWORD_CONSTRAINT
,
167 LINE_FIRST
= PREV_NEWLINE_CONSTRAINT
,
168 LINE_LAST
= NEXT_NEWLINE_CONSTRAINT
,
169 BUF_FIRST
= PREV_BEGBUF_CONSTRAINT
,
170 BUF_LAST
= NEXT_ENDBUF_CONSTRAINT
,
171 WORD_DELIM
= WORD_DELIM_CONSTRAINT
,
172 NOT_WORD_DELIM
= NOT_WORD_DELIM_CONSTRAINT
186 /* Node type, These are used by token, node, tree. */
192 #ifdef RE_ENABLE_I18N
195 #endif /* RE_ENABLE_I18N */
197 /* We define EPSILON_BIT as a macro so that OP_OPEN_SUBEXP is used
198 when the debugger shows values of this enum type. */
199 #define EPSILON_BIT 8
200 OP_OPEN_SUBEXP
= EPSILON_BIT
| 0,
201 OP_CLOSE_SUBEXP
= EPSILON_BIT
| 1,
202 OP_ALT
= EPSILON_BIT
| 2,
203 OP_DUP_ASTERISK
= EPSILON_BIT
| 3,
204 ANCHOR
= EPSILON_BIT
| 4,
206 /* Tree type, these are used only by tree. */
210 /* Token type, these are used only by token. */
222 OP_CLOSE_EQUIV_CLASS
,
233 #ifdef RE_ENABLE_I18N
236 /* Multibyte characters. */
239 /* Collating symbols. */
244 /* Equivalence classes. */
246 int32_t *equiv_classes
;
249 /* Range expressions. */
251 uint32_t *range_starts
;
252 uint32_t *range_ends
;
253 # else /* not _LIBC */
254 wchar_t *range_starts
;
256 # endif /* not _LIBC */
258 /* Character classes. */
259 wctype_t *char_classes
;
261 /* If this character set is the non-matching list. */
262 unsigned int non_match
: 1;
264 /* # of multibyte characters. */
267 /* # of collating symbols. */
270 /* # of equivalence classes. */
273 /* # of range expressions. */
276 /* # of character classes. */
279 #endif /* RE_ENABLE_I18N */
285 unsigned char c
; /* for CHARACTER */
286 re_bitset_ptr_t sbcset
; /* for SIMPLE_BRACKET */
287 #ifdef RE_ENABLE_I18N
288 re_charset_t
*mbcset
; /* for COMPLEX_BRACKET */
289 #endif /* RE_ENABLE_I18N */
290 int idx
; /* for BACK_REF */
291 re_context_type ctx_type
; /* for ANCHOR */
294 re_token_type_t type
: 8;
296 re_token_type_t type
;
298 unsigned int constraint
: 10; /* context constraint */
299 unsigned int duplicated
: 1;
300 unsigned int opt_subexp
: 1;
301 #ifdef RE_ENABLE_I18N
302 unsigned int accept_mb
: 1;
303 /* These 2 bits can be moved into the union if needed (e.g. if running out
304 of bits; move opr.c to opr.c.c and move the flags to opr.c.flags). */
305 unsigned int mb_partial
: 1;
307 unsigned int word_char
: 1;
310 #define IS_EPSILON_NODE(type) ((type) & EPSILON_BIT)
314 /* Indicate the raw buffer which is the original string passed as an
315 argument of regexec(), re_search(), etc.. */
316 const unsigned char *raw_mbs
;
317 /* Store the multibyte string. In case of "case insensitive mode" like
318 REG_ICASE, upper cases of the string are stored, otherwise MBS points
319 the same address that RAW_MBS points. */
321 #ifdef RE_ENABLE_I18N
322 /* Store the wide character string which is corresponding to MBS. */
327 /* Index in RAW_MBS. Each character mbs[i] corresponds to
328 raw_mbs[raw_mbs_idx + i]. */
330 /* The length of the valid characters in the buffers. */
332 /* The corresponding number of bytes in raw_mbs array. */
334 /* The length of the buffers MBS and WCS. */
336 /* The index in MBS, which is updated by re_string_fetch_byte. */
338 /* length of RAW_MBS array. */
340 /* This is RAW_LEN - RAW_MBS_IDX + VALID_LEN - VALID_RAW_LEN. */
342 /* End of the buffer may be shorter than its length in the cases such
343 as re_match_2, re_search_2. Then, we use STOP for end of the buffer
346 /* This is RAW_STOP - RAW_MBS_IDX adjusted through OFFSETS. */
349 /* The context of mbs[0]. We store the context independently, since
350 the context of mbs[0] may be different from raw_mbs[0], which is
351 the beginning of the input string. */
352 unsigned int tip_context
;
353 /* The translation passed as a part of an argument of re_compile_pattern. */
354 RE_TRANSLATE_TYPE trans
;
355 /* Copy of re_dfa_t's word_char. */
356 re_const_bitset_ptr_t word_char
;
357 /* 1 if REG_ICASE. */
359 unsigned char is_utf8
;
360 unsigned char map_notascii
;
361 unsigned char mbs_allocated
;
362 unsigned char offsets_needed
;
363 unsigned char newline_anchor
;
364 unsigned char word_ops_used
;
367 typedef struct re_string_t re_string_t
;
371 typedef struct re_dfa_t re_dfa_t
;
375 # define internal_function __attribute__ ((regparm (3), stdcall))
377 # define internal_function
382 static reg_errcode_t
re_string_realloc_buffers (re_string_t
*pstr
,
385 # ifdef RE_ENABLE_I18N
386 static void build_wcs_buffer (re_string_t
*pstr
) internal_function
;
387 static reg_errcode_t
build_wcs_upper_buffer (re_string_t
*pstr
)
389 # endif /* RE_ENABLE_I18N */
390 static void build_upper_buffer (re_string_t
*pstr
) internal_function
;
391 static void re_string_translate_buffer (re_string_t
*pstr
) internal_function
;
392 static unsigned int re_string_context_at (const re_string_t
*input
, int idx
,
394 internal_function
__attribute__ ((pure
));
396 #define re_string_peek_byte(pstr, offset) \
397 ((pstr)->mbs[(pstr)->cur_idx + offset])
398 #define re_string_fetch_byte(pstr) \
399 ((pstr)->mbs[(pstr)->cur_idx++])
400 #define re_string_first_byte(pstr, idx) \
401 ((idx) == (pstr)->valid_len || (pstr)->wcs[idx] != WEOF)
402 #define re_string_is_single_byte_char(pstr, idx) \
403 ((pstr)->wcs[idx] != WEOF && ((pstr)->valid_len == (idx) + 1 \
404 || (pstr)->wcs[(idx) + 1] != WEOF))
405 #define re_string_eoi(pstr) ((pstr)->stop <= (pstr)->cur_idx)
406 #define re_string_cur_idx(pstr) ((pstr)->cur_idx)
407 #define re_string_get_buffer(pstr) ((pstr)->mbs)
408 #define re_string_length(pstr) ((pstr)->len)
409 #define re_string_byte_at(pstr,idx) ((pstr)->mbs[idx])
410 #define re_string_skip_bytes(pstr,idx) ((pstr)->cur_idx += (idx))
411 #define re_string_set_index(pstr,idx) ((pstr)->cur_idx = (idx))
417 /* The OS usually guarantees only one guard page at the bottom of the stack,
418 and a page size can be as small as 4096 bytes. So we cannot safely
419 allocate anything larger than 4096 bytes. Also care for the possibility
420 of a few compiler-allocated temporary stack slots. */
421 # define __libc_use_alloca(n) ((n) < 4032)
423 /* alloca is implemented with malloc, so just use malloc. */
424 # define __libc_use_alloca(n) 0
428 #define re_malloc(t,n) ((t *) malloc ((n) * sizeof (t)))
429 #define re_realloc(p,t,n) ((t *) realloc (p, (n) * sizeof (t)))
430 #define re_free(p) free (p)
434 struct bin_tree_t
*parent
;
435 struct bin_tree_t
*left
;
436 struct bin_tree_t
*right
;
437 struct bin_tree_t
*first
;
438 struct bin_tree_t
*next
;
442 /* `node_idx' is the index in dfa->nodes, if `type' == 0.
443 Otherwise `type' indicate the type of this node. */
446 typedef struct bin_tree_t bin_tree_t
;
448 #define BIN_TREE_STORAGE_SIZE \
449 ((1024 - sizeof (void *)) / sizeof (bin_tree_t))
451 struct bin_tree_storage_t
453 struct bin_tree_storage_t
*next
;
454 bin_tree_t data
[BIN_TREE_STORAGE_SIZE
];
456 typedef struct bin_tree_storage_t bin_tree_storage_t
;
458 #define CONTEXT_WORD 1
459 #define CONTEXT_NEWLINE (CONTEXT_WORD << 1)
460 #define CONTEXT_BEGBUF (CONTEXT_NEWLINE << 1)
461 #define CONTEXT_ENDBUF (CONTEXT_BEGBUF << 1)
463 #define IS_WORD_CONTEXT(c) ((c) & CONTEXT_WORD)
464 #define IS_NEWLINE_CONTEXT(c) ((c) & CONTEXT_NEWLINE)
465 #define IS_BEGBUF_CONTEXT(c) ((c) & CONTEXT_BEGBUF)
466 #define IS_ENDBUF_CONTEXT(c) ((c) & CONTEXT_ENDBUF)
467 #define IS_ORDINARY_CONTEXT(c) ((c) == 0)
469 #define IS_WORD_CHAR(ch) (isalnum (ch) || (ch) == '_')
470 #define IS_NEWLINE(ch) ((ch) == NEWLINE_CHAR)
471 #define IS_WIDE_WORD_CHAR(ch) (iswalnum (ch) || (ch) == L'_')
472 #define IS_WIDE_NEWLINE(ch) ((ch) == WIDE_NEWLINE_CHAR)
474 #define NOT_SATISFY_PREV_CONSTRAINT(constraint,context) \
475 ((((constraint) & PREV_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
476 || ((constraint & PREV_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
477 || ((constraint & PREV_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context))\
478 || ((constraint & PREV_BEGBUF_CONSTRAINT) && !IS_BEGBUF_CONTEXT (context)))
480 #define NOT_SATISFY_NEXT_CONSTRAINT(constraint,context) \
481 ((((constraint) & NEXT_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
482 || (((constraint) & NEXT_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
483 || (((constraint) & NEXT_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context)) \
484 || (((constraint) & NEXT_ENDBUF_CONSTRAINT) && !IS_ENDBUF_CONTEXT (context)))
490 re_node_set non_eps_nodes
;
491 re_node_set inveclosure
;
492 re_node_set
*entrance_nodes
;
493 struct re_dfastate_t
**trtable
, **word_trtable
;
494 unsigned int context
: 4;
495 unsigned int halt
: 1;
496 /* If this state can accept `multi byte'.
497 Note that we refer to multibyte characters, and multi character
498 collating elements as `multi byte'. */
499 unsigned int accept_mb
: 1;
500 /* If this state has backreference node(s). */
501 unsigned int has_backref
: 1;
502 unsigned int has_constraint
: 1;
504 typedef struct re_dfastate_t re_dfastate_t
;
506 struct re_state_table_entry
510 re_dfastate_t
**array
;
513 /* Array type used in re_sub_match_last_t and re_sub_match_top_t. */
519 re_dfastate_t
**array
;
522 /* Store information about the node NODE whose type is OP_CLOSE_SUBEXP. */
527 int str_idx
; /* The position NODE match at. */
529 } re_sub_match_last_t
;
531 /* Store information about the node NODE whose type is OP_OPEN_SUBEXP.
532 And information about the node, whose type is OP_CLOSE_SUBEXP,
533 corresponding to NODE is stored in LASTS. */
540 int alasts
; /* Allocation size of LASTS. */
541 int nlasts
; /* The number of LASTS. */
542 re_sub_match_last_t
**lasts
;
543 } re_sub_match_top_t
;
545 struct re_backref_cache_entry
553 unsigned short int eps_reachable_subexps_map
;
558 /* The string object corresponding to the input string. */
560 #if defined _LIBC || (defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L)
561 const re_dfa_t
*const dfa
;
565 /* EFLAGS of the argument of regexec. */
567 /* Where the matching ends. */
570 /* The state log used by the matcher. */
571 re_dfastate_t
**state_log
;
573 /* Back reference cache. */
576 struct re_backref_cache_entry
*bkref_ents
;
580 re_sub_match_top_t
**sub_tops
;
581 } re_match_context_t
;
585 re_dfastate_t
**sifted_states
;
586 re_dfastate_t
**limited_states
;
592 struct re_fail_stack_ent_t
597 re_node_set eps_via_nodes
;
600 struct re_fail_stack_t
604 struct re_fail_stack_ent_t
*stack
;
615 re_node_set
*eclosures
;
616 re_node_set
*inveclosures
;
617 struct re_state_table_entry
*state_table
;
618 re_dfastate_t
*init_state
;
619 re_dfastate_t
*init_state_word
;
620 re_dfastate_t
*init_state_nl
;
621 re_dfastate_t
*init_state_begbuf
;
622 bin_tree_t
*str_tree
;
623 bin_tree_storage_t
*str_tree_storage
;
624 re_bitset_ptr_t sb_char
;
625 int str_tree_storage_idx
;
627 /* number of subexpressions `re_nsub' is in regex_t. */
628 unsigned int state_hash_mask
;
630 int nbackref
; /* The number of backreference in this dfa. */
632 /* Bitmap expressing which backreference is used. */
633 bitset_word_t used_bkref_map
;
634 bitset_word_t completed_bkref_map
;
636 unsigned int has_plural_match
: 1;
637 /* If this dfa has "multibyte node", which is a backreference or
638 a node which can accept multibyte character or multi character
639 collating element. */
640 unsigned int has_mb_node
: 1;
641 unsigned int is_utf8
: 1;
642 unsigned int map_notascii
: 1;
643 unsigned int word_ops_used
: 1;
651 __libc_lock_define (, lock
)
654 #define re_node_set_init_empty(set) memset (set, '\0', sizeof (re_node_set))
655 #define re_node_set_remove(set,id) \
656 (re_node_set_remove_at (set, re_node_set_contains (set, id) - 1))
657 #define re_node_set_empty(p) ((p)->nelem = 0)
658 #define re_node_set_free(set) re_free ((set)->elems)
672 bracket_elem_type type
;
682 /* Inline functions for bitset operation. */
683 static void __attribute__ ((unused
))
684 bitset_not (bitset_t set
)
687 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
688 set
[bitset_i
] = ~set
[bitset_i
];
691 static void __attribute__ ((unused
))
692 bitset_merge (bitset_t dest
, const bitset_t src
)
695 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
696 dest
[bitset_i
] |= src
[bitset_i
];
699 static void __attribute__ ((unused
))
700 bitset_mask (bitset_t dest
, const bitset_t src
)
703 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
704 dest
[bitset_i
] &= src
[bitset_i
];
707 #ifdef RE_ENABLE_I18N
708 /* Inline functions for re_string. */
710 internal_function
__attribute__ ((pure
, unused
))
711 re_string_char_size_at (const re_string_t
*pstr
, int idx
)
714 if (pstr
->mb_cur_max
== 1)
716 for (byte_idx
= 1; idx
+ byte_idx
< pstr
->valid_len
; ++byte_idx
)
717 if (pstr
->wcs
[idx
+ byte_idx
] != WEOF
)
723 internal_function
__attribute__ ((pure
, unused
))
724 re_string_wchar_at (const re_string_t
*pstr
, int idx
)
726 if (pstr
->mb_cur_max
== 1)
727 return (wint_t) pstr
->mbs
[idx
];
728 return (wint_t) pstr
->wcs
[idx
];
733 internal_function
__attribute__ ((pure
, unused
))
734 re_string_elem_size_at (const re_string_t
*pstr
, int idx
)
737 const unsigned char *p
, *extra
;
738 const int32_t *table
, *indirect
;
739 # include <locale/weight.h>
740 uint_fast32_t nrules
= _NL_CURRENT_WORD (LC_COLLATE
, _NL_COLLATE_NRULES
);
744 table
= (const int32_t *) _NL_CURRENT (LC_COLLATE
, _NL_COLLATE_TABLEMB
);
745 extra
= (const unsigned char *)
746 _NL_CURRENT (LC_COLLATE
, _NL_COLLATE_EXTRAMB
);
747 indirect
= (const int32_t *) _NL_CURRENT (LC_COLLATE
,
748 _NL_COLLATE_INDIRECTMB
);
750 findidx (&p
, pstr
->len
- idx
);
751 return p
- pstr
->mbs
- idx
;
758 #endif /* RE_ENABLE_I18N */
760 #endif /* _REGEX_INTERNAL_H */