/* CPP Library - charsets
- Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2006, 2008, 2009
- Free Software Foundation, Inc.
+ Copyright (C) 1998-2020 Free Software Foundation, Inc.
Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
return EILSEQ;
}
- if (s < 0xFFFF)
+ if (s <= 0xFFFF)
{
if (*outbytesleftp < 2)
{
if (to->len + flen > to->asize)
{
to->asize = to->len + flen;
+ to->asize += to->asize / 4;
to->text = XRESIZEVEC (uchar, to->text, to->asize);
}
memcpy (to->text + to->len, from, flen);
#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
-struct conversion
+struct cpp_conversion
{
const char *pair;
convert_f func;
iconv_t fake_cd;
};
-static const struct conversion conversion_tab[] = {
+static const struct cpp_conversion conversion_tab[] = {
{ "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 },
{ "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 },
{ "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 },
\f
+/* cpp_substring_ranges's constructor. */
+
+cpp_substring_ranges::cpp_substring_ranges () :
+ m_ranges (NULL),
+ m_num_ranges (0),
+ m_alloc_ranges (8)
+{
+ m_ranges = XNEWVEC (source_range, m_alloc_ranges);
+}
+
+/* cpp_substring_ranges's destructor. */
+
+cpp_substring_ranges::~cpp_substring_ranges ()
+{
+ free (m_ranges);
+}
+
+/* Add RANGE to the vector of source_range information. */
+
+void
+cpp_substring_ranges::add_range (source_range range)
+{
+ if (m_num_ranges >= m_alloc_ranges)
+ {
+ m_alloc_ranges *= 2;
+ m_ranges
+ = (source_range *)xrealloc (m_ranges,
+ sizeof (source_range) * m_alloc_ranges);
+ }
+ m_ranges[m_num_ranges++] = range;
+}
+
+/* Read NUM ranges from LOC_READER, adding them to the vector of source_range
+ information. */
+
+void
+cpp_substring_ranges::add_n_ranges (int num,
+ cpp_string_location_reader &loc_reader)
+{
+ for (int i = 0; i < num; i++)
+ add_range (loc_reader.get_next ());
+}
+
+\f
+
/* Utility routine that computes a mask of the form 0000...111... with
WIDTH 1-bits. */
static inline size_t
/* Valid in a C99 identifier? */
C99 = 1,
/* Valid in a C99 identifier, but not as the first character? */
- DIG = 2,
+ N99 = 2,
/* Valid in a C++ identifier? */
CXX = 4,
+ /* Valid in a C11/C++11 identifier? */
+ C11 = 8,
+ /* Valid in a C11/C++11 identifier, but not as the first character? */
+ N11 = 16,
/* NFC representation is not valid in an identifier? */
- CID = 8,
+ CID = 32,
/* Might be valid NFC form? */
- NFC = 16,
+ NFC = 64,
/* Might be valid NFKC form? */
- NKC = 32,
+ NKC = 128,
/* Certain preceding characters might make it not valid NFC/NKFC form? */
- CTX = 64
+ CTX = 256
};
-static const struct {
+struct ucnrange {
/* Bitmap of flags above. */
- unsigned char flags;
+ unsigned short flags;
/* Combining class of the character. */
unsigned char combine;
/* Last character in the range described by this entry. */
- unsigned short end;
-} ucnranges[] = {
-#include "ucnid.h"
+ unsigned int end;
};
+#include "ucnid.h"
+
+/* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
+#define UCS_LIMIT 0x10FFFF
/* Returns 1 if C is valid in an identifier, 2 if C is valid except at
the start of an identifier, and 0 if C is not valid in an
struct normalize_state *nst)
{
int mn, mx, md;
+ unsigned short valid_flags, invalid_start_flags;
- if (c > 0xFFFF)
+ if (c > UCS_LIMIT)
return 0;
mn = 0;
/* When -pedantic, we require the character to have been listed by
the standard for the current language. Otherwise, we accept the
- union of the acceptable sets for C++98 and C99. */
- if (! (ucnranges[mn].flags & (C99 | CXX)))
+ union of the acceptable sets for all supported language versions. */
+ valid_flags = C99 | CXX | C11;
+ if (CPP_PEDANTIC (pfile))
+ {
+ if (CPP_OPTION (pfile, c11_identifiers))
+ valid_flags = C11;
+ else if (CPP_OPTION (pfile, c99))
+ valid_flags = C99;
+ else if (CPP_OPTION (pfile, cplusplus))
+ valid_flags = CXX;
+ }
+ if (! (ucnranges[mn].flags & valid_flags))
return 0;
-
- if (CPP_PEDANTIC (pfile)
- && ((CPP_OPTION (pfile, c99) && !(ucnranges[mn].flags & C99))
- || (CPP_OPTION (pfile, cplusplus)
- && !(ucnranges[mn].flags & CXX))))
- return 0;
+ if (CPP_OPTION (pfile, c11_identifiers))
+ invalid_start_flags = N11;
+ else if (CPP_OPTION (pfile, c99))
+ invalid_start_flags = N99;
+ else
+ invalid_start_flags = 0;
/* Update NST. */
if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
bool safe;
cppchar_t p = nst->previous;
- /* Easy cases from Bengali, Oriya, Tamil, Jannada, and Malayalam. */
- if (c == 0x09BE)
- safe = p != 0x09C7; /* Use 09CB instead of 09C7 09BE. */
- else if (c == 0x0B3E)
- safe = p != 0x0B47; /* Use 0B4B instead of 0B47 0B3E. */
- else if (c == 0x0BBE)
- safe = p != 0x0BC6 && p != 0x0BC7; /* Use 0BCA/0BCB instead. */
- else if (c == 0x0CC2)
- safe = p != 0x0CC6; /* Use 0CCA instead of 0CC6 0CC2. */
- else if (c == 0x0D3E)
- safe = p != 0x0D46 && p != 0x0D47; /* Use 0D4A/0D4B instead. */
/* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
and are combined algorithmically from a sequence of the form
1100-1112 1161-1175 11A8-11C2
really a valid character).
Unfortunately, C99 allows (only) the NFC form, but C++ allows
only the combining characters. */
- else if (c >= 0x1161 && c <= 0x1175)
+ if (c >= 0x1161 && c <= 0x1175)
safe = p < 0x1100 || p > 0x1112;
else if (c >= 0x11A8 && c <= 0x11C2)
safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
else
+ safe = check_nfc (pfile, c, p);
+ if (!safe)
{
- /* Uh-oh, someone updated ucnid.h without updating this code. */
- cpp_error (pfile, CPP_DL_ICE, "Character %x might not be NFKC", c);
- safe = true;
+ if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2))
+ nst->level = MAX (nst->level, normalized_identifier_C);
+ else
+ nst->level = normalized_none;
}
- if (!safe && c < 0x1161)
- nst->level = normalized_none;
- else if (!safe)
- nst->level = MAX (nst->level, normalized_identifier_C);
}
else if (ucnranges[mn].flags & NKC)
;
nst->level = MAX (nst->level, normalized_identifier_C);
else
nst->level = normalized_none;
- nst->previous = c;
+ if (ucnranges[mn].combine == 0)
+ nst->previous = c;
nst->prev_class = ucnranges[mn].combine;
- /* In C99, UCN digits may not begin identifiers. */
- if (CPP_OPTION (pfile, c99) && (ucnranges[mn].flags & DIG))
+ /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
+ UCN combining characters may not begin identifiers. */
+ if (ucnranges[mn].flags & invalid_start_flags)
return 2;
return 1;
whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
or 0060 (`), nor one in the range D800 through DFFF inclusive.
+ If the hexadecimal value is larger than the upper bound of the UCS
+ codespace specified in ISO/IEC 10646, a pedantic warning is issued
+ in all versions of C and in the C++20 or later versions of C++.
+
*PSTR must be preceded by "\u" or "\U"; it is assumed that the
- buffer end is delimited by a non-hex digit. Returns zero if the
- UCN has not been consumed.
+ buffer end is delimited by a non-hex digit. Returns false if the
+ UCN has not been consumed, true otherwise.
- Otherwise the nonzero value of the UCN, whether valid or invalid,
- is returned. Diagnostics are emitted for invalid values. PSTR
- is updated to point one beyond the UCN, or to the syntactically
- invalid character.
+ The value of the UCN, whether valid or invalid, is returned in *CP.
+ Diagnostics are emitted for invalid values. PSTR is updated to point
+ one beyond the UCN, or to the syntactically invalid character.
IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
- an identifier, or 2 otherwise. */
+ an identifier, or 2 otherwise.
-cppchar_t
+ If LOC_READER is non-NULL, then position information is
+ read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
+
+bool
_cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
const uchar *limit, int identifier_pos,
- struct normalize_state *nst)
+ struct normalize_state *nst, cppchar_t *cp,
+ source_range *char_range,
+ cpp_string_location_reader *loc_reader)
{
cppchar_t result, c;
unsigned int length;
if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
cpp_error (pfile, CPP_DL_WARNING,
"universal character names are only valid in C++ and C99");
- else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
+ else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
+ && !CPP_OPTION (pfile, cplusplus))
cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\%c' is different in traditional C",
- (int) str[-1]);
+ "C99's universal character names are incompatible with C90");
+ else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\%c' is different in traditional C",
+ (int) str[-1]);
if (str[-1] == 'u')
length = 4;
if (!ISXDIGIT (c))
break;
str++;
+ if (loc_reader)
+ {
+ gcc_assert (char_range);
+ char_range->m_finish = loc_reader->get_next ().m_finish;
+ }
result = (result << 4) + hex_value (c);
}
while (--length && str < limit);
multiple tokens in identifiers, so we can't give a helpful
error message in that case. */
if (length && identifier_pos)
- return 0;
-
+ {
+ *cp = 0;
+ return false;
+ }
+
*pstr = str;
if (length)
{
CPP_OPTION (pfile, warn_dollars) = 0;
cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
}
- NORMALIZE_STATE_UPDATE_IDNUM (nst);
+ NORMALIZE_STATE_UPDATE_IDNUM (nst, result);
}
else if (identifier_pos)
{
"universal character %.*s is not valid at the start of an identifier",
(int) (str - base), base);
}
-
- if (result == 0)
- result = 1;
-
- return result;
+ else if (result > UCS_LIMIT
+ && (!CPP_OPTION (pfile, cplusplus)
+ || CPP_OPTION (pfile, lang) > CLK_CXX17))
+ cpp_error (pfile, CPP_DL_PEDWARN,
+ "%.*s is outside the UCS codespace",
+ (int) (str - base), base);
+
+ *cp = result;
+ return true;
}
/* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
- it to the execution character set and write the result into TBUF.
- An advanced pointer is returned. Issues all relevant diagnostics. */
+ it to the execution character set and write the result into TBUF,
+ if TBUF is non-NULL.
+ An advanced pointer is returned. Issues all relevant diagnostics.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
cppchar_t ucn;
uchar buf[6];
int rval;
struct normalize_state nst = INITIAL_NORMALIZE_STATE;
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
from++; /* Skip u/U. */
- ucn = _cpp_valid_ucn (pfile, &from, limit, 0, &nst);
+
+ if (loc_reader)
+ /* The u/U is part of the spelling of this character. */
+ char_range.m_finish = loc_reader->get_next ().m_finish;
+
+ _cpp_valid_ucn (pfile, &from, limit, 0, &nst,
+ &ucn, &char_range, loc_reader);
rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
if (rval)
cpp_errno (pfile, CPP_DL_ERROR,
"converting UCN to source character set");
}
- else if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
- cpp_errno (pfile, CPP_DL_ERROR,
- "converting UCN to execution character set");
+ else
+ {
+ if (tbuf)
+ if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
+ cpp_errno (pfile, CPP_DL_ERROR,
+ "converting UCN to execution character set");
+
+ if (loc_reader)
+ {
+ int num_encoded_bytes = 6 - bytesleft;
+ for (int i = 0; i < num_encoded_bytes; i++)
+ ranges->add_range (char_range);
+ }
+ }
return from;
}
+/* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
+ extended characters rather than UCNs. If the return value is TRUE, then a
+ character was successfully decoded and stored in *CP; *PSTR has been
+ updated to point one past the valid UTF-8 sequence. Diagnostics may have
+ been emitted if the character parsed is not allowed in the current context.
+ If the return value is FALSE, then *PSTR has not been modified and *CP may
+ equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
+ may, when processing an identifier in C mode, equal a codepoint that was
+ validly encoded but is not allowed to appear in an identifier. In either
+ case, no diagnostic is emitted, and the return value of FALSE should cause
+ a new token to be formed.
+
+ Unlike _cpp_valid_ucn, this will never be called when lexing a string; only
+ a potential identifier, or a CPP_OTHER token. NST is unused in the latter
+ case.
+
+ As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
+ the start of an identifier, or 2 otherwise. */
+
+extern bool
+_cpp_valid_utf8 (cpp_reader *pfile,
+ const uchar **pstr,
+ const uchar *limit,
+ int identifier_pos,
+ struct normalize_state *nst,
+ cppchar_t *cp)
+{
+ const uchar *base = *pstr;
+ size_t inbytesleft = limit - base;
+ if (one_utf8_to_cppchar (pstr, &inbytesleft, cp))
+ {
+ /* No diagnostic here as this byte will rather become a
+ new token. */
+ *cp = 0;
+ return false;
+ }
+
+ if (identifier_pos)
+ {
+ switch (ucn_valid_in_identifier (pfile, *cp, nst))
+ {
+
+ case 0:
+ /* In C++, this is an error for invalid character in an identifier
+ because logically, the UTF-8 was converted to a UCN during
+ translation phase 1 (even though we don't physically do it that
+ way). In C, this byte rather becomes grammatically a separate
+ token. */
+
+ if (CPP_OPTION (pfile, cplusplus))
+ cpp_error (pfile, CPP_DL_ERROR,
+ "extended character %.*s is not valid in an identifier",
+ (int) (*pstr - base), base);
+ else
+ {
+ *pstr = base;
+ return false;
+ }
+
+ break;
+
+ case 2:
+ if (identifier_pos == 1)
+ {
+ /* This is treated the same way in C++ or C99 -- lexed as an
+ identifier which is then invalid because an identifier is
+ not allowed to start with this character. */
+ cpp_error (pfile, CPP_DL_ERROR,
+ "extended character %.*s is not valid at the start of an identifier",
+ (int) (*pstr - base), base);
+ }
+ break;
+ }
+ }
+
+ return true;
+}
+
/* Subroutine of convert_hex and convert_oct. N is the representation
in the execution character set of a numeric escape; write it into the
string buffer TBUF and update the end-of-string pointer therein. WIDE
}
/* Convert a hexadecimal escape, pointed to by FROM, to the execution
- character set and write it into the string buffer TBUF. Returns an
- advanced pointer, and issues diagnostics as necessary.
+ character set and write it into the string buffer TBUF (if non-NULL).
+ Returns an advanced pointer, and issues diagnostics as necessary.
No character set translation occurs; this routine always produces the
execution-set character with numeric value equal to the given hex
- number. You can, e.g. generate surrogate pairs this way. */
+ number. You can, e.g. generate surrogate pairs this way.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
cppchar_t c, n = 0, overflow = 0;
int digits_found = 0;
size_t width = cvt.width;
size_t mask = width_to_mask (width);
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
if (CPP_WTRADITIONAL (pfile))
- cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\x' is different in traditional C");
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\x' is different in traditional C");
+
+ /* Skip 'x'. */
+ from++;
+
+ /* The 'x' is part of the spelling of this character. */
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
- from++; /* Skip 'x'. */
while (from < limit)
{
c = *from;
if (! hex_p (c))
break;
from++;
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
overflow |= n ^ (n << 4 >> 4);
n = (n << 4) + hex_value (c);
digits_found = 1;
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (tbuf)
+ emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (ranges)
+ ranges->add_range (char_range);
return from;
}
advanced pointer, and issues diagnostics as necessary.
No character set translation occurs; this routine always produces the
execution-set character with numeric value equal to the given octal
- number. */
+ number.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
+ contains the location of the character so far: location information
+ is read from *LOC_READER, and *RANGES is updated accordingly. */
static const uchar *
convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ source_range char_range,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
size_t count = 0;
cppchar_t c, n = 0;
size_t mask = width_to_mask (width);
bool overflow = false;
+ /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_reader != NULL) == (ranges != NULL));
+
while (from < limit && count++ < 3)
{
c = *from;
if (c < '0' || c > '7')
break;
from++;
+ if (loc_reader)
+ char_range.m_finish = loc_reader->get_next ().m_finish;
overflow |= n ^ (n << 3 >> 3);
n = (n << 3) + c - '0';
}
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (tbuf)
+ emit_numeric_escape (pfile, n, tbuf, cvt);
+ if (ranges)
+ ranges->add_range (char_range);
return from;
}
/* Convert an escape sequence (pointed to by FROM) to its value on
the target, and to the execution character set. Do not scan past
- LIMIT. Write the converted value into TBUF. Returns an advanced
- pointer. Handles all relevant diagnostics. */
+ LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
+ Returns an advanced pointer. Handles all relevant diagnostics.
+ If LOC_READER is non-NULL, then RANGES must be non-NULL: location
+ information is read from *LOC_READER, and *RANGES is updated
+ accordingly. */
static const uchar *
convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, struct cset_converter cvt)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt,
+ cpp_string_location_reader *loc_reader,
+ cpp_substring_ranges *ranges)
{
/* Values of \a \b \e \f \n \r \t \v respectively. */
#if HOST_CHARSET == HOST_CHARSET_ASCII
uchar c;
+ /* Record the location of the backslash. */
+ source_range char_range;
+ if (loc_reader)
+ char_range = loc_reader->get_next ();
+
c = *from;
switch (c)
{
/* UCNs, hex escapes, and octal escapes are processed separately. */
case 'u': case 'U':
- return convert_ucn (pfile, from, limit, tbuf, cvt);
+ return convert_ucn (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
case 'x':
- return convert_hex (pfile, from, limit, tbuf, cvt);
+ return convert_hex (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
break;
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7':
- return convert_oct (pfile, from, limit, tbuf, cvt);
+ return convert_oct (pfile, from, limit, tbuf, cvt,
+ char_range, loc_reader, ranges);
/* Various letter escapes. Get the appropriate host-charset
value into C. */
case 'a':
if (CPP_WTRADITIONAL (pfile))
- cpp_error (pfile, CPP_DL_WARNING,
- "the meaning of '\\a' is different in traditional C");
+ cpp_warning (pfile, CPP_W_TRADITIONAL,
+ "the meaning of '\\a' is different in traditional C");
c = charconsts[0];
break;
}
}
- /* Now convert what we have to the execution character set. */
- if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
- cpp_errno (pfile, CPP_DL_ERROR,
- "converting escape sequence to execution character set");
+ if (tbuf)
+ /* Now convert what we have to the execution character set. */
+ if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
+ cpp_errno (pfile, CPP_DL_ERROR,
+ "converting escape sequence to execution character set");
+
+ if (loc_reader)
+ {
+ char_range.m_finish = loc_reader->get_next ().m_finish;
+ ranges->add_range (char_range);
+ }
return from + 1;
}
{
default:
return pfile->narrow_cset_desc;
+ case CPP_UTF8CHAR:
case CPP_UTF8STRING:
return pfile->utf8_cset_desc;
case CPP_CHAR16:
are to be converted from the source to the execution character set,
escape sequences translated, and finally all are to be
concatenated. WIDE indicates whether or not to produce a wide
- string. The result is written into TO. Returns true for success,
- false for failure. */
-bool
-cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
- cpp_string *to, enum cpp_ttype type)
+ string. If TO is non-NULL, the result is written into TO.
+ If LOC_READERS and OUT are non-NULL, then location information
+ is read from LOC_READERS (which must be an array of length COUNT),
+ and location information is written to *RANGES.
+
+ Returns true for success, false for failure. */
+
+static bool
+cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count,
+ cpp_string *to, enum cpp_ttype type,
+ cpp_string_location_reader *loc_readers,
+ cpp_substring_ranges *out)
{
struct _cpp_strbuf tbuf;
const uchar *p, *base, *limit;
size_t i;
struct cset_converter cvt = converter_for_type (pfile, type);
- tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
- tbuf.text = XNEWVEC (uchar, tbuf.asize);
- tbuf.len = 0;
+ /* loc_readers and out must either be both NULL, or both be non-NULL. */
+ gcc_assert ((loc_readers != NULL) == (out != NULL));
+ if (to)
+ {
+ tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
+ tbuf.text = XNEWVEC (uchar, tbuf.asize);
+ tbuf.len = 0;
+ }
+
+ cpp_string_location_reader *loc_reader = NULL;
for (i = 0; i < count; i++)
{
+ if (loc_readers)
+ loc_reader = &loc_readers[i];
+
p = from[i].text;
if (*p == 'u')
{
- if (*++p == '8')
- p++;
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ if (*p == '8')
+ {
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ }
}
else if (*p == 'L' || *p == 'U') p++;
if (*p == 'R')
/* Skip over 'R"'. */
p += 2;
+ if (loc_reader)
+ {
+ loc_reader->get_next ();
+ loc_reader->get_next ();
+ }
prefix = p;
while (*p != '(')
- p++;
+ {
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+ }
p++;
+ if (loc_reader)
+ loc_reader->get_next ();
limit = from[i].text + from[i].len;
if (limit >= p + (p - prefix) + 1)
limit -= (p - prefix) + 1;
- for (;;)
- {
- base = p;
- while (p < limit && (*p != '\\' || (p[1] != 'u' && p[1] != 'U')))
- p++;
- if (p > base)
- {
- /* We have a run of normal characters; these can be fed
- directly to convert_cset. */
- if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
- goto fail;
- }
- if (p == limit)
- break;
+ /* Raw strings are all normal characters; these can be fed
+ directly to convert_cset. */
+ if (to)
+ if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
+ goto fail;
- p = convert_ucn (pfile, p + 1, limit, &tbuf, cvt);
+ if (loc_reader)
+ {
+ /* If generating source ranges, assume we have a 1:1
+ correspondence between bytes in the source encoding and bytes
+ in the execution encoding (e.g. if we have a UTF-8 to UTF-8
+ conversion), so that this run of bytes in the source file
+ corresponds to a run of bytes in the execution string.
+ This requirement is guaranteed by an early-reject in
+ cpp_interpret_string_ranges. */
+ gcc_assert (cvt.func == convert_no_conversion);
+ out->add_n_ranges (limit - p, *loc_reader);
}
continue;
}
- p++; /* Skip leading quote. */
+ /* If we don't now have a leading quote, something has gone wrong.
+ This can occur if cpp_interpret_string_ranges is handling a
+ stringified macro argument, but should not be possible otherwise. */
+ if (*p != '"' && *p != '\'')
+ {
+ gcc_assert (out != NULL);
+ cpp_error (pfile, CPP_DL_ERROR, "missing open quote");
+ if (to)
+ free (tbuf.text);
+ return false;
+ }
+
+ /* Skip leading quote. */
+ p++;
+ if (loc_reader)
+ loc_reader->get_next ();
+
limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
for (;;)
{
/* We have a run of normal characters; these can be fed
directly to convert_cset. */
- if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
- goto fail;
+ if (to)
+ if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
+ goto fail;
+ /* Similar to above: assumes we have a 1:1 correspondence
+ between bytes in the source encoding and bytes in the
+ execution encoding. */
+ if (loc_reader)
+ {
+ gcc_assert (cvt.func == convert_no_conversion);
+ out->add_n_ranges (p - base, *loc_reader);
+ }
}
- if (p == limit)
+ if (p >= limit)
break;
- p = convert_escape (pfile, p + 1, limit, &tbuf, cvt);
+ struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL;
+ p = convert_escape (pfile, p + 1, limit, tbuf_ptr, cvt,
+ loc_reader, out);
}
}
- /* NUL-terminate the 'to' buffer and translate it to a cpp_string
- structure. */
- emit_numeric_escape (pfile, 0, &tbuf, cvt);
- tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
- to->text = tbuf.text;
- to->len = tbuf.len;
+
+ if (to)
+ {
+ /* NUL-terminate the 'to' buffer and translate it to a cpp_string
+ structure. */
+ emit_numeric_escape (pfile, 0, &tbuf, cvt);
+ tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
+ to->text = tbuf.text;
+ to->len = tbuf.len;
+ }
+ /* Use the location of the trailing quote as the location of the
+ NUL-terminator. */
+ if (loc_reader)
+ {
+ source_range range = loc_reader->get_next ();
+ out->add_range (range);
+ }
+
return true;
fail:
cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set");
- free (tbuf.text);
+ if (to)
+ free (tbuf.text);
return false;
}
+/* FROM is an array of cpp_string structures of length COUNT. These
+ are to be converted from the source to the execution character set,
+ escape sequences translated, and finally all are to be
+ concatenated. WIDE indicates whether or not to produce a wide
+ string. The result is written into TO. Returns true for success,
+ false for failure. */
+bool
+cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
+ cpp_string *to, enum cpp_ttype type)
+{
+ return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL);
+}
+
+/* A "do nothing" diagnostic-handling callback for use by
+ cpp_interpret_string_ranges, so that it can temporarily suppress
+ diagnostic-handling. */
+
+static bool
+noop_diagnostic_cb (cpp_reader *, enum cpp_diagnostic_level,
+ enum cpp_warning_reason, rich_location *,
+ const char *, va_list *)
+{
+ /* no-op. */
+ return true;
+}
+
+/* This function mimics the behavior of cpp_interpret_string, but
+ rather than generating a string in the execution character set,
+ *OUT is written to with the source code ranges of the characters
+ in such a string.
+ FROM and LOC_READERS should both be arrays of length COUNT.
+ Returns NULL for success, or an error message for failure. */
+
+const char *
+cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
+ cpp_string_location_reader *loc_readers,
+ size_t count,
+ cpp_substring_ranges *out,
+ enum cpp_ttype type)
+{
+ /* There are a couple of cases in the range-handling in
+ cpp_interpret_string_1 that rely on there being a 1:1 correspondence
+ between bytes in the source encoding and bytes in the execution
+ encoding, so that each byte in the execution string can correspond
+ to the location of a byte in the source string.
+
+ This holds for the typical case of a UTF-8 to UTF-8 conversion.
+ Enforce this requirement by only attempting to track substring
+ locations if we have source encoding == execution encoding.
+
+ This is a stronger condition than we need, since we could e.g.
+ have ASCII to EBCDIC (with 1 byte per character before and after),
+ but it seems to be a reasonable restriction. */
+ struct cset_converter cvt = converter_for_type (pfile, type);
+ if (cvt.func != convert_no_conversion)
+ return "execution character set != source character set";
+
+ /* For on-demand strings we have already lexed the strings, so there
+ should be no diagnostics. However, if we have bogus source location
+ data (or stringified macro arguments), the attempt to lex the
+ strings could fail with an diagnostic. Temporarily install an
+ diagnostic-handler to catch the diagnostic, so that it can lead to this call
+ failing, rather than being emitted as a user-visible diagnostic.
+ If an diagnostic does occur, we should see it via the return value of
+ cpp_interpret_string_1. */
+ bool (*saved_diagnostic_handler) (cpp_reader *, enum cpp_diagnostic_level,
+ enum cpp_warning_reason, rich_location *,
+ const char *, va_list *)
+ ATTRIBUTE_FPTR_PRINTF(5,0);
+
+ saved_diagnostic_handler = pfile->cb.diagnostic;
+ pfile->cb.diagnostic = noop_diagnostic_cb;
+
+ bool result = cpp_interpret_string_1 (pfile, from, count, NULL, type,
+ loc_readers, out);
+
+ /* Restore the saved diagnostic-handler. */
+ pfile->cb.diagnostic = saved_diagnostic_handler;
+
+ if (!result)
+ return "cpp_interpret_string_1 failed";
+
+ /* Success. */
+ return NULL;
+}
+
/* Subroutine of do_line and do_linemarker. Convert escape sequences
in a string, but do not perform character set conversion. */
bool
/* Subroutine of cpp_interpret_charconst which performs the conversion
to a number, for narrow strings. STR is the string structure returned
by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
- cpp_interpret_charconst. */
+ cpp_interpret_charconst. TYPE is the token type. */
static cppchar_t
narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
- unsigned int *pchars_seen, int *unsignedp)
+ unsigned int *pchars_seen, int *unsignedp,
+ enum cpp_ttype type)
{
size_t width = CPP_OPTION (pfile, char_precision);
size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
result = c;
}
+ if (type == CPP_UTF8CHAR)
+ max_chars = 1;
if (i > max_chars)
{
i = max_chars;
- cpp_error (pfile, CPP_DL_WARNING,
+ cpp_error (pfile, type == CPP_UTF8CHAR ? CPP_DL_ERROR : CPP_DL_WARNING,
"character constant too long for its type");
}
else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
- cpp_error (pfile, CPP_DL_WARNING, "multi-character character constant");
+ cpp_warning (pfile, CPP_W_MULTICHAR, "multi-character character constant");
/* Multichar constants are of type int and therefore signed. */
if (i > 1)
unsigned_p = 0;
+ else if (type == CPP_UTF8CHAR && !CPP_OPTION (pfile, cplusplus))
+ unsigned_p = 1;
else
unsigned_p = CPP_OPTION (pfile, unsigned_char);
size_t off, i;
cppchar_t result = 0, c;
+ if (str.len <= nbwc)
+ {
+ /* Error recovery, if no errors have been diagnosed previously,
+ there should be at least two wide characters. Empty literals
+ are diagnosed earlier and we can get just the zero terminator
+ only if there were errors diagnosed during conversion. */
+ *pchars_seen = 0;
+ *unsignedp = 0;
+ return 0;
+ }
+
/* This is finicky because the string is in the target's byte order,
which may not be our byte order. Only the last character, ignoring
the NUL terminator, is relevant. */
character exactly fills a wchar_t, so a multi-character wide
character constant is guaranteed to overflow. */
if (str.len > nbwc * 2)
- cpp_error (pfile, CPP_DL_WARNING,
+ cpp_error (pfile, (CPP_OPTION (pfile, cplusplus)
+ && (type == CPP_CHAR16 || type == CPP_CHAR32))
+ ? CPP_DL_ERROR : CPP_DL_WARNING,
"character constant too long for its type");
/* Truncate the constant to its natural width, and simultaneously
unsigned int *pchars_seen, int *unsignedp)
{
cpp_string str = { 0, 0 };
- bool wide = (token->type != CPP_CHAR);
+ bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR);
+ int u8 = 2 * int(token->type == CPP_UTF8CHAR);
cppchar_t result;
- /* an empty constant will appear as L'', u'', U'' or '' */
- if (token->val.str.len == (size_t) (2 + wide))
+ /* An empty constant will appear as L'', u'', U'', u8'', or '' */
+ if (token->val.str.len == (size_t) (2 + wide + u8))
{
cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
+ *pchars_seen = 0;
+ *unsignedp = 0;
+ return 0;
+ }
+ else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str,
+ token->type))
+ {
+ *pchars_seen = 0;
+ *unsignedp = 0;
return 0;
}
- else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, token->type))
- return 0;
if (wide)
result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
token->type);
else
- result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
+ result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp,
+ token->type);
if (str.text != token->val.str.text)
free ((void *)str.text);
}
\f
/* Convert an identifier denoted by ID and LEN, which might contain
- UCN escapes, to the source character set, either UTF-8 or
- UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
+ UCN escapes or UTF-8 multibyte chars, to the source character set,
+ either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
+ a valid identifier. */
cpp_hashnode *
_cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
{
iconv_close (input_cset.cd);
/* Resize buffer if we allocated substantially too much, or if we
- haven't enough space for the \n-terminator. */
- if (to.len + 4096 < to.asize || to.len >= to.asize)
- to.text = XRESIZEVEC (uchar, to.text, to.len + 1);
+ haven't enough space for the \n-terminator or following
+ 15 bytes of padding (used to quiet warnings from valgrind or
+ Address Sanitizer, when the optimized lexer accesses aligned
+ 16-byte memory chunks, including the bytes after the malloced,
+ area, and stops lexing on '\n'). */
+ if (to.len + 4096 < to.asize || to.len + 16 > to.asize)
+ to.text = XRESIZEVEC (uchar, to.text, to.len + 16);
+
+ memset (to.text + to.len, '\0', 16);
/* If the file is using old-school Mac line endings (\r only),
terminate with another \r, not an \n, so that we do not mistake
return current_encoding;
}
+
+/* Implementation of class cpp_string_location_reader. */
+
+/* Constructor for cpp_string_location_reader. */
+
+cpp_string_location_reader::
+cpp_string_location_reader (location_t src_loc,
+ line_maps *line_table)
+{
+ src_loc = get_range_from_loc (line_table, src_loc).m_start;
+
+ /* SRC_LOC might be a macro location. It only makes sense to do
+ column-by-column calculations on ordinary maps, so get the
+ corresponding location in an ordinary map. */
+ m_loc
+ = linemap_resolve_location (line_table, src_loc,
+ LRK_SPELLING_LOCATION, NULL);
+
+ const line_map_ordinary *map
+ = linemap_check_ordinary (linemap_lookup (line_table, m_loc));
+ m_offset_per_column = (1 << map->m_range_bits);
+}
+
+/* Get the range of the next source byte. */
+
+source_range
+cpp_string_location_reader::get_next ()
+{
+ source_range result;
+ result.m_start = m_loc;
+ result.m_finish = m_loc;
+ if (m_loc <= LINE_MAP_MAX_LOCATION_WITH_COLS)
+ m_loc += m_offset_per_column;
+ return result;
+}
+
+/* Helper for cpp_byte_column_to_display_column and its inverse. Given a
+ pointer to a UTF-8-encoded character, compute its display width. *INBUFP
+ points on entry to the start of the UTF-8 encoding of the character, and
+ is updated to point just after the last byte of the encoding. *INBYTESLEFTP
+ contains on entry the remaining size of the buffer into which *INBUFP
+ points, and this is also updated accordingly. If *INBUFP does not
+ point to a valid UTF-8-encoded sequence, then it will be treated as a single
+ byte with display width 1. */
+
+static inline int
+compute_next_display_width (const uchar **inbufp, size_t *inbytesleftp)
+{
+ cppchar_t c;
+ if (one_utf8_to_cppchar (inbufp, inbytesleftp, &c) != 0)
+ {
+ /* Input is not convertible to UTF-8. This could be fine, e.g. in a
+ string literal, so don't complain. Just treat it as if it has a width
+ of one. */
+ ++*inbufp;
+ --*inbytesleftp;
+ return 1;
+ }
+
+ /* one_utf8_to_cppchar() has updated inbufp and inbytesleftp for us. */
+ return cpp_wcwidth (c);
+}
+
+/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
+ how many display columns are occupied by the first COLUMN bytes. COLUMN
+ may exceed DATA_LENGTH, in which case the phantom bytes at the end are
+ treated as if they have display width 1. */
+
+int
+cpp_byte_column_to_display_column (const char *data, int data_length,
+ int column)
+{
+ int display_col = 0;
+ const uchar *udata = (const uchar *) data;
+ const int offset = MAX (0, column - data_length);
+ size_t inbytesleft = column - offset;
+ while (inbytesleft)
+ display_col += compute_next_display_width (&udata, &inbytesleft);
+ return display_col + offset;
+}
+
+/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
+ the least number of bytes that will result in at least DISPLAY_COL display
+ columns. The return value may exceed DATA_LENGTH if the entire string does
+ not occupy enough display columns. */
+
+int
+cpp_display_column_to_byte_column (const char *data, int data_length,
+ int display_col)
+{
+ int column = 0;
+ const uchar *udata = (const uchar *) data;
+ size_t inbytesleft = data_length;
+ while (column < display_col && inbytesleft)
+ column += compute_next_display_width (&udata, &inbytesleft);
+ return data_length - inbytesleft + MAX (0, display_col - column);
+}
+
+/* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
+ because that will inspect the user's locale, and in particular in an ASCII
+ locale, it will not return anything useful for extended characters. But GCC
+ in other respects (see e.g. _cpp_default_encoding()) behaves as if
+ everything is UTF-8. We also make some tweaks that are useful for the way
+ GCC needs to use this data, e.g. tabs and other control characters should be
+ treated as having width 1. The lookup tables are generated from
+ contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
+ wcwidth() on all codepoints, then applying the small tweaks. These tables
+ are not highly optimized, but for the present purpose of outputting
+ diagnostics, they are sufficient. */
+
+#include "generated_cpp_wcwidth.h"
+int cpp_wcwidth (cppchar_t c)
+{
+ if (__builtin_expect (c <= wcwidth_range_ends[0], true))
+ return wcwidth_widths[0];
+
+ /* Binary search the tables. */
+ int begin = 1;
+ static const int end
+ = sizeof wcwidth_range_ends / sizeof (*wcwidth_range_ends);
+ int len = end - begin;
+ do
+ {
+ int half = len/2;
+ int middle = begin + half;
+ if (c > wcwidth_range_ends[middle])
+ {
+ begin = middle + 1;
+ len -= half + 1;
+ }
+ else
+ len = half;
+ } while (len);
+
+ if (__builtin_expect (begin != end, true))
+ return wcwidth_widths[begin];
+ return 1;
+}
projects / thirdparty / gcc.git / blobdiff