Attached you find an update of the Unicode implementation.
The patch is against the current CVS version. I would appreciate
if someone with CVS checkin permissions could check the changes
in.
The patch contains all bugs and patches sent this week and also
fixes a leak in the codecs code and a bug in the free list code
for Unicode objects (which only shows up when compiling Python
with Py_DEBUG; thanks to MarkH for spotting this one).
#ifndef Py_UNICODEOBJECT_H
#define Py_UNICODEOBJECT_H
-#ifdef __cplusplus
-extern "C" {
-#endif
/*
/* --- Internal Unicode Operations ---------------------------------------- */
/* If you want Python to use the compiler's wctype.h functions instead
- of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS.
- This reduces the interpreter's code size. */
+ of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
+ configure Python using --with-ctype-functions. This reduces the
+ interpreter's code size. */
#if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
(!memcmp((string)->str + (offset), (substring)->str,\
(substring)->length*sizeof(Py_UNICODE)))
+#ifdef __cplusplus
+extern "C" {
+#endif
+
/* --- Unicode Type ------------------------------------------------------- */
typedef struct {
int direction /* Find direction: +1 forward, -1 backward */
);
-/* Count the number of occurances of substr in str[start:end]. */
+/* Count the number of occurrences of substr in str[start:end]. */
extern DL_IMPORT(int) PyUnicode_Count(
PyObject *str, /* String */
int end /* Stop index */
);
-/* Replace at most maxcount occurances of substr in str with replstr
+/* Replace at most maxcount occurrences of substr in str with replstr
and return the resulting Unicode object. */
extern DL_IMPORT(PyObject *) PyUnicode_Replace(
import string,codecs,aliases
_cache = {}
-_unkown = '--unkown--'
+_unknown = '--unknown--'
def search_function(encoding):
# Cache lookup
- entry = _cache.get(encoding,_unkown)
- if entry is not _unkown:
+ entry = _cache.get(encoding,_unknown)
+ if entry is not _unknown:
return entry
# Import the module
test('translate', 'xyz', 'xyz', table)
test('replace', 'one!two!three!', 'one@two!three!', '!', '@', 1)
+test('replace', 'one!two!three!', 'onetwothree', '!', '')
test('replace', 'one!two!three!', 'one@two@three!', '!', '@', 2)
test('replace', 'one!two!three!', 'one@two@three@', '!', '@', 3)
test('replace', 'one!two!three!', 'one@two@three@', '!', '@', 4)
test('translate', u'xyz', u'xyz', table)
test('replace', u'one!two!three!', u'one@two!three!', u'!', u'@', 1)
+test('replace', u'one!two!three!', u'onetwothree', '!', '')
test('replace', u'one!two!three!', u'one@two@three!', u'!', u'@', 2)
test('replace', u'one!two!three!', u'one@two@three@', u'!', u'@', 3)
test('replace', u'one!two!three!', u'one@two@three@', u'!', u'@', 4)
stream codecs as available through the codecs module should
be used.
-XXX There should be a short-cut open(filename,mode,encoding) available which
- also assures that mode contains the 'b' character when needed.
+The codecs module should provide a short-cut open(filename,mode,encoding)
+available which also assures that mode contains the 'b' character when
+needed.
File/Stream Input:
Introduction to Unicode (a little outdated by still nice to read):
http://www.nada.kth.se/i18n/ucs/unicode-iso10646-oview.html
+For comparison:
+ Introducing Unicode to ECMAScript --
+ http://www-4.ibm.com/software/developer/library/internationalization-support.html
+
Encodings:
Overview:
History of this Proposal:
-------------------------
-1.2:
+1.2: Removed POD about codecs.open()
1.1: Added note about comparisons and hash values. Added note about
case mapping algorithms. Changed stream codecs .read() and
.write() method to match the standard file-like object methods
strstr replacement for arbitrary blocks of memory.
- Locates the first occurance in the memory pointed to by MEM of the
+ Locates the first occurrence in the memory pointed to by MEM of the
contents of memory pointed to by PAT. Returns the index into MEM if
found, or -1 if not found. If len of PAT is greater than length of
MEM, the function returns -1.
strstr replacement for arbitrary blocks of memory.
- Locates the first occurance in the memory pointed to by MEM of the
+ Locates the first occurrence in the memory pointed to by MEM of the
contents of memory pointed to by PAT. Returns the index into MEM if
found, or -1 if not found. If len of PAT is greater than length of
MEM, the function returns -1.
return NULL;
if (sub_len <= 0) {
- PyErr_SetString(PyExc_ValueError, "empty replacement string");
+ PyErr_SetString(PyExc_ValueError, "empty pattern string");
return NULL;
}
new_s = mymemreplace(str,len,sub,sub_len,repl,repl_len,count,&out_len);
all objects on the free list having a size less than this
limit. This reduces malloc() overhead for small Unicode objects.
- At worse this will result in MAX_UNICODE_FREELIST_SIZE *
+ At worst this will result in MAX_UNICODE_FREELIST_SIZE *
(sizeof(PyUnicodeObject) + STAYALIVE_SIZE_LIMIT +
malloc()-overhead) bytes of unused garbage.
unicode_freelist = *(PyUnicodeObject **)unicode_freelist;
unicode_freelist_size--;
unicode->ob_type = &PyUnicode_Type;
- _Py_NewReference(unicode);
+ _Py_NewReference((PyObject *)unicode);
if (unicode->str) {
if (unicode->length < length &&
_PyUnicode_Resize(unicode, length)) {
unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
}
- if (!unicode->str) {
- PyMem_DEL(unicode);
- PyErr_NoMemory();
- return NULL;
- }
+ if (!unicode->str)
+ goto onError;
unicode->str[length] = 0;
unicode->length = length;
unicode->hash = -1;
unicode->utf8str = NULL;
return unicode;
+
+ onError:
+ _Py_ForgetReference((PyObject *)unicode);
+ PyMem_DEL(unicode);
+ PyErr_NoMemory();
+ return NULL;
}
static
*(PyUnicodeObject **)unicode = unicode_freelist;
unicode_freelist = unicode;
unicode_freelist_size++;
- _Py_ForgetReference(unicode);
}
else {
free(unicode->str);
}
else {
PyErr_Format(PyExc_ValueError,
- "UTF-8 decoding error; unkown error handling code: %s",
+ "UTF-8 decoding error; unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"UTF-8 encoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
}
else {
PyErr_Format(PyExc_ValueError,
- "UTF-16 decoding error; unkown error handling code: %s",
+ "UTF-16 decoding error; unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"Unicode-Escape decoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
*/
+static const Py_UNICODE *findchar(const Py_UNICODE *s,
+ int size,
+ Py_UNICODE ch);
+
static
PyObject *unicodeescape_string(const Py_UNICODE *s,
int size,
p = q = PyString_AS_STRING(repr);
if (quotes) {
- static const Py_UNICODE *findchar(const Py_UNICODE *s,
- int size,
- Py_UNICODE ch);
*p++ = 'u';
*p++ = (findchar(s, size, '\'') &&
!findchar(s, size, '"')) ? '"' : '\'';
else {
PyErr_Format(PyExc_ValueError,
"Latin-1 encoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"ASCII decoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"ASCII encoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"charmap decoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"charmap encoding error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
else {
PyErr_Format(PyExc_ValueError,
"translate error; "
- "unkown error handling code: %s",
+ "unknown error handling code: %s",
errors);
return -1;
}
PyObject *_PyCodec_Lookup(const char *encoding)
{
- PyObject *result, *args = NULL, *v;
+ PyObject *result, *args = NULL, *v = NULL;
int i, len;
+ if (_PyCodec_SearchCache == NULL || _PyCodec_SearchPath == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "codec module not properly initialized");
+ goto onError;
+ }
if (!import_encodings_called)
import_encodings();
result = PyDict_GetItem(_PyCodec_SearchCache, v);
if (result != NULL) {
Py_INCREF(result);
+ Py_DECREF(v);
return result;
}
if (args == NULL)
goto onError;
PyTuple_SET_ITEM(args,0,v);
+ v = NULL;
for (i = 0; i < len; i++) {
PyObject *func;
if (i == len) {
/* XXX Perhaps we should cache misses too ? */
PyErr_SetString(PyExc_LookupError,
- "unkown encoding");
+ "unknown encoding");
goto onError;
}
return result;
onError:
+ Py_XDECREF(v);
Py_XDECREF(args);
return NULL;
}
void _PyCodecRegistry_Fini()
{
Py_XDECREF(_PyCodec_SearchPath);
+ _PyCodec_SearchPath = NULL;
Py_XDECREF(_PyCodec_SearchCache);
+ _PyCodec_SearchCache = NULL;
}
projects / thirdparty / Python / cpython.git / commitdiff
