diff options
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 32 |
1 files changed, 16 insertions, 16 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 1d0a4aa3f23..bce2bdaa023 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -12,7 +12,7 @@ #ifndef PGEN #include "unicodeobject.h" -#include "stringobject.h" +#include "bytesobject.h" #include "fileobject.h" #include "codecs.h" #include "abstract.h" @@ -344,7 +344,7 @@ check_bom(int get_char(struct tok_state *), 1) NULL: need to call tok->decoding_readline to get a new line 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and stored the result in tok->decoding_buffer - 3) PyStringObject *: previous call to fp_readl did not have enough room + 3) PyBytesObject *: previous call to fp_readl did not have enough room (in the s buffer) to copy entire contents of the line read by tok->decoding_readline. tok->decoding_buffer has the overflow. In this case, fp_readl is called in a loop (with an expanded buffer) @@ -375,7 +375,7 @@ fp_readl(char *s, int size, struct tok_state *tok) return error_ret(tok); } else { tok->decoding_buffer = NULL; - if (PyString_CheckExact(buf)) + if (PyBytes_CheckExact(buf)) utf8 = buf; } if (utf8 == NULL) { @@ -384,10 +384,10 @@ fp_readl(char *s, int size, struct tok_state *tok) if (utf8 == NULL) return error_ret(tok); } - str = PyString_AsString(utf8); - utf8len = PyString_GET_SIZE(utf8); + str = PyBytes_AsString(utf8); + utf8len = PyBytes_GET_SIZE(utf8); if (utf8len > size) { - tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size); + tok->decoding_buffer = PyBytes_FromStringAndSize(str+size, utf8len-size); if (tok->decoding_buffer == NULL) { Py_DECREF(utf8); return error_ret(tok); @@ -591,7 +591,7 @@ decode_str(const char *str, struct tok_state *tok) utf8 = translate_into_utf8(str, tok->enc); if (utf8 == NULL) return error_ret(tok); - str = PyString_AsString(utf8); + str = PyBytes_AsString(utf8); } #endif for (s = str;; s++) { @@ -624,7 +624,7 @@ decode_str(const char *str, struct tok_state *tok) "unknown encoding: %s", tok->enc); return error_ret(tok); } - str = PyString_AsString(utf8); + str = PyBytes_AsString(utf8); } #endif assert(tok->decoding_buffer == NULL); @@ -706,11 +706,11 @@ tok_stdin_decode(struct tok_state *tok, char **inp) return 0; enc = ((PyFileObject *)sysstdin)->f_encoding; - if (enc == NULL || !PyString_Check(enc)) + if (enc == NULL || !PyBytes_Check(enc)) return 0; Py_INCREF(enc); - encoding = PyString_AsString(enc); + encoding = PyBytes_AsString(enc); decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL); if (decoded == NULL) goto error_clear; @@ -720,9 +720,9 @@ tok_stdin_decode(struct tok_state *tok, char **inp) if (utf8 == NULL) goto error_clear; - assert(PyString_Check(utf8)); - converted = new_string(PyString_AS_STRING(utf8), - PyString_GET_SIZE(utf8)); + assert(PyBytes_Check(utf8)); + converted = new_string(PyBytes_AS_STRING(utf8), + PyBytes_GET_SIZE(utf8)); Py_DECREF(utf8); if (converted == NULL) goto error_nomem; @@ -1609,8 +1609,8 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) /* convert source to original encondig */ PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len); if (lineobj != NULL) { - int linelen = PyString_Size(lineobj); - const char *line = PyString_AsString(lineobj); + int linelen = PyBytes_Size(lineobj); + const char *line = PyBytes_AsString(lineobj); text = PyObject_MALLOC(linelen + 1); if (text != NULL && line != NULL) { if (linelen) @@ -1624,7 +1624,7 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) PyObject *offsetobj = dec_utf8(tok->encoding, tok->buf, *offset-1); if (offsetobj) { - *offset = PyString_Size(offsetobj) + 1; + *offset = PyBytes_Size(offsetobj) + 1; Py_DECREF(offsetobj); } } |