gh-153569: use tokenizer source for parser diagnostics

Prepared and interactive input already lives in the tokenizer source object.
Read syntax-error text from that bounded storage instead of keeping separate
source pointers in tok_state.

A shared accessor supplies an empty string for empty prepared input. Error-line
lookup uses the recorded source length, so it never scans beyond stored text.

Author: pablogsal

Parser/lexer/state.h

@@ -102,7 +102,6 @@ struct tok_state {
     /* Stuff for PEP 0263 */
     char *encoding;         /* Source encoding. */
     const char* line_start;     /* pointer to start of current line */
-    char* str;          /* Source string being tokenized (if tokenizing from a string)*/
 
     _PyTok_SourceText source;
     struct _PyTok_Reader *reader;

Parser/pegen.c

@@ -961,12 +961,9 @@ _PyPegen_set_syntax_error_metadata(Parser *p) {
         PyErr_SetRaisedException(exc);
         return;
     }
-    const char *source = NULL;
-    if (p->tok->str != NULL) {
-        source = p->tok->str;
-    }
-    if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
-        source = p->tok->interactive_src_start;
+    const char *source = p->tok->source.bytes;
+    if (source == NULL && p->tok->fp == NULL) {
+        source = _PyTok_SourceData(&p->tok->source);
     }
     PyObject* the_source = NULL;
     if (source) {

Parser/pegen_errors.c

@@ -225,45 +225,23 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
 }
 
 static PyObject *
-get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
+get_error_line_from_source(Parser *p, Py_ssize_t lineno)
 {
-    /* If the file descriptor is interactive, the source lines of the current
-     * (multi-line) statement are stored in p->tok->interactive_src_start.
-     * If not, we're parsing from a string, which means that the whole source
-     * is stored in p->tok->str. */
-    assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL);
-
-    char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
-    if (cur_line == NULL) {
-        assert(p->tok->fp_interactive);
-        // We can reach this point if the tokenizer buffers for interactive source have not been
-        // initialized because we failed to decode the original source with the given locale.
-        return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
-    }
+    const char *cur_line = _PyTok_SourceData(&p->tok->source);
 
     Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
-    const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
-
-    if (buf_end < cur_line) {
-        buf_end = cur_line + strlen(cur_line);
-    }
+    const char *buf_end = cur_line + p->tok->source.len;
 
     for (int i = 0; i < relative_lineno - 1; i++) {
-        char *new_line = strchr(cur_line, '\n');
-        // The assert is here for debug builds but the conditional that
-        // follows is there so in release builds we do not crash at the cost
-        // to report a potentially wrong line.
-        assert(new_line != NULL && new_line + 1 < buf_end);
-        if (new_line == NULL || new_line + 1 > buf_end) {
+        const char *new_line = memchr(cur_line, '\n', buf_end - cur_line);
+        if (new_line == NULL) {
             break;
         }
         cur_line = new_line + 1;
     }
 
-    char *next_newline;
-    if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
-        next_newline = cur_line + strlen(cur_line);
-    }
+    const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line);
+    next_newline = next_newline != NULL ? next_newline : buf_end;
     return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
 }
 
@@ -296,7 +274,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     }
 
     if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
-        error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+        error_line = get_error_line_from_source(p, lineno);
     }
     else if (p->start_rule == Py_file_input) {
         error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
@@ -318,7 +296,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
             error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
         }
         else if (p->tok->fp == NULL || p->tok->fp == stdin) {
-            error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+            error_line = get_error_line_from_source(p, lineno);
         }
         else {
             error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);

Parser/tokenizer/decoder.c

@@ -399,10 +399,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
     if (stored < 0) {
         return -1;
     }
-    tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)"";
     if (!utf8_only &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
-            !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) {
+            !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
         return -1;
     }
     return 0;

Parser/tokenizer/reader.c

@@ -753,7 +753,9 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
         _PyTokenizer_Free(tok);
         return NULL;
     }
-    tok->buf = tok->cur = tok->inp = tok->str;
+    char *source = (char *)_PyTok_SourceData(&tok->source);
+    tok->buf = tok->cur = tok->inp = source;
+    tok->line_start = source;
     return tok;
 }
 

Parser/tokenizer/source.h

@@ -27,6 +27,12 @@ typedef struct {
     Py_ssize_t implicit_cap;
 } _PyTok_SourceText;
 
+static inline const char *
+_PyTok_SourceData(const _PyTok_SourceText *source)
+{
+    return source->bytes != NULL ? source->bytes : "";
+}
+
 PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
 /* Clear invalidates all spans and views for the source. */
 PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);