gh-153569: make the reader own interactive state

Interactive prompting and parser stop requests describe how the reader obtains
more input. Store that state on the reader and expose small queries for the
lexer and parser.

The accumulated interactive statement is already the tokenizer source. Return
it directly after a successful parse and remove duplicate source pointers from
tok_state.

Author: pablogsal

Parser/lexer/lexer.c

@@ -204,15 +204,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
         tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\r') {
+            int interactive = _PyTok_ReaderIsInteractive(tok);
             /* Lines with only whitespace and/or comments
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
                mode, which signal the end of a command group. */
-            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+            if (col == 0 && c == '\n' && interactive) {
                 blankline = 0; /* Let it through */
             }
-            else if (tok->prompt != NULL && tok->lineno == 1) {
+            else if (interactive && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;

Parser/lexer/state.h

@@ -13,14 +13,6 @@
 #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
     (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
 
-enum interactive_underflow_t {
-    /* Normal mode of operation: return a new token when asked in interactive mode */
-    IUNDERFLOW_NORMAL,
-    /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
-     * can be used to prevent the tokenizer to prompt the user for new tokens */
-    IUNDERFLOW_STOP,
-};
-
 struct token {
     int level;
     _PyTok_Span span;
@@ -76,9 +68,6 @@ struct tok_state {
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
     _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
-    int fp_interactive; /* If the file descriptor is interactive */
-    char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
-    char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
@@ -87,7 +76,6 @@ struct tok_state {
     int indstack[MAXINDENT];            /* Stack of indents */
     int atbol;          /* Nonzero if at begin of new line */
     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
-    const char *prompt;          /* For interactive prompting */
     int lineno;         /* Current line number */
     _PyTok_Loc start_loc;
     int level;          /* () [] {} Parentheses nesting level */
@@ -108,8 +96,6 @@ struct tok_state {
 
     int type_comments;      /* Whether to look for type comments */
 
-    /* How to proceed when asked for a new token in interactive mode */
-    enum interactive_underflow_t interactive_underflow;
     int report_warnings;
     // TODO: Factor this into its own thing
     tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];

Parser/pegen.c

@@ -8,8 +8,9 @@
 #include <errcode.h>
 
 #include "lexer/lexer.h"
-#include "tokenizer/tokenizer.h"
 #include "tokenizer/helpers.h"
+#include "tokenizer/reader.h"
+#include "tokenizer/tokenizer.h"
 #include "pegen.h"
 
 #define IDENTIFIER_CACHE_SIZE 2048  // Must be a power of two.
@@ -943,9 +944,7 @@ reset_parser_state_for_error_pass(Parser *p)
     }
     p->mark = 0;
     p->call_invalid_rules = 1;
-    // Don't try to get extra tokens in interactive mode when trying to
-    // raise specialized errors in the second pass.
-    p->tok->interactive_underflow = IUNDERFLOW_STOP;
+    _PyTok_ReaderStopInteractive(p->tok);
 }
 
 static inline int
@@ -1066,10 +1065,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
         }
         return NULL;
     }
-    if (!tok->fp || ps1 != NULL || ps2 != NULL ||
-        PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
-        tok->fp_interactive = 1;
-    }
     // This transfers the ownership to the tokenizer
     tok->filename = Py_NewRef(filename_ob);
 
@@ -1091,8 +1086,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
     result = _PyPegen_run_parser(p);
     _PyPegen_Parser_Free(p);
 
-    if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
-        *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+    if (tok->source.bytes != NULL && result && interactive_src != NULL) {
+        *interactive_src = PyUnicode_FromString(tok->source.bytes);
         if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
             Py_XDECREF(*interactive_src);
             result = NULL;

Parser/pegen_errors.c

@@ -7,6 +7,7 @@
 #include "lexer/state.h"
 #include "lexer/lexer.h"
 #include "pegen.h"
+#include "tokenizer/reader.h"
 
 // TOKENIZER ERRORS
 
@@ -122,7 +123,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
     // before the one that we had for the generic error.
 
     // We don't want to tokenize to the end for interactive input
-    if (p->tok->prompt != NULL) {
+    if (_PyTok_ReaderIsInteractive(p->tok)) {
         return 0;
     }
 
@@ -273,7 +274,8 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
         goto error;
     }
 
-    if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
+    if (_PyTok_ReaderIsInteractive(p->tok) &&
+            p->tok->source.bytes != NULL) {
         error_line = get_error_line_from_source(p, lineno);
     }
     else if (p->start_rule == Py_file_input) {

Parser/tokenizer/reader.c

@@ -529,13 +529,13 @@ static _PyTok_ReadResult
 next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
     _PyTok_Reader *reader = tok->reader;
-    if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+    if (reader->stop_interactive) {
         return _PYTOK_READ_STOPPED;
     }
     char *input = PyOS_Readline(
-        tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt);
+        tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
     if (reader->nextprompt != NULL) {
-        tok->prompt = reader->nextprompt;
+        reader->prompt = reader->nextprompt;
     }
     if (input == NULL) {
         return _PYTOK_READ_INTERRUPT;
@@ -569,6 +569,20 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
     return _PYTOK_READ_LINE;
 }
 
+int
+_PyTok_ReaderIsInteractive(const struct tok_state *tok)
+{
+    return tok->reader->kind == _PYTOK_READER_INTERACTIVE;
+}
+
+void
+_PyTok_ReaderStopInteractive(struct tok_state *tok)
+{
+    if (_PyTok_ReaderIsInteractive(tok)) {
+        tok->reader->stop_interactive = 1;
+    }
+}
+
 static _PyTok_ReadResult
 reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
@@ -686,10 +700,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         tok->inp = tok->source.bytes + source_start + scan_len;
     }
-    if (tok->fp_interactive) {
-        tok->interactive_src_start = tok->source.bytes;
-        tok->interactive_src_end = tok->source.bytes + tok->source.len;
-    }
     if (prepared) {
         if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
             tok->buf = tok->cur;
@@ -801,7 +811,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
         return NULL;
     }
     tok->fp = fp;
-    tok->prompt = ps1;
+    tok->reader->prompt = ps1;
     tok->reader->nextprompt = ps2;
     return tok;
 }

Parser/tokenizer/reader.h

@@ -5,5 +5,7 @@ struct tok_state;
 
 void _PyTok_ReaderFree(struct tok_state *);
 int _PyTok_ReaderUnderflow(struct tok_state *);
+int _PyTok_ReaderIsInteractive(const struct tok_state *);
+void _PyTok_ReaderStopInteractive(struct tok_state *);
 
 #endif

Parser/tokenizer/reader_internal.h

@@ -42,6 +42,7 @@ typedef struct _PyTok_Reader {
     _PyTok_ReaderKind kind;
     PyObject *readline;
     PyObject *decoder;
+    const char *prompt;
     const char *nextprompt;
 
     Py_ssize_t input_buffer_cap;
@@ -61,6 +62,7 @@ typedef struct _PyTok_Reader {
     int file_initialized;
     int file_eof;
     int decoder_finalized;
+    unsigned char stop_interactive;
 } _PyTok_Reader;
 
 struct tok_state;