gh-153569: remove redundant reader counters

The reader stores two prefetched lines with an index and count that only
describe whether each slot is occupied. Its control flags also use full
integers.

Use the chunk slots as their own occupancy state and keep reader control flags
byte-sized. This removes bookkeeping without changing read order or buffer
ownership.

Author: pablogsal

Parser/tokenizer/decoder.c

@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
     }
     result[write] = '\0';
     *out_len = write;
-    *implicit_newline = implicit;
+    if (implicit_newline != NULL) {
+        *implicit_newline = implicit;
+    }
     return result;
 }
 

Parser/tokenizer/reader.c

@@ -190,10 +190,10 @@ chunk_is_line(const _PyTok_Chunk *chunk)
 static _PyTok_ReadResult
 next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
-    int lineno = tok->lineno + 1;
-    if (lineno > tok->source.nlines) {
+    if (tok->lineno >= tok->source.nlines) {
         return _PYTOK_READ_EOF;
     }
+    int lineno = tok->lineno + 1;
     const char *start = tok->inp;
     const char *newline = memchr(
         start, '\n', tok->source.bytes + tok->source.len - start);
@@ -259,7 +259,6 @@ initialize_file(struct tok_state *tok)
     if (result != _PYTOK_READ_LINE) {
         return -1;
     }
-    reader->prefetched_count = 1;
     Py_ssize_t bom_len;
     _PyTok_EncodingResult detection = _PyTok_DetectEncoding(
         tok, &reader->prefetched_lines[0], NULL, 0, &bom_len);
@@ -277,16 +276,13 @@ initialize_file(struct tok_state *tok)
         reader->prefetched_lines[0].data = first;
         reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM;
         result = read_file_line(tok, &reader->prefetched_lines[1]);
-        if (result == _PYTOK_READ_LINE) {
-            reader->prefetched_count = 2;
-        }
-        else if (result == _PYTOK_READ_EOF) {
+        if (result == _PYTOK_READ_EOF) {
             reader->file_eof = 1;
         }
-        else {
+        else if (result != _PYTOK_READ_LINE) {
             return -1;
         }
-        _PyTok_Chunk *second = reader->prefetched_count == 2
+        _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL
             ? &reader->prefetched_lines[1] : NULL;
         detection = _PyTok_DetectEncoding(
             tok, &reader->prefetched_lines[0], second, 1, &bom_len);
@@ -356,10 +352,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk)
             return _PYTOK_READ_LINE;
         }
         _PyTok_Chunk input = {0};
-        if (reader->prefetched_index < reader->prefetched_count) {
-            input = reader->prefetched_lines[reader->prefetched_index];
-            reader->prefetched_lines[reader->prefetched_index++] =
-                (_PyTok_Chunk){0};
+        if (reader->prefetched_lines[0].data != NULL) {
+            input = reader->prefetched_lines[0];
+            reader->prefetched_lines[0] = (_PyTok_Chunk){0};
+        }
+        else if (reader->prefetched_lines[1].data != NULL) {
+            input = reader->prefetched_lines[1];
+            reader->prefetched_lines[1] = (_PyTok_Chunk){0};
         }
         else if (!reader->file_eof) {
             _PyTok_ReadResult result = read_file_line(tok, &input);
@@ -557,7 +556,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
     }
     chunk->data = _PyTok_NormalizeNewlines(
         decoded.data, decoded.len, 0, 0,
-        &chunk->len, &chunk->implicit_newline);
+        &chunk->len, NULL);
     _PyTok_ChunkClear(&decoded);
     if (chunk->data == NULL) {
         PyErr_NoMemory();
@@ -646,6 +645,12 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         return 0;
     }
+    if (tok->lineno == INT_MAX) {
+        PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines");
+        tok->done = E_ERROR;
+        _PyTok_ChunkClear(&chunk);
+        return 0;
+    }
 
     Py_ssize_t scan_len = chunk.len;
     if (kind == _PYTOK_READER_INTERACTIVE &&

Parser/tokenizer/reader_internal.h

@@ -32,14 +32,13 @@ typedef enum {
 
 typedef struct {
     char *data;
-    Py_ssize_t len;
-    int implicit_newline;
     PyObject *owner;
+    Py_ssize_t len;
     _PyTok_ChunkOwnership ownership;
+    unsigned char implicit_newline;
 } _PyTok_Chunk;
 
 typedef struct _PyTok_Reader {
-    _PyTok_ReaderKind kind;
     PyObject *readline;
     PyObject *decoder;
     const char *prompt;
@@ -50,18 +49,16 @@ typedef struct _PyTok_Reader {
     char *file_buffer;
     Py_ssize_t file_buffer_cap;
     _PyTok_Chunk prefetched_lines[2];
-    int prefetched_index;
-    int prefetched_count;
 
     char *decoded;
     Py_ssize_t decoded_pos;
     Py_ssize_t decoded_len;
     Py_ssize_t decoded_cap;
-    int decoded_tail_is_implicit;
-
-    int file_initialized;
-    int file_eof;
-    int decoder_finalized;
+    _PyTok_ReaderKind kind;
+    unsigned char decoded_tail_is_implicit;
+    unsigned char file_initialized;
+    unsigned char file_eof;
+    unsigned char decoder_finalized;
     unsigned char stop_interactive;
 } _PyTok_Reader;