[3.11] Fix typo in the tokenizer (GH-104950) (#104952)

(cherry picked from commit 705e387dd8)

Co-authored-by: Stepfen Shawn <m18824909883@163.com>
This commit is contained in:
Miss Islington (bot)
2023-05-25 23:32:04 -07:00
committed by GitHub
parent 161fc18edc
commit b764347572
+1 -1
View File
@@ -20,7 +20,7 @@ enum decoding_state {
};
enum interactive_underflow_t {
/* Normal mode of operation: return a new token when asked in interactie mode */
/* Normal mode of operation: return a new token when asked in interactive mode */
IUNDERFLOW_NORMAL,
/* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
* can be used to prevent the tokenizer to prompt the user for new tokens */