fix: correct nl.toknize -> nl.tokenize typo in TopicSegmentationChunking.extract_keywords

crawl4ai/chunking_strategy.py

@@ -119,7 +119,7 @@ def extract_keywords(self, text: str) -> list:
         # Tokenize and remove stopwords and punctuation
         import nltk as nl
 
-        tokens = nl.toknize.word_tokenize(text)
+        tokens = nl.tokenize.word_tokenize(text)
         tokens = [
             token.lower()
             for token in tokens

tests/unit/test_chunking_strategy_unit.py

@@ -0,0 +1,31 @@
+"""Unit tests for chunking_strategy.py."""
+import sys
+import types
+from unittest.mock import patch, MagicMock
+from crawl4ai.chunking_strategy import TopicSegmentationChunking
+
+
+class TestTopicSegmentationExtractKeywords:
+
+    def test_extract_keywords_does_not_raise(self):
+        # extract_keywords previously called the non-existent `nl.toknize`
+        # instead of `nl.tokenize`, raising AttributeError on every call.
+        # __init__ is bypassed since it builds a real TextTilingTokenizer,
+        # which needs NLTK data unrelated to this bug.
+        #
+        # nltk's real `corpus`/`tokenize` modules are lazy-loaded and touch
+        # disk data on first attribute access even under mock.patch, so a
+        # bare stand-in module is swapped into sys.modules instead.
+        fake_nltk = types.ModuleType("nltk")
+        fake_nltk.tokenize = MagicMock()
+        fake_nltk.tokenize.word_tokenize.return_value = ["fast", "car", "fast", "car", "road"]
+        fake_nltk.corpus = MagicMock()
+        fake_nltk.corpus.stopwords.words.return_value = ["the", "a"]
+
+        chunker = TopicSegmentationChunking.__new__(TopicSegmentationChunking)
+        chunker.num_keywords = 2
+
+        with patch.dict(sys.modules, {"nltk": fake_nltk}):
+            keywords = chunker.extract_keywords("Fast car, fast car, on the road.")
+
+        assert keywords == ["fast", "car"]