From 96056d33997da0a87a72675ea520af5f607f6b03 Mon Sep 17 00:00:00 2001 From: STiFLeR7 Date: Mon, 31 Aug 2026 09:17:31 +0530 Subject: [PATCH] fix: correct nl.toknize -> nl.tokenize typo in TopicSegmentationChunking.extract_keywords extract_keywords() called the non-existent nl.toknize.word_tokenize instead of nl.tokenize.word_tokenize, raising AttributeError on every invocation. This is the same typo class reported and fixed in issue #59 (2024), but that fix only patched the constructor's nl.toknize.TextTilingTokenizer() call and missed this second, separate occurrence in extract_keywords(), which has been broken ever since. Adds a unit test that reproduces the AttributeError against the unfixed code (verified red) and passes after the fix (verified green). --- crawl4ai/chunking_strategy.py | 2 +- tests/unit/test_chunking_strategy_unit.py | 31 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_chunking_strategy_unit.py diff --git a/crawl4ai/chunking_strategy.py b/crawl4ai/chunking_strategy.py index a0bfe1bf4..9a6ccbf2f 100644 --- a/crawl4ai/chunking_strategy.py +++ b/crawl4ai/chunking_strategy.py @@ -119,7 +119,7 @@ def extract_keywords(self, text: str) -> list: # Tokenize and remove stopwords and punctuation import nltk as nl - tokens = nl.toknize.word_tokenize(text) + tokens = nl.tokenize.word_tokenize(text) tokens = [ token.lower() for token in tokens diff --git a/tests/unit/test_chunking_strategy_unit.py b/tests/unit/test_chunking_strategy_unit.py new file mode 100644 index 000000000..f16f2e823 --- /dev/null +++ b/tests/unit/test_chunking_strategy_unit.py @@ -0,0 +1,31 @@ +"""Unit tests for chunking_strategy.py.""" +import sys +import types +from unittest.mock import patch, MagicMock +from crawl4ai.chunking_strategy import TopicSegmentationChunking + + +class TestTopicSegmentationExtractKeywords: + + def test_extract_keywords_does_not_raise(self): + # extract_keywords previously called the non-existent `nl.toknize` + # instead of `nl.tokenize`, raising AttributeError on every call. + # __init__ is bypassed since it builds a real TextTilingTokenizer, + # which needs NLTK data unrelated to this bug. + # + # nltk's real `corpus`/`tokenize` modules are lazy-loaded and touch + # disk data on first attribute access even under mock.patch, so a + # bare stand-in module is swapped into sys.modules instead. + fake_nltk = types.ModuleType("nltk") + fake_nltk.tokenize = MagicMock() + fake_nltk.tokenize.word_tokenize.return_value = ["fast", "car", "fast", "car", "road"] + fake_nltk.corpus = MagicMock() + fake_nltk.corpus.stopwords.words.return_value = ["the", "a"] + + chunker = TopicSegmentationChunking.__new__(TopicSegmentationChunking) + chunker.num_keywords = 2 + + with patch.dict(sys.modules, {"nltk": fake_nltk}): + keywords = chunker.extract_keywords("Fast car, fast car, on the road.") + + assert keywords == ["fast", "car"]