Untrack dataset files and add BM25 evaluation script

.gitignore

@@ -22,3 +22,16 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+
+# Datasets (LIMIT jsonl files and any generated/downloaded data)
+data/
+
+# Virtual environments
+.venv/
+venv/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# HuggingFace / model caches
+.cache/

code/run_bm25_limit.py

@@ -0,0 +1,22 @@
+"""Run the MTEB BM25 baseline on the LIMIT retrieval tasks."""
+
+import mteb
+
+model = mteb.get_model("mteb/baseline-bm25s")
+
+for task_name in ["LIMITSmallRetrieval", "LIMITRetrieval"]:
+    tasks = mteb.get_tasks(tasks=[task_name])
+    results = mteb.evaluate(model, tasks=tasks)
+    for res in results:
+        scores = res.only_main_score().to_dict()["scores"]
+        print(f"\n=== {res.task_name} ===")
+        full = res.to_dict()["scores"]["test"][0]
+        for metric in [
+            "main_score",
+            "recall_at_2",
+            "recall_at_10",
+            "recall_at_100",
+            "ndcg_at_10",
+        ]:
+            if metric in full:
+                print(f"{metric}: {full[metric]}")