diff --git a/.github/workflows/repo-checks.yml b/.github/workflows/repo-checks.yml index 85786ce0e..77087d21b 100644 --- a/.github/workflows/repo-checks.yml +++ b/.github/workflows/repo-checks.yml @@ -49,7 +49,8 @@ jobs: services: typesense: - image: typesense/typesense:0.24.0 + # Keep in sync with infra/Dockerfile.search + image: typesense/typesense:30.2 ports: - 8108:8108 env: diff --git a/.gitignore b/.gitignore index f2e2b9fb1..5b6d43a6d 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,9 @@ CLAUDE.md #gcloud .gcloudignore +# search-eval: every corpus is committed, because none of them can be rebuilt +# byte-for-byte from what the repo holds — bills now joins `summary` in from a +# live project on top of the emulator fixture, and hearings/publishedTestimony +# come from prod outright. See tests/search-eval/README.md. Labeling sheets are +# authoring scratch. +tests/search-eval/labeling-sheet-*.md diff --git a/components/search/SortBy.tsx b/components/search/SortBy.tsx index 5e286c395..1e6df5f50 100644 --- a/components/search/SortBy.tsx +++ b/components/search/SortBy.tsx @@ -42,7 +42,11 @@ export type SortByWithConfigurationItem = SortByItem & { export const SortBy = ({ items }: { items: SortByWithConfigurationItem[] }) => { const sortBy = useSortBy({ items }), - selected = items.find(i => i.value === sortBy.currentRefinement)! + // A routed URL can restore a sort value that no longer exists — an option + // renamed since the link was shared. connectSortBy passes it through with + // only a dev-mode warning, so fall back to the default option instead of + // crashing the page on `.configure` of undefined. + selected = items.find(i => i.value === sortBy.currentRefinement) ?? items[0] useConfigure(selected.configure ?? {}) return ( ` would print the petition boilerplate ("By Mr. X of Y, a petition + * (accompanied by bill, House, No. N)…") under every result. Show a snippet + * only where it explains why this bill is in the list. + */ +const matched = (hit: Hit, attribute: "summary" | "pinslip") => { + // A plain string attribute snippets to a single result, never an array. + const snippet = hit._snippetResult?.[attribute] + return !!snippet && !Array.isArray(snippet) && snippet.matchLevel !== "none" +} + +/** Which blurb to show under the title, or nothing — in priority order, so the + * summary wins where both matched: it says what the bill does in plain + * language, where the pinslip is the clerk's procedural note. The fallback is + * not a rare path — roughly one bill in four on the current court has no + * summary yet, because the trigger in llm/ needs DocumentText the scrape has + * not always captured (#8), and procedural orders are excluded by the converter + * on purpose. + */ +const snippetAttribute = (hit: Hit) => + (["summary", "pinslip"] as const).find(attribute => matched(hit, attribute)) + export const BillHit = ({ hit }: { hit: Hit }) => { const url = maple.bill({ id: hit.number, court: hit.court }) const hearingDate = hit.nextHearingAt && hit.nextHearingAt / 1000 // convert to seconds + const snippet = snippetAttribute(hit) const { t } = useTranslation("common") return ( @@ -159,6 +188,11 @@ export const BillHit = ({ hit }: { hit: Hit }) => { {formatBillId(hit.number)} -{" "} + {snippet && ( +
+ +
+ )}
{(() => { diff --git a/components/search/bills/BillSearch.tsx b/components/search/bills/BillSearch.tsx index 748907f25..6ee7bdfb8 100644 --- a/components/search/bills/BillSearch.tsx +++ b/components/search/bills/BillSearch.tsx @@ -23,16 +23,14 @@ import { BillHit } from "./BillHit" import { useBillRefinements } from "./useBillRefinements" import { SortBy, SortByWithConfigurationItem } from "../SortBy" import { getServerConfig, VirtualFilters } from "../common" +import { billsSearchParams } from "../searchParams" import { useBillSort } from "./useBillSort" import { FC, useState } from "react" import { pathToSearchState, searchStateToUrl } from "../routingHelpers" const searchClient = new TypesenseInstantSearchAdapter({ server: getServerConfig(), - additionalSearchParameters: { - query_by: "number,title,body", - exclude_fields: "body" - } + additionalSearchParameters: billsSearchParams }).searchClient const extractLastSegmentOfRefinements = (items: any[]) => { diff --git a/components/search/bills/useBillRefinements.tsx b/components/search/bills/useBillRefinements.tsx index b42a16f81..0fcf62805 100644 --- a/components/search/bills/useBillRefinements.tsx +++ b/components/search/bills/useBillRefinements.tsx @@ -37,6 +37,11 @@ export const useBillRefinements = () => { attribute: "court" }, { attribute: "currentCommittee" }, + /** Bills outrank procedural documents by default (see + * billsRelevanceSort in ../searchParams.ts, and #95). This is how a + * searcher who actually wants an Extension Order gets back to it — + * refining to the type lifts the whole demoted class. */ + { attribute: "legislationType" }, { attribute: "city" }, { attribute: "primarySponsor" }, { attribute: "cosponsors" } diff --git a/components/search/bills/useBillSort.tsx b/components/search/bills/useBillSort.tsx index ca73c151e..1cd5d219a 100644 --- a/components/search/bills/useBillSort.tsx +++ b/components/search/bills/useBillSort.tsx @@ -1,6 +1,7 @@ import { useMemo, useRef } from "react" import { SortByWithConfigurationItem } from "../SortBy" import { useTranslation } from "next-i18next" +import { billsRelevanceSort } from "../searchParams" export const useBillSort = () => { const now = useRef(new Date().getTime()) @@ -16,7 +17,7 @@ export const useBillSort = () => { }, { label: t("sort_by.relevance"), - value: "bills/sort/_text_match:desc,testimonyCount:desc" + value: `bills/sort/${billsRelevanceSort}` }, { label: t("sort_by.testimony_count"), diff --git a/components/search/common.tsx b/components/search/common.tsx index 3ed88c1c5..a15d07660 100644 --- a/components/search/common.tsx +++ b/components/search/common.tsx @@ -41,6 +41,7 @@ export function VirtualFilters({ type }: { type: "bill" | "testimony" }) { : [ "court", "currentCommittee", + "legislationType", "city", "primarySponsor", "cosponsors", diff --git a/components/search/hearings/HearingSearch.tsx b/components/search/hearings/HearingSearch.tsx index 762809d42..2168613f4 100644 --- a/components/search/hearings/HearingSearch.tsx +++ b/components/search/hearings/HearingSearch.tsx @@ -1,6 +1,6 @@ import { Hit } from "instantsearch.js" import { useInstantSearch } from "react-instantsearch" -import { SearchPage } from "../shared" +import { SearchPage, SortOptionInput } from "../shared" import { HearingHit } from "./HearingHit" import { CURRENT_COURT_NUMBER, @@ -8,33 +8,20 @@ import { formatCourtSubtitle } from "../courtSessions" import { useMemo, useRef } from "react" - -/* carbon copy of type in functions/src/hearings/search.ts */ -type HearingSearchRecord = { - id: string - eventId: number - title: string - description?: string - startsAt: number - month: string - year: number - committeeCode?: string - committeeName?: string - locationName?: string - locationCity?: string - chairNames: string[] - agendaTopics: string[] - billNumbers: string[] - billSlugs: string[] - court: number - hasVideo: boolean -} +import type { HearingSearchRecord } from "functions/src/hearings/types" +import { hearingsRelevanceSort, hearingsSearchParams } from "../searchParams" export type HearingHitData = Hit -const useHearingSort = () => { +/** Relevance searches every hearing, past and upcoming. Clearing the window + * explicitly is load-bearing: SortBy calls useConfigure(selected.configure ?? {}), + * so the previously selected option's startsAt bound has to be overwritten. + */ +const noTimeWindow: SortOptionInput["configure"] = { numericRefinements: {} } + +const useHearingSort = (): SortOptionInput[] => { const now = useRef(new Date().getTime()) - return useMemo( + return useMemo( () => [ { labelKey: "sort_by.past_newest", @@ -59,8 +46,11 @@ const useHearingSort = () => { } }, { + // "upcoming" already owns startsAt:asc, and react-instantsearch needs a + // unique index name per sort option — eventId is the tiebreak that + // makes this one distinct without changing the ordering users see. labelKey: "sort_by.past_oldest", - value: "hearings/sort/startsAt:asc,startsAt:asc", + value: "hearings/sort/startsAt:asc,eventId:asc", configure: { numericRefinements: { startsAt: { @@ -68,6 +58,12 @@ const useHearingSort = () => { } } } + }, + { + // The only sort under which text ranking is observable. + labelKey: "sort_by.relevance", + value: `hearings/sort/${hearingsRelevanceSort}`, + configure: noTimeWindow } ], [] @@ -89,11 +85,7 @@ export const HearingSearch = () => { } } }} - searchParameters={{ - query_by: - "title,description,agendaTopics,billNumbers,chairNames,locationName,locationCity", - sort_by: "startsAt:asc" - }} + searchParameters={hearingsSearchParams} hitComponent={HearingHit} filterPanelConfig={{ filters: [ diff --git a/components/search/searchParams.ts b/components/search/searchParams.ts new file mode 100644 index 000000000..fa3206471 --- /dev/null +++ b/components/search/searchParams.ts @@ -0,0 +1,159 @@ +/** Ranking-relevant Typesense search parameters shared between the app's + * search pages and the relevance eval harness (scripts/search-eval). Keep this + * module free of React/Next imports so it can be loaded from ts-node scripts. + */ +import { SYNONYM_SET_NAME } from "functions/src/search/synonyms" +import type { TypesenseInstantsearchAdapterOptions } from "typesense-instantsearch-adapter" + +export type SearchParameters = NonNullable< + TypesenseInstantsearchAdapterOptions["additionalSearchParameters"] +> + +/** Derives query_by + query_by_weights from one ordered field-to-weight map, + * so the two parallel comma-separated lists can't drift out of sync. + */ +const weighted = (fields: Record) => ({ + query_by: Object.keys(fields).join(","), + query_by_weights: Object.values(fields).join(",") +}) + +/** Server-level synonym set (Typesense >= 30) applied at query time; every + * collection that searches legislative prose wants it. Deploys keep the set + * itself current — see functions/src/search/synonyms.ts. + */ +const legislativeSynonyms = SYNONYM_SET_NAME + +export const billsSearchParams = { + ...weighted({ + number: 10, + numberVariants: 10, + title: 5, + /** Between title and body: denser and more topical than the statutory + * text in `body`, but not the bill's own name. Weights 2, 3 and 4 return + * byte-identical results on the golden set (1 is a hair worse at 0.754); + * at 5 or above, where pinslip meets or beats `title`, bills nDCG@10 + * drops 0.755 -> 0.744. The useful constraint is the ceiling, not the + * exact value: keep it strictly below `title`. + */ + pinslip: 3, + /** The LLM plain-language description (functions/src/bills/search.ts). + * Below pinslip because the clerk's blurb states the bill's purpose in the + * legislature's own words, while this restates it; both sit under `title`. + * + * Weights 1 through 4 are a plateau at nDCG@10 0.739 (2 has the best + * recall@10 at 0.880); at 5, where summary meets `title`, it collapses to + * 0.693 and topic falls 0.500 -> 0.394. As with pinslip, the useful + * constraint is the ceiling, not the exact value. + */ + summary: 2, + primarySponsor: 4, + cosponsors: 3, + currentCommittee: 4, + body: 1 + }), + synonym_sets: legislativeSynonyms, + /** The adapter defaults highlight_full_fields to query_by, which returns + * every searched field a second time in full with tags. BillHit + * renders exactly one , on title, and reads pinslip only through + * — so title is the only field that needs the full copy. Naming it + * here drops roughly 2.4KB per page of pinslip nobody reads. Add a field to + * this list before rendering a for it, or it will render + * unmarked. + */ + highlight_full_fields: "title", + /** numberVariants holds the space-separated form of each bill number (see + * functions/src/bills/search.ts). It is a matching artifact, so excluding it + * keeps it out of hits and out of the adapter's highlight output, which + * defaults to query_by. pinslip and summary are deliberately NOT excluded: + * the adapter builds _snippetResult by walking the returned document + * (SearchResponseAdapter._adaptHighlightInObjectValue), so excluding either + * would leave its with nothing to render. summary costs roughly + * 10.8KB per 20-hit page; `highlight_full_fields` above is what keeps that + * from doubling. + */ + exclude_fields: "body,numberVariants" +} satisfies SearchParameters + +/** The app's "Relevance" sort option (see useBillSort.tsx). The eval harness + * must use this sort; the UI's default latestTestimonyAt:desc sort does not + * measure text relevance. + * + * The `_eval` clause sorts every Order and Extension Order below every other + * document. They are procedural — "Extension Order - Education", "Order + * relative to authorizing the joint committee on Education to make an + * investigation and study" — and they match a committee query in title, + * pinslip and body without being what anyone searching a committee wants. + * Demoting rather than dropping them scores better than removing them from the + * index outright (0.752 against 0.749, member-committee 0.934 against 0.920) + * and keeps them findable: no bill shares their number, so "H4394" still + * returns the order at rank 1. + * + * Only Order and Extension Order. Widening it to everything that is not a Bill + * scores higher overall (0.756) but demotes Resolves and constitutional + * amendment proposals, which are legislation people search for, and it + * regresses hyp-006. + * + * What it does cost is searching for the type by name: "Extension Order + * Education" puts bills first and the real orders at rank 26 of 32. The + * legislationType refinement (useBillRefinements.tsx) is how a searcher who + * wants them gets them back, which is why the facet ships with this sort. + * + * Written as two `!=` clauses rather than the equivalent `!=[...]` list form: + * this string doubles as the InstantSearch index name, which travels through + * the URL as a qs key, and qs parses percent-encoded square brackets in a key + * as nesting — mangling the routed uiState so a reload or shared link loses + * the query, refinements and sort. No other character here is special to qs. + */ +export const billsRelevanceSort = + "_eval(legislationType:!=`Order` && legislationType:!=`Extension Order`):desc,_text_match:desc,testimonyCount:desc" + +export const testimonySearchParams = { + ...weighted({ + billId: 10, + billIdVariants: 10, + authorDisplayName: 6, + content: 3, + authorRole: 1 + }), + synonym_sets: legislativeSynonyms, + /** TestimonyHit renders content through , which reads Typesense's + * windowed `highlight.snippet` regardless of this setting — only the + * full-field `highlight.value` that needs is gated by it. No + * field here is rendered with , so leaving the adapter's + * highlight_full_fields default (every query_by field) would send a second, + * full-length marked copy of `content` with every hit for nothing. */ + highlight_full_fields: "", + /** A matching artifact, like bills' numberVariants: excluding it keeps it out + * of hits and out of the adapter's highlight output, which defaults to + * query_by. */ + exclude_fields: "billIdVariants" +} satisfies SearchParameters + +/** The app's "Relevance" sort option (see useTestimonySort in + * testimony/TestimonySearch.tsx), and the sort the eval harness must use. + */ +export const testimonyRelevanceSort = "_text_match:desc,publishedAt:desc" + +export const hearingsSearchParams = { + ...weighted({ + billNumbers: 10, + title: 8, + chairNames: 6, + agendaTopics: 5, + description: 2, + locationName: 2, + locationCity: 2 + }), + synonym_sets: legislativeSynonyms +} satisfies SearchParameters + +/** The app's "Relevance" sort option (see useHearingSort in + * hearings/HearingSearch.tsx). Every other hearings sort is date-ordered, so + * this is the only one under which text ranking is observable — and the only + * one the eval harness can score. + * + * No sort_by is pinned in hearingsSearchParams above: the adapter derives it + * from the "hearings/sort/" index name of the selected option, which + * takes precedence over additionalSearchParameters. + */ +export const hearingsRelevanceSort = "_text_match:desc,startsAt:desc" diff --git a/components/search/shared/SearchPage.tsx b/components/search/shared/SearchPage.tsx index 81276015a..cf96bc7b3 100644 --- a/components/search/shared/SearchPage.tsx +++ b/components/search/shared/SearchPage.tsx @@ -116,13 +116,17 @@ export const SearchPage = ({ return Array.from(items) }, [filtersWithDefaults, menuAttributes]) + // Keyed on the params' content, not object identity, so an inline + // searchParameters literal doesn't recreate the Typesense client per render. + const searchParametersKey = JSON.stringify(searchParameters) const searchClient = useMemo( () => new TypesenseInstantSearchAdapter({ server: getServerConfig(), additionalSearchParameters: searchParameters }).searchClient, - [searchParameters] + // eslint-disable-next-line react-hooks/exhaustive-deps + [searchParametersKey] ) return ( diff --git a/components/search/shared/index.ts b/components/search/shared/index.ts index 9d7fe6fa5..7972ed9a0 100644 --- a/components/search/shared/index.ts +++ b/components/search/shared/index.ts @@ -1 +1,2 @@ export { SearchPage } from "./SearchPage" +export type { SortOptionInput } from "./SearchPage" diff --git a/components/search/testimony/TestimonyHit.tsx b/components/search/testimony/TestimonyHit.tsx index 53edd415a..38a4b4ebb 100644 --- a/components/search/testimony/TestimonyHit.tsx +++ b/components/search/testimony/TestimonyHit.tsx @@ -2,6 +2,7 @@ import { Hit } from "instantsearch.js" import Link from "next/link" import { Trans, useTranslation } from "next-i18next" import { Card, Image } from "react-bootstrap" +import { Snippet } from "react-instantsearch" import styled from "styled-components" import { useMediaQuery } from "usehooks-ts" @@ -12,7 +13,6 @@ import { useFlags } from "components/featureFlags" import { formatBillId, truncateText } from "components/formatting" import { maple } from "components/links" import { FollowUserButton } from "components/shared/FollowButton" -import { trimContent } from "components/TestimonyCallout/TestimonyCallout" const StyledCard = styled(Card)` background: white; @@ -217,7 +217,9 @@ const TestimonyResult = ({ hit }: { hit: Hit }) => { - "{trimContent(hit.content, 220)}" + + "" + {committee && ( diff --git a/components/search/testimony/TestimonySearch.tsx b/components/search/testimony/TestimonySearch.tsx index cb45bd164..9c8e4e751 100644 --- a/components/search/testimony/TestimonySearch.tsx +++ b/components/search/testimony/TestimonySearch.tsx @@ -25,6 +25,7 @@ import { SearchContainer } from "../SearchContainer" import { SearchErrorBoundary } from "../SearchErrorBoundary" import { SortBy } from "../SortBy" import { getServerConfig, VirtualFilters } from "../common" +import { testimonyRelevanceSort, testimonySearchParams } from "../searchParams" import { TestimonyHit } from "./TestimonyHit" import { useTestimonyRefinements } from "./useTestimonyRefinements" import { FollowContext, OrgFollowStatus } from "components/shared/FollowContext" @@ -33,10 +34,7 @@ import { useTranslation } from "next-i18next" const searchClient = new TypesenseInstantSearchAdapter({ server: getServerConfig(), - additionalSearchParameters: { - query_by: "billId,content,authorDisplayName,authorRole", - exclude_fields: "" - } + additionalSearchParameters: testimonySearchParams }).searchClient export const useTestimonySort = () => { @@ -53,7 +51,7 @@ export const useTestimonySort = () => { }, { label: t("sort_by.relevance"), - value: "publishedTestimony/sort/_text_match:desc,publishedAt:desc" + value: `publishedTestimony/sort/${testimonyRelevanceSort}` } ], [t] diff --git a/docs/search-deployment.md b/docs/search-deployment.md new file mode 100644 index 000000000..959b443b0 --- /dev/null +++ b/docs/search-deployment.md @@ -0,0 +1,44 @@ +# Deploying the 2026 search relevance work + +Four merges, in this order. Nothing is run by hand: every step is a merge that a +pipeline picks up. + +## Dev + +1. Merge the infra PR pointing dev's Typesense at 30.2. maple-testimony/infra + deploys itself on push to main through the maple-cicd CodePipeline. When it + finishes, dev is running 30.2 and its index is empty. + +2. Merge the search branch to main in codeforboston/maple. deploy-backend-dev.yml + deploys the functions and publishes checkSearchIndexVersion, which starts the + reindex and upserts the synonym set server-side (the functions hold the + Typesense key; CI never does), and Vercel builds the frontend from the same + commit. Dev search comes back when the reindex finishes; bills takes longest, + at roughly 49,000 documents. + +Confirm with `yarn typesense-admin status --env dev`, which prints each alias, +the collection it points at and its document count, and the synonym sets the +server holds — `legislative` with its item count should be there. + +## Prod + +The same shape, but as separate PRs, and prod search is down between the two +steps. Schedule it and say so beforehand. + +3. Merge the infra PR pointing prod's Typesense at 30.2. Same pipeline. Prod's + index is now empty and prod search is down. + +4. Merge main to the prod branch in codeforboston/maple. This is the same code + already running on dev, and deploy-prod.yml does what step 2 did. Prod search + comes back when the reindex finishes. + +Confirm with `yarn typesense-admin status --env prod`. + +Two things fail quietly. A missing synonym set makes `synonym_sets: "legislative"` +resolve to nothing, and relevance regresses with no error — deploys re-upsert the +set from inside checkSearchIndexVersion so it cannot be forgotten, but if that +invocation fails the gap reopens; `typesense-admin status` prints the server's +synonym sets, so the confirm step catches it. And +components/search/common.tsx falls back to the hardcoded dev cluster when +`NEXT_PUBLIC_TYPESENSE_*` are unset, so a misconfigured prod frontend searches dev +and looks healthy. Check the Vercel variables before step 4, not after. diff --git a/functions/package.json b/functions/package.json index 21b337856..d16819f27 100644 --- a/functions/package.json +++ b/functions/package.json @@ -33,7 +33,7 @@ "pdf-parse": "1.1.1", "runtypes": "6.6.0", "ssl-root-cas": "^1.3.1", - "typesense": "^1.2.2", + "typesense": "^3.0.6", "unzipper": "^0.12.3", "zod": "^3.20.2" }, @@ -51,7 +51,7 @@ "jest": "^29.7.0", "rimraf": "^3.0.2", "ts-jest": "^29.2.5", - "typescript": "4.5.5" + "typescript": "5.3.3" }, "private": true } diff --git a/functions/src/bills/numberVariants.ts b/functions/src/bills/numberVariants.ts new file mode 100644 index 000000000..bd8cee1ad --- /dev/null +++ b/functions/src/bills/numberVariants.ts @@ -0,0 +1,26 @@ +/** Bump when the variant logic below changes what it emits. + * + * All three search configs set `convertVersion: billNumberVariantsVersion`, so + * this one line reindexes every collection that indexes bill numbers. That + * matters because `searchCollectionName` hashes `convert.toString()` and + * nothing it calls — a helper shared by three converters is invisible to all + * three hashes. Editing the function without bumping this would not give three + * stale indexes so much as three DIVERGENT ones, with bills matching "H 100" + * while testimony quietly stopped. + */ +export const billNumberVariantsVersion = 1 + +/** Space-separated forms of a bill number, to be indexed alongside it. + * + * "H100" is a single token, so the queries people actually type — "H 100", + * "H. 100", "H-100" — tokenize to ["h", "100"] and can never match it. + * Indexing "H 100" as its own array element makes them match, and lets + * prioritize_exact_match fire on the whole element. + * + * Returns an empty array when the number has no distinct spaced form, so + * nothing redundant is indexed. + */ +export const billNumberVariants = (number: string): string[] => { + const spaced = number.replace(/^[A-Za-z]+(?=\d)/, "$& ") + return spaced === number ? [] : [spaced] +} diff --git a/functions/src/bills/search.ts b/functions/src/bills/search.ts index cfa7c5c09..2c3be73c1 100644 --- a/functions/src/bills/search.ts +++ b/functions/src/bills/search.ts @@ -1,21 +1,36 @@ import { isString } from "lodash" import { db } from "../firebase" import { createSearchIndexer } from "../search" +import { billNumberVariants, billNumberVariantsVersion } from "./numberVariants" import { Bill, BillTopic } from "./types" export const { syncToSearchIndex: syncBillToSearchIndex, - upgradeSearchIndex: upgradeBillSearchIndex + upgradeSearchIndex: upgradeBillSearchIndex, + runBackfillChunk: runBillBackfillChunk } = createSearchIndexer({ sourceCollection: db.collectionGroup("bills"), documentTrigger: `generalCourts/{court}/bills/{id}`, alias: "bills", idField: "id", + convertVersion: billNumberVariantsVersion, schema: { + /** Hyphens do not split tokens by default, so "vote-by-mail" indexes as one + * token and the spaced form people type cannot reach it — the two spellings + * return disjoint result sets. Splitting on "-" makes each form find both, + * and makes the parts individually searchable ("income" reaches + * "low-income"). Hyphenated compounds are everywhere in this corpus: + * low-income, long-term, well-being, community-based, in-person. + */ + token_separators: ["-"], fields: [ { name: "number", type: "string", facet: false }, + { name: "numberVariants", type: "string[]", facet: false }, { name: "court", type: "int32", facet: true }, { name: "title", type: "string", facet: false }, + { name: "pinslip", type: "string", facet: false, optional: true }, + { name: "summary", type: "string", facet: false, optional: true }, + { name: "legislationType", type: "string", facet: true, optional: true }, { name: "body", type: "string", facet: false, optional: true }, { name: "city", type: "string", facet: true, optional: true }, { name: "currentCommittee", type: "string", facet: true, optional: true }, @@ -52,7 +67,57 @@ export const { id: `${bill.court}-${bill.id}`, court: bill.court, number: bill.id, + numberVariants: billNumberVariants(bill.id), title: bill.content.Title, + /** Nullable in Firestore, but an optional Typesense field rejects an + * explicit null, so the absent case has to be undefined. + * + * The cap drops the ~20 procedural study orders whose Pinslip is a + * concatenated committee docket rather than a description; indexed whole + * they match nearly any topical query. Numbers and the top-10 evidence: + * "Long fields are topic-agnostic magnets" in + * tests/search-eval/README.md. + * + * Inline rather than a named constant — SearchIndexer hashes + * `convert.toString()`, the same trap documented on buildTopicsForSearch + * below. + */ + pinslip: + bill.content.Pinslip && bill.content.Pinslip.length <= 2000 + ? bill.content.Pinslip + : undefined, + /** The LLM plain-language description of what the bill does, written by + * the `bill_on_document_created` trigger in llm/. Deliberately uncapped, + * unlike `pinslip` above: across prod's court 192 these run 166 to 922 + * characters (median 538), so there is no long-field tail to guard + * against. Do not add a cap here by analogy. + * + * Orders are withheld for the same reason the pinslip cap exists, and + * against the same documents. A procedural order ("Extension Order - + * Education", "Order relative to authorizing the joint committee on + * Education to make an investigation and study") matches a committee + * query in title, pinslip and body already; a summary hands it a fourth + * field, and `fields_matched` sits in the lowest bits of `_text_match`, + * so it outranks the bills actually before that committee. Measured on + * "Education Committee": nDCG@10 0.558 -> 0.064 with order summaries + * indexed, and back to 0.558 with them withheld — the whole + * member-committee cost of this field, and nothing else. + * + * Only Order and Extension Order. Resolves and constitutional amendment + * proposals are legislation people search for; "Bill or nothing" would + * be the wrong cut. + * + * Inline rather than a named constant — SearchIndexer hashes + * `convert.toString()`, as on `pinslip` above. + */ + summary: + bill.content.LegislationTypeName === "Order" || + bill.content.LegislationTypeName === "Extension Order" + ? undefined + : bill.summary, + /** Faceted so procedural documents can be told apart from legislation at + * query time rather than dropped at index time — see #95. */ + legislationType: bill.content.LegislationTypeName ?? undefined, body: bill.content.DocumentText, city: bill.city, currentCommittee: bill.currentCommittee?.name, @@ -71,6 +136,13 @@ export const { } }) +/** Called from `convert`, so the collection-name hash does not see this + * function's body (it hashes `convert.toString()` only). Rewriting it changes + * what gets indexed without scheduling a reindex — which is what happened in + * ce137bdc, leaving the live topic facets in the pre-Nov-2024 format until an + * unrelated schema change finally forced a rebuild. Set `convertVersion` on the + * config below when you edit this, or inline it into `convert`. + */ const buildTopicsForSearch = (billTopics: BillTopic[] = []) => { const categoriesSorted = billTopics.map(t => t.category).sort() diff --git a/functions/src/bills/types.ts b/functions/src/bills/types.ts index c75e8787a..6612a3cc2 100644 --- a/functions/src/bills/types.ts +++ b/functions/src/bills/types.ts @@ -44,6 +44,12 @@ export type BillContent = Static export const BillContent = Record({ Pinslip: Nullable(String), Title: String, + /** What kind of document this is: "Bill", "Order", "Extension Order", + * "Amendment", "Resolve", "Governor's Message (Veto)" and nine more. Roughly + * one document in ten in a court is not a Bill. The search converter reads it + * to keep procedural orders from competing with legislation + * (functions/src/bills/search.ts). */ + LegislationTypeName: Maybe(String), PrimarySponsor: Nullable(Record({ Name: String })), DocumentText: Maybe(String), Cosponsors: Array(Record({ Name: Maybe(String) })) diff --git a/functions/src/hearings/search.ts b/functions/src/hearings/search.ts index b36fe5e2f..011ac63a0 100644 --- a/functions/src/hearings/search.ts +++ b/functions/src/hearings/search.ts @@ -4,30 +4,12 @@ import { createSearchIndexer } from "../search" import { Hearing } from "../events/types" import { timeZone } from "../malegislature" import { generalCourts, currentGeneralCourt } from "../shared/constants" - -type HearingSearchRecord = { - id: string - eventId: number - title: string - description?: string - startsAt: number - month: string - year: number - committeeCode?: string - committeeName?: string - locationName?: string - locationCity?: string - chairNames: string[] - agendaTopics: string[] - billNumbers: string[] - billSlugs: string[] - court: number - hasVideo: boolean -} +import { HearingSearchRecord } from "./types" export const { syncToSearchIndex: syncHearingToSearchIndex, - upgradeSearchIndex: upgradeHearingSearchIndex + upgradeSearchIndex: upgradeHearingSearchIndex, + runBackfillChunk: runHearingBackfillChunk } = createSearchIndexer({ sourceCollection: db.collection("events").where("type", "==", "hearing"), documentTrigger: "events/{eventId}", @@ -35,6 +17,14 @@ export const { idField: "id", filter: data => data.type === "hearing", schema: { + /** Hyphens do not split tokens by default, so "vote-by-mail" indexes as one + * token and the spaced form people type cannot reach it — the two spellings + * return disjoint result sets. Splitting on "-" makes each form find both, + * and makes the parts individually searchable ("income" reaches + * "low-income"). Hyphenated compounds are everywhere in this corpus: + * low-income, long-term, well-being, community-based, in-person. + */ + token_separators: ["-"], fields: [ { name: "eventId", type: "int32", facet: false }, { name: "title", type: "string", facet: false }, diff --git a/functions/src/hearings/types.ts b/functions/src/hearings/types.ts new file mode 100644 index 000000000..408cc6339 --- /dev/null +++ b/functions/src/hearings/types.ts @@ -0,0 +1,23 @@ +/** The hearing document as it is indexed in Typesense. Lives here rather than + * in ./search.ts so the search UI can import it without pulling firebase-admin + * into the client bundle — the same split bills and testimony use. + */ +export type HearingSearchRecord = { + id: string + eventId: number + title: string + description?: string + startsAt: number + month: string + year: number + committeeCode?: string + committeeName?: string + locationName?: string + locationCity?: string + chairNames: string[] + agendaTopics: string[] + billNumbers: string[] + billSlugs: string[] + court: number + hasVideo: boolean +} diff --git a/functions/src/index.ts b/functions/src/index.ts index e11b30569..91138a764 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -3,6 +3,7 @@ export { backfillTestimonyCounts, fetchBillBatch, startBillBatches, + runBillBackfillChunk, syncBillToSearchIndex, updateBillReferences, updateBillSearchIndex, @@ -24,6 +25,7 @@ export { scrapeSingleHearingv2 } from "./events" export { + runHearingBackfillChunk, syncHearingToSearchIndex, upgradeHearingSearchIndex } from "./hearings/search" @@ -37,6 +39,7 @@ export { checkSearchIndexVersion, searchHealthCheck } from "./search" export { deleteTestimony, publishTestimony, + runTestimonyBackfillChunk, syncTestimonyToSearchIndex, upgradeTestimonySearchIndex, resolveReport as adminResolveReport diff --git a/functions/src/search/SearchIndexer.test.ts b/functions/src/search/SearchIndexer.test.ts new file mode 100644 index 000000000..83ef2e4e1 --- /dev/null +++ b/functions/src/search/SearchIndexer.test.ts @@ -0,0 +1,188 @@ +import { CollectionConfig } from "./config" +import { IMPORT_BYTE_BUDGET, SearchIndexer } from "./SearchIndexer" + +jest.mock("../firebase", () => ({ + db: { doc: (path: string) => ({ path }), recursiveDelete: jest.fn() }, + FieldPath: { documentId: () => "__name__" }, + Timestamp: { now: () => ({}) } +})) + +const imports: any[][] = [] +jest.mock("./client", () => ({ + createClient: () => ({ + collections: () => ({ + exists: async () => true, + documents: () => ({ + // The indexer sends a pre-serialized JSONL body and reads the raw + // per-line results back, like the real client's string form. + import: async (body: string) => { + const docs = body.split("\n").map(line => JSON.parse(line)) + imports.push(docs) + return docs.map(() => JSON.stringify({ success: true })).join("\n") + } + }) + }) + }) +})) + +/** A source that behaves like an ordered, cursor-paged Firestore query, keyed + * by document path the way listPage's documentId() ordering is. `path` + * defaults to `id`; give docs distinct paths to model duplicated idField + * values (the same bill number in several courts). + */ +function fakeSource(docs: { id: string; body: string; path?: string }[]) { + const pathOf = (d: { id: string; path?: string }) => d.path ?? d.id + const build = (startAfter: string | null, pageSize: number): any => ({ + orderBy: () => build(startAfter, pageSize), + limit: (n: number) => build(startAfter, n), + startAfter: (cursor: { path: string }) => build(cursor.path, pageSize), + get: async () => { + const rest = + startAfter === null ? docs : docs.filter(d => pathOf(d) > startAfter) + const page = rest.slice(0, pageSize) + return { + size: page.length, + docs: page.map(d => ({ + exists: true, + id: pathOf(d), + ref: { path: pathOf(d) }, + data: () => d, + get: (field: string) => (d as any)[field] + })) + } + } + }) + return build(null, 0) +} + +const pad = (n: number) => String(n).padStart(6, "0") + +const makeIndexer = (docs: { id: string; body: string }[]) => + new SearchIndexer({ + alias: "widgets", + schema: { fields: [{ name: "body", type: "string" }] }, + sourceCollection: fakeSource(docs) as any, + documentTrigger: "widgets/{id}", + idField: "id", + convert: (data: any) => ({ id: data.id, body: data.body }) + } as CollectionConfig) + +beforeEach(() => { + imports.length = 0 +}) + +describe("importInSlices", () => { + const chunk = { startAfter: null, budgetMs: 60_000 } + + it("imports a page that fits the byte budget in one call", async () => { + const docs = [1, 2, 3].map(n => ({ id: pad(n), body: "x".repeat(1000) })) + await makeIndexer(docs).backfillChunk(chunk) + expect(imports).toHaveLength(1) + expect(imports[0]).toHaveLength(3) + }) + + it("splits a page on the byte boundary, not the document count", async () => { + // Three documents at 40% of the budget each: two fit, the third does not. + const body = "x".repeat(Math.floor(IMPORT_BYTE_BUDGET * 0.4)) + const docs = [1, 2, 3].map(n => ({ id: pad(n), body })) + await makeIndexer(docs).backfillChunk(chunk) + expect(imports.map(i => i.length)).toEqual([2, 1]) + }) + + it("sends a single oversized document on its own", async () => { + const docs = [ + { id: pad(1), body: "x".repeat(1000) }, + { id: pad(2), body: "x".repeat(IMPORT_BYTE_BUDGET + 1) }, + { id: pad(3), body: "x".repeat(1000) } + ] + await makeIndexer(docs).backfillChunk(chunk) + expect(imports.map(i => i.map((d: any) => d.id))).toEqual([ + [pad(1)], + [pad(2)], + [pad(3)] + ]) + }) +}) + +describe("backfillChunk", () => { + const docs = Array.from({ length: 600 }, (_, n) => ({ + id: pad(n), + body: "x" + })) + + it("reports a null cursor and every document when the source is exhausted", async () => { + const result = await makeIndexer(docs).backfillChunk({ + startAfter: null, + budgetMs: 60_000 + }) + expect(result.cursor).toBeNull() + expect(result.batches).toBe(3) + expect(result.documents).toBe(600) + }) + + it("stops on the batch budget and hands back a resumable cursor", async () => { + const result = await makeIndexer(docs).backfillChunk({ + startAfter: null, + maxBatches: 1, + budgetMs: 60_000 + }) + expect(result.batches).toBe(1) + expect(result.documents).toBe(250) + expect(result.cursor).toBe(pad(249)) + }) + + it("resumes after the cursor it was handed", async () => { + const result = await makeIndexer(docs).backfillChunk({ + startAfter: pad(249), + budgetMs: 60_000 + }) + expect(result.documents).toBe(350) + expect(result.cursor).toBeNull() + expect(imports.flat().map(d => d.id)).not.toContain(pad(249)) + expect(imports.flat()[0].id).toBe(pad(250)) + }) + + it("resumes across a boundary that splits documents sharing an idField value", async () => { + // Three source docs per idField value — a bill number appearing in three + // general courts — with distinct paths. 250 is not a multiple of 3, so the + // page boundary lands inside a group; a value cursor on idField would skip + // the rest of that group on resume, where the path cursor does not. + const grouped = Array.from({ length: 600 }, (_, n) => ({ + id: `H${pad(Math.floor(n / 3))}`, + body: "x", + path: pad(n) + })) + const first = await makeIndexer(grouped).backfillChunk({ + startAfter: null, + maxBatches: 1, + budgetMs: 60_000 + }) + expect(first.cursor).toBe(pad(249)) + const resumed = await makeIndexer(grouped).backfillChunk({ + startAfter: first.cursor!, + budgetMs: 60_000 + }) + expect(first.documents + resumed.documents).toBe(600) + }) + + it("counts documents that fail to convert without aborting the chunk", async () => { + const indexer = new SearchIndexer({ + alias: "widgets", + schema: { fields: [{ name: "body", type: "string" }] }, + sourceCollection: fakeSource(docs.slice(0, 3)) as any, + documentTrigger: "widgets/{id}", + idField: "id", + convert: (data: any) => { + if (data.id === pad(1)) throw Error("Invalid widget") + return { id: data.id, body: data.body } + } + } as CollectionConfig) + + const result = await indexer.backfillChunk({ + startAfter: null, + budgetMs: 60_000 + }) + expect(result.convertFailures).toBe(1) + expect(result.documents).toBe(2) + }) +}) diff --git a/functions/src/search/SearchIndexer.ts b/functions/src/search/SearchIndexer.ts index b1488edba..3371ab08e 100644 --- a/functions/src/search/SearchIndexer.ts +++ b/functions/src/search/SearchIndexer.ts @@ -1,40 +1,62 @@ import { Change } from "firebase-functions" import { isEqual, last } from "lodash" -import hash from "object-hash" import Collection from "typesense/lib/Typesense/Collection" -import { ImportResponse } from "typesense/lib/Typesense/Documents" -import { ImportError, ObjectNotFound } from "typesense/lib/Typesense/Errors" -import { db, DocumentData, DocumentSnapshot, QuerySnapshot } from "../firebase" +import { ObjectNotFound } from "typesense/lib/Typesense/Errors" +import { + db, + DocumentData, + DocumentSnapshot, + FieldPath, + QueryDocumentSnapshot +} from "../firebase" +import { BackfillConfig, upgradePath } from "./backfillRun" import { createClient } from "./client" +import { searchCollectionName } from "./collectionName" import { CollectionConfig } from "./config" -import { z } from "zod" import { Timestamp } from "../firebase" -export const BackfillConfig = z.object({ - numBatches: z.number().positive().optional() -}) -export type BackfillConfig = z.infer +/** Ceiling on one `documents().import()` body, held far enough under the 10 MB + * maximum payload of the AWS API Gateway HTTP API these collections sit behind + * that headers and encoding cannot push a batch over it. A batch over the cap + * is rejected outright and would fail the same way on every retry. Budgeting + * the import by serialized bytes rather than by document count is what makes + * that impossible: bills carry their full text in `body`, so a fixed count is + * safe at the mean and unsafe in the tail. + */ +export const IMPORT_BYTE_BUDGET = 6 * 1024 * 1024 + +export type BackfillChunkResult = { + /** Document path the next chunk resumes `startAfter`, or null when the + * source is spent. */ + cursor: string | null + batches: number + documents: number + convertFailures: number +} + +/** The id of a failed import's `document`, which the server echoes back as the + * original JSONL line (a string) — parsed defensively, for logs only. */ +const failedDocumentId = (document: unknown): string | undefined => { + try { + return typeof document === "string" + ? JSON.parse(document).id + : (document as { id?: string } | undefined)?.id + } catch { + return undefined + } +} export class SearchIndexer { - private readonly batchSize = 100 + /** How many source documents to read per Firestore page. Independent of the + * import payload, which `importInSlices` sizes by bytes. */ + private readonly batchSize = 250 private readonly client = createClient() private readonly collectionName: string private collection: Collection | undefined constructor(private readonly config: CollectionConfig) { - const schemaHash = hash( - { - schema: config.schema, - filter: config.filter?.toString(), - convert: config.convert.toString() - }, - { - algorithm: "md5", - unorderedArrays: true - } - ) - this.collectionName = `${config.alias}_${schemaHash}` + this.collectionName = searchCollectionName(config) } private passesFilter(data: DocumentData | undefined) { @@ -48,7 +70,12 @@ export class SearchIndexer { } } - static upgradePath = (alias: string) => `/search/upgrade-${alias}` + /** The collection this config currently hashes to. A chunk compares it + * against the name recorded on the run to notice that the deployed code + * changed underneath an in-flight backfill. */ + get targetCollectionName() { + return this.collectionName + } async scheduleUpgradeIfNeeded(backfillConfig: unknown) { const config = BackfillConfig.parse(backfillConfig) @@ -58,8 +85,11 @@ export class SearchIndexer { console.log(`Index for alias ${alias} up to date: ${isCollectionUpToDate}`) if (!isCollectionUpToDate) { console.log(`Scheduling upgrade for alias ${alias}`) - const upgradeDoc = db.doc(SearchIndexer.upgradePath(alias)) - await upgradeDoc.delete() + const upgradeDoc = db.doc(upgradePath(alias)) + // Recursive, not a plain delete: deleting a document leaves its + // subcollections behind, and a surviving `chunks/0` from the previous run + // would make this run's `create` throw. + await db.recursiveDelete(upgradeDoc) await upgradeDoc.create({ createdAt: Timestamp.now(), ...config @@ -67,11 +97,14 @@ export class SearchIndexer { } } - async performUpgrade(backfillConfig: unknown) { - const config = BackfillConfig.parse(backfillConfig) - // Ensure collection exists + /** Ensures the target collection exists so chunks can import into it. */ + async beginUpgrade() { await this.getCollection() - await this.backfill(config) + } + + /** Points the alias at the freshly backfilled collection and drops the old + * one. Only called once the source is exhausted. */ + async finishUpgrade() { await this.upgradeAlias() } @@ -83,7 +116,7 @@ export class SearchIndexer { if (!afterData || !this.passesFilter(afterData)) { if (beforeData && this.passesFilter(beforeData)) { const { id } = this.config.convert(beforeData) - await (await this.getCollection()).documents().delete(id) + await (await this.getCollection()).documents(id).delete() } return } @@ -127,45 +160,110 @@ export class SearchIndexer { .create({ name: this.collectionName, ...this.config.schema }) } - private async backfill({ numBatches }: BackfillConfig) { + /** Moves as much of the source into the target collection as the given budget + * allows, starting after `startAfter`, and reports where it stopped. A null + * cursor in the result means the source is exhausted; anything else is the + * checkpoint the next chunk resumes from. + */ + async backfillChunk({ + startAfter, + maxBatches, + budgetMs + }: { + startAfter: string | null + maxBatches?: number + budgetMs: number + }): Promise { const { convert } = this.config - let currentBatch = 0 - for await (const batch of this.listCollection()) { - currentBatch++ - if (numBatches && currentBatch > numBatches) return - - const docs = batch.reduce((acc, d) => { - try { - const data = d.data() - if (!this.passesFilter(data)) return acc - const doc = convert(data) - acc.push(doc) - } catch (error: any) { - console.error(`Failed to convert document: ${error.message}`) - } - return acc - }, [] as any[]) - const collection = await this.getCollection() - try { - await collection.documents().import(docs, { action: "upsert" }) - } catch (e) { - if (e instanceof ImportError) { - const results = e.importResults as unknown as ImportResponse[] - console.error( - results - .map( - r => - !r.success && { - code: r.code, - error: r.error, - id: r.document.id - } - ) - .filter(Boolean) - ) - } - throw e + const deadline = Date.now() + budgetMs + let cursor: string | null = startAfter + let batches = 0 + let documents = 0 + let convertFailures = 0 + + while (maxBatches === undefined || batches < maxBatches) { + // Checked after the first page so a chunk always makes progress, however + // little budget it inherited. + if (batches > 0 && Date.now() >= deadline) break + + const page = await this.listPage(cursor) + cursor = page.cursor + + if (page.docs.length) { + batches++ + const docs = page.docs.reduce((acc, d) => { + try { + const data = d.data() + if (!this.passesFilter(data)) return acc + acc.push(convert(data)) + } catch (error: any) { + convertFailures++ + console.error(`Failed to convert document: ${error.message}`) + } + return acc + }, [] as any[]) + + await this.importInSlices(docs) + documents += docs.length } + + if (cursor === null) break + } + + return { cursor, batches, documents, convertFailures } + } + + private async importInSlices(docs: any[]) { + if (!docs.length) return + const collection = await this.getCollection() + let slice: string[] = [] + let bytes = 0 + + const flush = async () => { + if (!slice.length) return + await this.importDocuments(collection, slice) + slice = [] + bytes = 0 + } + + for (const doc of docs) { + // Serialized once, here: the same line is measured against the budget + // and shipped as the import body, rather than stringified a second time + // inside the client. Bytes, not string length: the cap is on the encoded + // body, and `.length` counts UTF-16 units, which undercounts anything + // non-ASCII. +1 for the JSONL newline. + const line = JSON.stringify(doc) + const size = Buffer.byteLength(line) + 1 + if (slice.length && bytes + size > IMPORT_BYTE_BUDGET) await flush() + slice.push(line) + bytes += size + } + await flush() + } + + /** Imports pre-serialized JSONL lines. The client's string form returns the + * raw per-line results WITHOUT throwing ImportError — only its array form + * does — so failures are detected here, and must be: a silently rejected + * batch would otherwise count as progress. */ + private async importDocuments(collection: Collection, lines: string[]) { + const response = await collection + .documents() + .import(lines.join("\n"), { action: "upsert" }) + const failures = String(response) + .split("\n") + .map(line => JSON.parse(line)) + .filter(r => r.success === false) + if (failures.length) { + console.error( + failures.map(r => ({ + code: r.code, + error: r.error, + id: failedDocumentId(r.document) + })) + ) + throw Error( + `${failures.length} of ${lines.length} documents failed to import` + ) } } @@ -191,18 +289,32 @@ export class SearchIndexer { } } - private async *listCollection() { - let token: string | undefined = "" - while (token !== undefined) { - const result: QuerySnapshot = await this.config.sourceCollection - .orderBy(this.config.idField) - .startAfter(token) - .limit(this.batchSize) - .get() + /** One ordered page of the source, with the cursor to resume after it — null + * once the source is exhausted, which a short page already tells us. + * + * Ordered by document name, with the last document's path as the cursor: the + * one ordering Firestore always serves without an index, and the one value + * unique per document, so it survives being written into a chunk document. + * Ordering by `idField` instead would make the persisted value-cursor skip + * documents — Firestore positions a value cursor after ALL documents equal + * to it, and bills' collectionGroup holds the same bill number once per + * general court, adjacent in that ordering, so page boundaries would + * silently drop the rest of a boundary-straddling group from the index. + */ + private async listPage(startAfter: string | null): Promise<{ + docs: QueryDocumentSnapshot[] + cursor: string | null + }> { + let query = this.config.sourceCollection + .orderBy(FieldPath.documentId()) + .limit(this.batchSize) + if (startAfter !== null) query = query.startAfter(db.doc(startAfter)) - const docs = result.docs.filter(d => d.exists) - token = last(docs)?.id - if (docs.length) yield docs + const { docs } = await query.get() + const tail = docs.length < this.batchSize ? undefined : last(docs) + return { + docs, + cursor: tail ? tail.ref.path : null } } } diff --git a/functions/src/search/backfillRun.test.ts b/functions/src/search/backfillRun.test.ts new file mode 100644 index 000000000..bcf2deceb --- /dev/null +++ b/functions/src/search/backfillRun.test.ts @@ -0,0 +1,53 @@ +import { + aliasForUpgradeId, + chunkPath, + MAX_CHUNKS, + nextChunk, + upgradePath +} from "./backfillRun" + +const totals = { batches: 4, documents: 1000, convertFailures: 2 } +const run = { runId: "run-1", index: 0, cursor: "H1000", totals } + +describe("nextChunk", () => { + it("finishes when the source is exhausted", () => { + expect(nextChunk({ ...run, cursor: null })).toEqual({ type: "done" }) + }) + + it("chains the next chunk from the cursor and totals it was handed", () => { + expect(nextChunk(run)).toEqual({ + type: "next", + chunk: { runId: "run-1", index: 1, cursor: "H1000", before: totals } + }) + }) + + it("finishes once the numBatches budget is spent, mid-source", () => { + expect(nextChunk({ ...run, numBatches: 4 })).toEqual({ type: "done" }) + }) + + it("keeps chaining while the numBatches budget has room", () => { + expect(nextChunk({ ...run, numBatches: 5 })).toMatchObject({ + type: "next", + chunk: { index: 1, cursor: "H1000" } + }) + }) + + it("fails rather than chaining past the loop guard", () => { + expect(nextChunk({ ...run, index: MAX_CHUNKS - 1 })).toEqual({ + type: "failed", + error: expect.stringContaining("not advancing") + }) + }) +}) + +describe("paths", () => { + it("round trips the alias through the upgrade document id", () => { + expect(upgradePath("bills")).toBe("/search/upgrade-bills") + expect(aliasForUpgradeId("upgrade-bills")).toBe("bills") + expect(chunkPath("bills", 7)).toBe("/search/upgrade-bills/chunks/7") + }) + + it("ignores documents under /search that are not upgrade runs", () => { + expect(aliasForUpgradeId("billSearchIndex")).toBeNull() + }) +}) diff --git a/functions/src/search/backfillRun.ts b/functions/src/search/backfillRun.ts new file mode 100644 index 000000000..73b23a77f --- /dev/null +++ b/functions/src/search/backfillRun.ts @@ -0,0 +1,122 @@ +import { z } from "zod" + +/** An index upgrade is a chain of bounded chunks rather than one invocation. + * + * The run lives at `/search/upgrade-`; each chunk is a document in that + * run's `chunks` subcollection, and finishing a chunk writes the next one. The + * chunk document is the checkpoint: it carries both the cursor to resume from + * and the run totals as of its creation, so a chunk that times out or throws is + * replayed from its own starting point and converges on the same numbers rather + * than adding its work to the run twice. + */ + +/** Wall-clock a single chunk spends paging before it hands off. The chunk + * function gets the 540s v1 ceiling; the remainder is room to write progress + * and create the successor after the budget runs out mid-page. */ +export const CHUNK_BUDGET_MS = 300_000 + +/** Loop guard. Nothing legitimate reaches this — bills is ~49k documents at 250 + * per batch, and one chunk covers hundreds of batches — so hitting it means the + * cursor stopped advancing, and the run fails loudly instead of chaining. */ +export const MAX_CHUNKS = 500 + +/** `failurePolicy: true` retries the same event for up to seven days. Past this + * age the chunk gives up and marks the run failed, so a poisoned chunk cannot + * retry indefinitely. */ +export const MAX_EVENT_AGE_MS = 30 * 60_000 + +const UPGRADE_PREFIX = "upgrade-" + +export const upgradePath = (alias: string) => + `/search/${UPGRADE_PREFIX}${alias}` + +export const chunkPath = (alias: string, index: number) => + `${upgradePath(alias)}/chunks/${index}` + +/** Recovers the alias from an upgrade document's id, or null for a document + * under `/search` that is not an upgrade run. */ +export const aliasForUpgradeId = (upgradeId: string) => + upgradeId.startsWith(UPGRADE_PREFIX) + ? upgradeId.slice(UPGRADE_PREFIX.length) + : null + +/** How much of the source a run has moved so far. Carried on the chunk document + * as the baseline for that chunk, and on the run as the live total. */ +export const Totals = z.object({ + batches: z.number().int().nonnegative(), + documents: z.number().int().nonnegative(), + convertFailures: z.number().int().nonnegative() +}) +export type Totals = z.infer + +export const NO_TOTALS: Totals = { + batches: 0, + documents: 0, + convertFailures: 0 +} + +/** The only knob a scheduled upgrade takes: a whole-run batch budget that builds + * a deliberately partial index and swaps the alias anyway. A batch is one source + * page, so the document count it corresponds to is `numBatches * batchSize` — + * see `SearchIndexer.batchSize`. */ +export const BackfillConfig = z.object({ + numBatches: z.number().positive().optional() +}) +export type BackfillConfig = z.infer + +export const ChunkDoc = z.object({ + runId: z.string(), + index: z.number().int().nonnegative(), + /** Exclusive `startAfter` document path; null starts from the top. */ + cursor: z.string().nullable(), + before: Totals +}) +export type ChunkDoc = z.infer + +/** The run document. Timestamps are written for operators and never parsed, so + * they stay out of the schema. */ +export const UpgradeRun = BackfillConfig.extend({ + runId: z.string(), + collectionName: z.string(), + status: z.enum(["running", "done", "failed"]), + cursor: z.string().nullable(), + totals: Totals, + chunks: z.number().int().positive() +}) +export type UpgradeRun = z.infer + +export type NextChunk = + | { type: "done" } + | { type: "failed"; error: string } + | { type: "next"; chunk: ChunkDoc } + +/** Decides what happens after a chunk finishes: swap the alias, chain another + * chunk, or fail the run. Kept pure so the `numBatches` budget and the loop + * guard are settled in one place that tests can reach without Firestore. + */ +export const nextChunk = ({ + runId, + index, + cursor, + totals, + numBatches +}: { + runId: string + index: number + cursor: string | null + totals: Totals + numBatches?: number +}): NextChunk => { + if (cursor === null) return { type: "done" } + if (numBatches !== undefined && totals.batches >= numBatches) + return { type: "done" } + if (index + 1 >= MAX_CHUNKS) + return { + type: "failed", + error: `Backfill exceeded ${MAX_CHUNKS} chunks at cursor ${cursor}; the cursor is not advancing` + } + return { + type: "next", + chunk: { runId, index: index + 1, cursor, before: totals } + } +} diff --git a/functions/src/search/checkSearchIndexVersion.ts b/functions/src/search/checkSearchIndexVersion.ts index 8af4bf0db..f3d0b72e8 100644 --- a/functions/src/search/checkSearchIndexVersion.ts +++ b/functions/src/search/checkSearchIndexVersion.ts @@ -1,6 +1,8 @@ import { runWith } from "firebase-functions" +import { createClient } from "./client" import { getRegisteredConfigs } from "./config" import { SearchIndexer } from "./SearchIndexer" +import { upsertLegislativeSynonyms } from "./synonyms" /** Schedules index upgrades for each config/alias(bills/hearing/testimony) if necessary. Requires a message * wtih content `{ "check": true}` */ @@ -14,4 +16,10 @@ export const checkSearchIndexVersion = runWith({ for (const config of getRegisteredConfigs()) { await new SearchIndexer(config).scheduleUpgradeIfNeeded(message.json) } + // After the upgrades are scheduled, so a failure here cannot lose a + // reindex; a throw still fails the invocation and shows up in monitoring. + const synonyms = await upsertLegislativeSynonyms(createClient()) + console.log( + `Upserted synonym set "${synonyms.name}" with ${synonyms.items} items` + ) }) diff --git a/functions/src/search/collectionName.test.ts b/functions/src/search/collectionName.test.ts new file mode 100644 index 000000000..187849277 --- /dev/null +++ b/functions/src/search/collectionName.test.ts @@ -0,0 +1,99 @@ +import { CollectionVersionInputs, searchCollectionName } from "./collectionName" + +const base: CollectionVersionInputs = { + alias: "widgets", + schema: { + fields: [ + { name: "name", type: "string", facet: false }, + { name: "count", type: "int32" } + ], + default_sorting_field: "count" + }, + convert: (data: any) => ({ id: data.id, name: data.name, count: data.count }) +} + +describe("searchCollectionName", () => { + /** Pinned so that adding a key to the hashed object — the easy mistake, since + * it renames every collection at once and forces a full backfill of each — + * fails here rather than in production. Only `base` above, or something that + * changes how it is formatted or transpiled, should move this literal. + */ + it("hashes a fixed config to a stable name", () => { + expect(searchCollectionName(base)).toEqual( + "widgets_8efb32501b99da0cd32c2ef16fe735c7" + ) + }) + + it("distinguishes collections by alias", () => { + expect(searchCollectionName({ ...base, alias: "gadgets" })).not.toEqual( + searchCollectionName(base) + ) + }) + + it("changes when the schema changes", () => { + const schema = { + ...base.schema, + fields: [ + ...base.schema.fields, + { name: "color", type: "string" as const } + ] + } + expect(searchCollectionName({ ...base, schema })).not.toEqual( + searchCollectionName(base) + ) + }) + + it("changes when convert's own source changes", () => { + const convert = (data: any) => ({ + id: data.id, + name: data.name.trim(), + count: data.count + }) + expect(searchCollectionName({ ...base, convert })).not.toEqual( + searchCollectionName(base) + ) + }) + + it("changes when a filter is added or edited", () => { + const filtered = { ...base, filter: (data: any) => data.count > 0 } + expect(searchCollectionName(filtered)).not.toEqual( + searchCollectionName(base) + ) + expect( + searchCollectionName({ ...base, filter: (data: any) => data.count > 1 }) + ).not.toEqual(searchCollectionName(filtered)) + }) + + /** The reason convertVersion exists: convert.toString() covers only the + * function's own body, so a config whose convert delegates to a helper is + * blind to changes in that helper. + */ + it("is blind to a helper change until convertVersion is bumped", () => { + let build = (data: any) => ({ id: data.id, name: data.name }) + const convert = (data: any) => build(data) + + const before = searchCollectionName({ ...base, convert }) + build = (data: any) => ({ id: data.id, name: data.name.toUpperCase() }) + expect(searchCollectionName({ ...base, convert })).toEqual(before) + + // ...which is what bumping convertVersion is for. + expect( + searchCollectionName({ ...base, convert, convertVersion: 2 }) + ).not.toEqual(before) + }) + + describe("convertVersion", () => { + it("is omitted from the hash when unset, so existing collections keep their names", () => { + expect( + searchCollectionName({ ...base, convertVersion: undefined }) + ).toEqual(searchCollectionName(base)) + }) + + it("yields a distinct name for every value", () => { + const names = [undefined, 1, 2, "2", "rebuild-2026-08"].map( + convertVersion => searchCollectionName({ ...base, convertVersion }) + ) + expect(names).toEqual(Array.from(new Set(names))) + }) + }) +}) diff --git a/functions/src/search/collectionName.ts b/functions/src/search/collectionName.ts new file mode 100644 index 000000000..b5e736772 --- /dev/null +++ b/functions/src/search/collectionName.ts @@ -0,0 +1,39 @@ +import hash from "object-hash" +import type { CollectionConfig } from "./config" + +/** Everything about a config that determines the contents of its index. */ +export type CollectionVersionInputs = Pick< + CollectionConfig, + "alias" | "schema" | "convert" | "filter" | "convertVersion" +> + +/** Names a collection after a hash of the inputs that decide what ends up in + * it, so changing any of them yields a new name. SearchIndexer compares that + * name against the one the alias currently points at to decide whether to + * build and backfill a replacement. + * + * The hash sees `convert`'s own source text and nothing more, so changes it + * cannot see need `convertVersion` — see its doc comment in ./config.ts. + */ +export const searchCollectionName = ({ + alias, + schema, + convert, + filter, + convertVersion +}: CollectionVersionInputs) => { + const versionHash = hash( + { + schema, + // The shapes below are what the live collections already hash to: filter + // passes undefined when unset, convertVersion omits its key entirely. + // Making the two consistent, in either direction, renames every + // collection and forces a full backfill of each. + filter: filter?.toString(), + convert: convert.toString(), + ...(convertVersion === undefined ? {} : { convertVersion }) + }, + { algorithm: "md5", unorderedArrays: true } + ) + return `${alias}_${versionHash}` +} diff --git a/functions/src/search/config.ts b/functions/src/search/config.ts index e17da089e..a1571e17a 100644 --- a/functions/src/search/config.ts +++ b/functions/src/search/config.ts @@ -12,6 +12,12 @@ export type CollectionConfig = { readonly idField: string readonly convert: (data: DocumentData) => T readonly filter?: (data: DocumentData) => boolean + /** Bump to force a reindex when the indexed output changed but `convert`'s + * own source did not — the collection name hashes `convert.toString()`, which + * covers none of the helpers, validators or defaults `convert` imports from + * other modules. See searchCollectionName in ./collectionName.ts. + */ + readonly convertVersion?: string | number } const registered: CollectionConfig[] = [] @@ -19,3 +25,19 @@ export const registerConfig = (config: CollectionConfig) => { registered.push(config) } export const getRegisteredConfigs = () => registered + +/** The config for one alias. A config registers itself when its module is + * loaded, so a miss usually means the caller forgot to require + * `functions/src//search` — the error names what is loaded so that + * is visible rather than guessed at. + */ +export const configForAlias = (alias: string): CollectionConfig => { + const config = registered.find(c => c.alias === alias) + if (!config) + throw Error( + `No search config registered for "${alias}". Registered: ${ + registered.map(c => c.alias).join(", ") || "(none)" + }` + ) + return config +} diff --git a/functions/src/search/createSearchIndexer.ts b/functions/src/search/createSearchIndexer.ts index 8a6eee724..223634cba 100644 --- a/functions/src/search/createSearchIndexer.ts +++ b/functions/src/search/createSearchIndexer.ts @@ -1,4 +1,18 @@ -import { runWith } from "firebase-functions" +import { EventContext, runWith } from "firebase-functions" +import { nanoid } from "nanoid" +import { db, QueryDocumentSnapshot, Timestamp } from "../firebase" +import { + BackfillConfig, + ChunkDoc, + chunkPath, + CHUNK_BUDGET_MS, + MAX_EVENT_AGE_MS, + NO_TOTALS, + nextChunk, + Totals, + upgradePath, + UpgradeRun +} from "./backfillRun" import { BaseRecord, CollectionConfig, registerConfig } from "./config" import { SearchIndexer } from "./SearchIndexer" @@ -8,13 +22,32 @@ export function createSearchIndexer( registerConfig(config) return { upgradeSearchIndex: runWith({ - timeoutSeconds: 240, + /** This only starts the run — create the collection, record it, write the + * first chunk. The backfill itself is a chain of `runBackfillChunk` + * invocations, each with its own 540s budget and its own cursor, so the + * work no longer has to fit in one timeout to converge. + */ + timeoutSeconds: 120, secrets: ["TYPESENSE_API_KEY"] }) - .firestore.document(SearchIndexer.upgradePath(config.alias)) - .onCreate(async snap => { - await new SearchIndexer(config).performUpgrade(snap.data()) - }), + .firestore.document(upgradePath(config.alias)) + .onCreate(snap => startUpgrade(config, snap)), + + /** Runs one chunk of a backfill, then either chains the next chunk or swaps + * the alias. `failurePolicy` gives the retries: a chunk resumes from its own + * cursor, imports upserts into a collection nothing is aliased to yet, and + * writes totals relative to the baseline on its own document, so running it + * more than once converges rather than double-counting. + */ + runBackfillChunk: runWith({ + timeoutSeconds: 540, + memory: "512MB", + secrets: ["TYPESENSE_API_KEY"], + failurePolicy: true + }) + .firestore.document(`${upgradePath(config.alias)}/chunks/{chunkId}`) + .onCreate((snap, context) => advanceBackfill(config, snap, context)), + syncToSearchIndex: runWith({ timeoutSeconds: 30, secrets: ["TYPESENSE_API_KEY"] @@ -25,3 +58,235 @@ export function createSearchIndexer( }) } } + +async function startUpgrade( + config: CollectionConfig, + snap: QueryDocumentSnapshot +) { + const { numBatches } = BackfillConfig.parse(snap.data()) + + // Deliveries are at-least-once: a replayed create event must not mint a + // second runId — overwriting the recorded one would orphan every chunk of + // the run already in flight, since each would fail loadRun's runId check. + // A replay only makes sure the recorded run has its first chunk, which also + // repairs a first delivery that crashed between the two writes below. + const existing = (await snap.ref.get()).data() + if (existing?.runId) { + console.log( + `Upgrade ${snap.ref.path} already started (run ${existing.runId}); ensuring chunk 0 exists` + ) + await createChunk(config.alias, { + runId: String(existing.runId), + index: 0, + cursor: null, + before: NO_TOTALS + }) + return + } + + const indexer = new SearchIndexer(config) + await indexer.beginUpgrade() + const runId = nanoid() + + const run: UpgradeRun = { + runId, + collectionName: indexer.targetCollectionName, + status: "running", + cursor: null, + totals: NO_TOTALS, + chunks: 1, + ...(numBatches === undefined ? {} : { numBatches }) + } + // update(), not a merge set: if the upgrade document was deleted while this + // ran (a rescheduled upgrade), resurrecting it here would collide with the + // replacement run's create. + await snap.ref.update({ + ...run, + startedAt: Timestamp.now(), + updatedAt: Timestamp.now() + }) + + const chunk: ChunkDoc = { runId, index: 0, cursor: null, before: NO_TOTALS } + await createChunk(config.alias, chunk) + console.log( + `Started backfill of ${run.collectionName} (run ${runId})`, + numBatches === undefined ? "" : `limited to ${numBatches} batches` + ) +} + +/** Creates a chunk document, treating "it already exists" as success: chunk + * events are delivered at least once, so a replayed invocation can find the + * successor it already created. Throwing instead would loop failurePolicy's + * retries into the age gate, which would mark a healthy run failed. */ +async function createChunk(alias: string, chunk: ChunkDoc) { + try { + await db.doc(chunkPath(alias, chunk.index)).create(chunk) + } catch (e: any) { + const alreadyExists = + e?.code === 6 || /ALREADY[_-]?EXISTS/i.test(String(e?.message ?? e)) + if (!alreadyExists) throw e + console.log( + `Chunk ${chunk.index} of ${alias} already exists; leaving it to its own event` + ) + } +} + +async function advanceBackfill( + config: CollectionConfig, + snap: QueryDocumentSnapshot, + context: EventContext +) { + const { alias } = config + const runRef = snap.ref.parent.parent! + const indexer = new SearchIndexer(config) + + // The age gate runs before anything that can throw — parsing included — + // or a persistently unparseable chunk or run document would retry for + // failurePolicy's full seven days with the run never marked failed. + // Returning without a throw is what ends the retry chain; the run is + // additionally failed when the chunk provably belongs to it, since only + // then is writing to the run document safe. + const age = Date.now() - Date.parse(context.timestamp) + if (age > MAX_EVENT_AGE_MS) { + const error = `Chunk ${snap.id} of ${alias} still failing ${Math.round( + age / 60_000 + )} minutes after it was created; giving up` + console.error(error) + try { + const staleChunk = ChunkDoc.parse(snap.data()) + const staleRun = await loadRun( + runRef, + staleChunk, + indexer.targetCollectionName + ) + if (staleRun) await fail(runRef, error) + } catch (e) { + console.error(`Could not mark the run failed: ${e}`) + } + return + } + + const chunk = ChunkDoc.parse(snap.data()) + const run = await loadRun(runRef, chunk, indexer.targetCollectionName) + if (!run) return + + try { + const result = await indexer.backfillChunk({ + startAfter: chunk.cursor, + maxBatches: + run.numBatches === undefined + ? undefined + : Math.max(run.numBatches - chunk.before.batches, 0), + budgetMs: CHUNK_BUDGET_MS + }) + + // Absolute, not incremental: a replayed chunk redoes exactly its own range, + // so baseline + result lands on the same totals however often it runs. + const totals: Totals = { + batches: chunk.before.batches + result.batches, + documents: chunk.before.documents + result.documents, + convertFailures: chunk.before.convertFailures + result.convertFailures + } + console.log( + `Chunk ${chunk.index} of ${alias}: ${result.batches} batches, ${result.documents} documents, cursor ${result.cursor}` + ) + + const progress = { + cursor: result.cursor, + totals, + updatedAt: Timestamp.now() + } + const next = nextChunk({ + runId: chunk.runId, + index: chunk.index, + cursor: result.cursor, + totals, + numBatches: run.numBatches + }) + + switch (next.type) { + case "done": + await indexer.finishUpgrade() + await runRef.update({ + ...progress, + status: "done", + finishedAt: Timestamp.now() + }) + console.log( + `Backfill of ${run.collectionName} complete after ${totals.batches} batches` + ) + break + case "failed": + await fail(runRef, next.error, progress) + break + case "next": + await runRef.update({ ...progress, chunks: next.chunk.index + 1 }) + await createChunk(alias, next.chunk) + break + } + } catch (e: any) { + // Recorded for operators, then rethrown so the retry policy sees a failure + // and runs this chunk again from the same cursor. update(), not a merge + // set: if the run document was deleted mid-chunk by a rescheduled upgrade, + // resurrecting a partial copy of it would break the replacement run. + try { + await runRef.update({ + lastError: String(e?.message ?? e), + updatedAt: Timestamp.now() + }) + } catch { + // Run document gone; the retry's loadRun will drop this chunk. + } + throw e + } +} + +/** Reads the run this chunk belongs to, or null when the chunk has been + * superseded — a second upgrade was scheduled, the run already finished, or the + * deployed config now hashes to a different collection. */ +async function loadRun( + runRef: FirebaseFirestore.DocumentReference, + chunk: ChunkDoc, + targetCollectionName: string +) { + const snap = await runRef.get() + if (!snap.exists) { + console.log(`Upgrade ${runRef.path} is gone; dropping chunk ${chunk.index}`) + return null + } + const run = UpgradeRun.parse(snap.data()) + if (run.runId !== chunk.runId) { + console.log( + `Chunk ${chunk.index} belongs to run ${chunk.runId}, but ${run.runId} is current; dropping it` + ) + return null + } + if (run.status !== "running") { + console.log( + `Run ${run.runId} is ${run.status}; dropping chunk ${chunk.index}` + ) + return null + } + if (run.collectionName !== targetCollectionName) { + console.log( + `Run ${run.runId} targets ${run.collectionName} but this deploy builds ${targetCollectionName}; dropping chunk ${chunk.index}` + ) + return null + } + return run +} + +async function fail( + runRef: FirebaseFirestore.DocumentReference, + error: string, + progress: object = {} +) { + // update(), not a merge set, so a deleted run document stays deleted — see + // the catch in advanceBackfill. + await runRef.update({ + ...progress, + status: "failed", + lastError: error, + finishedAt: Timestamp.now() + }) +} diff --git a/functions/src/search/synonyms.test.ts b/functions/src/search/synonyms.test.ts new file mode 100644 index 000000000..0a36cf07f --- /dev/null +++ b/functions/src/search/synonyms.test.ts @@ -0,0 +1,11 @@ +import { legislativeSynonymItems } from "./synonyms" + +describe("legislativeSynonymItems", () => { + /** The constructors make the other invariants unrepresentable; a duplicate + * id is the one mistake left — two entries for the same head or root, where + * the second silently replaces the first on the server. */ + it("has unique ids", () => { + const ids = legislativeSynonymItems.map(i => i.id) + expect(new Set(ids).size).toEqual(ids.length) + }) +}) diff --git a/functions/src/search/synonyms.ts b/functions/src/search/synonyms.ts new file mode 100644 index 000000000..56359c89e --- /dev/null +++ b/functions/src/search/synonyms.ts @@ -0,0 +1,80 @@ +/** The curated legislative synonym set, referenced by name from every search's + * `synonym_sets` param (components/search/searchParams.ts imports the name). + * + * It is server state that no collection carries, so `checkSearchIndexVersion` + * re-upserts it on every deploy — the same trigger that keeps the collections + * current. See docs/search-deployment.md for how a missing set fails. + */ +import type { Client } from "typesense" +import type { SynonymItemSchema } from "typesense/lib/Typesense/SynonymSets" + +export const SYNONYM_SET_NAME = "legislative" + +/** A multi-way set expands every member to every other member. The head term + * is the id and always a member, so the set cannot omit it (#90). */ +const multiWay = (head: string, ...rest: string[]): SynonymItemSchema => ({ + id: head, + synonyms: [head, ...rest] +}) + +/** A one-way set matches `root` on the query side only; keeping the root out + * of `synonyms` is what keeps it one-way. */ +const oneWay = (root: string, ...synonyms: string[]): SynonymItemSchema => ({ + id: root.replace(/\s+/g, "-"), + root, + synonyms +}) + +export const legislativeSynonymItems: SynonymItemSchema[] = [ + multiWay("firearms", "firearm", "gun", "guns"), + multiWay("educators", "educator", "teacher", "teachers"), + multiWay("cannabis", "marijuana", "marihuana"), + oneWay("doctor", "physician"), + oneWay("doctors", "physicians", "physician"), + ...["senior citizens", "seniors", "elders", "older adults"].map(root => + oneWay(root, "elderly") + ), + multiWay("alcohol", "liquor", "alcoholic beverages", "alcoholic beverage"), + ...["child care", "daycare", "day care"].map(root => + oneWay(root, "childcare") + ), + oneWay("healthcare", "health care"), + multiWay( + "oui", + "dui", + "dwi", + "drunk driving", + "operating under the influence" + ), + multiWay("mbta", "massachusetts bay transportation authority"), + oneWay("public transit", "mbta"), + ...["servicemember", "service member"].map(root => + oneWay(root, "veteran", "veterans") + ), + multiWay("climate", "climate change", "global warming"), + multiWay("tenants", "tenant", "renter", "renters"), + multiWay("police", "law enforcement"), + multiWay( + "vehicles", + "vehicle", + "car", + "cars", + "automobile", + "automobiles", + "motor vehicle", + "motor vehicles" + ), + multiWay("children", "child", "kids"), + oneWay("opiate", "opioid"), + oneWay("opiates", "opioids", "opioid"), + multiWay("incarceration", "prison", "prisons", "correctional facility"), + multiWay("disability", "disabled", "disabilities"), + oneWay("clean energy", "renewable energy") +] + +export async function upsertLegislativeSynonyms(client: Client) { + await client + .synonymSets(SYNONYM_SET_NAME) + .upsert({ items: legislativeSynonymItems }) + return { name: SYNONYM_SET_NAME, items: legislativeSynonymItems.length } +} diff --git a/functions/src/testimony/search.ts b/functions/src/testimony/search.ts index 5947b7ae5..0e3fc027d 100644 --- a/functions/src/testimony/search.ts +++ b/functions/src/testimony/search.ts @@ -1,18 +1,33 @@ import { db } from "../firebase" +import { + billNumberVariants, + billNumberVariantsVersion +} from "../bills/numberVariants" import { createSearchIndexer } from "../search" import { Testimony, TestimonySearchRecord } from "./types" export const { syncToSearchIndex: syncTestimonyToSearchIndex, - upgradeSearchIndex: upgradeTestimonySearchIndex + upgradeSearchIndex: upgradeTestimonySearchIndex, + runBackfillChunk: runTestimonyBackfillChunk } = createSearchIndexer({ sourceCollection: db.collectionGroup("publishedTestimony"), documentTrigger: "users/{uid}/publishedTestimony/{id}", alias: "publishedTestimony", idField: "id", + convertVersion: billNumberVariantsVersion, schema: { + /** Hyphens do not split tokens by default, so "vote-by-mail" indexes as one + * token and the spaced form people type cannot reach it — the two spellings + * return disjoint result sets. Splitting on "-" makes each form find both, + * and makes the parts individually searchable ("income" reaches + * "low-income"). Hyphenated compounds are everywhere in this corpus: + * low-income, long-term, well-being, community-based, in-person. + */ + token_separators: ["-"], fields: [ { name: "billId", type: "string", facet: true }, + { name: "billIdVariants", type: "string[]", facet: false }, { name: "court", type: "int32", facet: true }, { name: "position", type: "string", facet: true }, { name: "content", type: "string", facet: false }, @@ -38,6 +53,7 @@ export const { const record: TestimonySearchRecord = { id: testimony.id, billId: testimony.billId, + billIdVariants: billNumberVariants(testimony.billId), authorDisplayName: testimony.authorDisplayName, court: testimony.court, position: testimony.position, diff --git a/functions/src/testimony/types.ts b/functions/src/testimony/types.ts index 429ad08bb..450792509 100644 --- a/functions/src/testimony/types.ts +++ b/functions/src/testimony/types.ts @@ -1,4 +1,5 @@ import { + Array, Boolean, InstanceOf, Literal as L, @@ -74,6 +75,9 @@ export const countsByPositions = { export const TestimonySearchRecord = R({ id: RtString, billId: RtString, + /** Space-separated forms of billId so "S 531" matches. See + * functions/src/bills/numberVariants.ts. */ + billIdVariants: Array(RtString), court: Number, position: Union(L("endorse"), L("oppose"), L("neutral")), content: RtString, diff --git a/functions/yarn.lock b/functions/yarn.lock index c4190ab58..bfd740395 100644 --- a/functions/yarn.lock +++ b/functions/yarn.lock @@ -400,6 +400,11 @@ dependencies: regenerator-runtime "^0.14.0" +"@babel/runtime@^7.24.1": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.29.7.tgz#12022450c45a4da6d8d8287b18a4ff2ddb23f768" + integrity sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw== + "@babel/template@^7.22.15", "@babel/template@^7.3.3": version "7.22.15" resolved "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz" @@ -1885,12 +1890,15 @@ axios@^0.25.0: dependencies: follow-redirects "^1.14.7" -axios@^0.26.0: - version "0.26.1" - resolved "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz" - integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== +axios@^1.15.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.19.0.tgz#ddf864d4c8233c0e6873746ab59361537d05ad39" + integrity sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw== dependencies: - follow-redirects "^1.14.8" + follow-redirects "^1.16.0" + form-data "^4.0.6" + https-proxy-agent "^5.0.1" + proxy-from-env "^2.1.0" b4a@^1.6.4: version "1.6.7" @@ -3477,11 +3485,16 @@ fn.name@1.x.x: resolved "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.7, follow-redirects@^1.14.8: +follow-redirects@^1.14.7: version "1.15.3" resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz" integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== +follow-redirects@^1.16.0: + version "1.16.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" + integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== + foreground-child@^3.1.0: version "3.2.1" resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz" @@ -3512,6 +3525,17 @@ form-data@^4.0.1: es-set-tostringtag "^2.1.0" mime-types "^2.1.12" +form-data@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.6.tgz#28e864e1b786dbebb68db1f452f9635278665827" + integrity sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.4" + mime-types "^2.1.35" + formdata-polyfill@^4.0.10: version "4.0.10" resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" @@ -4037,6 +4061,13 @@ hasown@^2.0.0, hasown@^2.0.2: dependencies: function-bind "^1.1.2" +hasown@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.4.tgz#8c62d8cb90beb2aad5d0a5b67581ad9854c3f003" + integrity sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A== + dependencies: + function-bind "^1.1.2" + heap-js@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/heap-js/-/heap-js-2.3.0.tgz" @@ -4102,7 +4133,7 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1, http-proxy-agent@^7.0.2: agent-base "^7.1.0" debug "^4.3.4" -https-proxy-agent@^5.0.0: +https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz" integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== @@ -5251,10 +5282,10 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" -loglevel@^1.8.0: - version "1.8.1" - resolved "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz" - integrity sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg== +loglevel@^1.9.2: + version "1.9.2" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" + integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== long@^4.0.0: version "4.0.0" @@ -6390,6 +6421,11 @@ proxy-from-env@^1.1.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== +proxy-from-env@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" + integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== + pump@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz" @@ -7422,6 +7458,11 @@ tslib@^2.0.1, tslib@^2.1.0: resolved "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tslib@^2.8.1: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + tsscmp@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/tsscmp/-/tsscmp-1.0.6.tgz#85b99583ac3589ec4bfef825b5000aa911d605eb" @@ -7469,18 +7510,19 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -typescript@4.5.5: - version "4.5.5" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz" - integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== +typescript@5.3.3: + version "5.3.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" + integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== -typesense@^1.2.2: - version "1.7.2" - resolved "https://registry.npmjs.org/typesense/-/typesense-1.7.2.tgz" - integrity sha512-hgQESOiyNJq+w2mpRJa/a1UMhWtJ/+sb0p7NoeCDSkikm9sasisJdnc7uhQchM6vTWKw2sMLWUBNbAhItR6zUQ== +typesense@^3.0.6: + version "3.0.6" + resolved "https://registry.yarnpkg.com/typesense/-/typesense-3.0.6.tgz#41a4094d26b214bb8c1be52e7824eccf728764e8" + integrity sha512-d3LL1qOLS8FCRxgAOqH+uDuK+VVA+/HYI1frU9fjYVwOg68mg/dX6XTtZ4yKKAkDCasB/se5uh/ZpMdOz/uJNg== dependencies: - axios "^0.26.0" - loglevel "^1.8.0" + axios "^1.15.0" + loglevel "^1.9.2" + tslib "^2.8.1" uc.micro@^1.0.1, uc.micro@^1.0.5: version "1.0.6" diff --git a/infra/Dockerfile.search b/infra/Dockerfile.search index e5dfe7d8f..8ebce0bb1 100644 --- a/infra/Dockerfile.search +++ b/infra/Dockerfile.search @@ -1,3 +1,4 @@ -FROM typesense/typesense:0.24.0 +# Keep in sync with the typesense service image in .github/workflows/repo-checks.yml +FROM typesense/typesense:30.2 RUN apt update && apt install -y curl \ No newline at end of file diff --git a/infra/app/templates/typesense-deployment.yml b/infra/app/templates/typesense-deployment.yml index afb266e15..4bf84ce9b 100644 --- a/infra/app/templates/typesense-deployment.yml +++ b/infra/app/templates/typesense-deployment.yml @@ -17,6 +17,8 @@ spec: spec: containers: - name: typesense + # Dormant: this chart is not deployed. The live Typesense servers run + # on AWS ECS, managed in https://github.com/maple-testimony/infra. image: typesense/typesense:0.24.0 ports: - containerPort: 8108 diff --git a/next.config.js b/next.config.js index 154e17d70..81c66826a 100644 --- a/next.config.js +++ b/next.config.js @@ -1,6 +1,12 @@ const i18Config = require("./next-i18next.config") const config = { + /** typesense-instantsearch-adapter 3.x ships ESM syntax in .js files whose + * package main is not marked "type": "module", so Next's default server + * externalization require()s it and node fails on its extensionless relative + * imports — 500ing every search page. Transpiling bundles it instead. + */ + transpilePackages: ["typesense-instantsearch-adapter"], images: { loader: "custom" }, diff --git a/package.json b/package.json index 5ef1ce0a4..dc66ff436 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "dev:backfill": "sh ./scripts/backfill-dev.sh", "emulators:start": "firebase --project demo-dtp emulators:start --only auth,functions,pubsub,firestore,storage --import tests/integration/exportedTestData", "typesense-admin": "ts-node --swc -P tsconfig.script.json scripts/typesense-admin.ts", + "search-eval": "ts-node --swc -P tsconfig.script.json scripts/search-eval", + "search-eval:corpus": "firebase --project demo-dtp emulators:exec --only firestore --import tests/integration/exportedTestData \"yarn firebase-admin run-script exportSearchCorpus --env local bills\"", "firebase-admin": "ts-node --swc -P tsconfig.script.json scripts/firebase-admin", "generate-stories": "ts-node -P tsconfig.script.json scripts/generate-stories", "compose": "docker compose -f infra/docker-compose.yml", @@ -137,8 +139,8 @@ "runtypes": "6.6.0", "sidebar-v2": "^0.4.0", "styled-components": "^5.3.3", - "typesense": "^1.2.2", - "typesense-instantsearch-adapter": "^2.4.0", + "typesense": "^3.0.6", + "typesense-instantsearch-adapter": "^3.0.2", "usehooks-ts": "^2.5.3", "web-vitals": "^0.2.4", "zod": "^3.20.2" diff --git a/public/locales/en/search.json b/public/locales/en/search.json index 2fa57a111..790d19775 100644 --- a/public/locales/en/search.json +++ b/public/locales/en/search.json @@ -8,6 +8,7 @@ "bill": { "court": "Legislative Session", "currentCommittee": "Current Committee", + "legislationType": "Document Type", "city": "City", "primarySponsor": "Primary Sponsor", "cosponsors": "Cosponsor" diff --git a/scripts/README.md b/scripts/README.md index 85d135f4b..0e10d451b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -300,6 +300,14 @@ Manages the Typesense search index. Integrates with Firebase secrets to retrieve yarn typesense-admin --env ``` +Unlike the rest of these scripts, `upsert-synonyms` writes **production search +state**: the deployed `legislative` synonym set must match +`functions/src/search/synonyms.ts` or synonym relevance silently degrades. +Deploys keep it current on their own (`checkSearchIndexVersion` upserts it), +so the command exists for local Typesense and for recovery. Deploy classes, +ordering rules, and verification live in +[`docs/search-deployment.md`](../docs/search-deployment.md). + ### Commands diff --git a/scripts/firebase-admin/exportSearchCorpus.ts b/scripts/firebase-admin/exportSearchCorpus.ts new file mode 100644 index 000000000..a86c5b354 --- /dev/null +++ b/scripts/firebase-admin/exportSearchCorpus.ts @@ -0,0 +1,79 @@ +import { last } from "lodash" +// Imported for the registerConfig side effect — all three, so any alias the +// caller passes resolves rather than only bills. +import "../../functions/src/bills/search" +import "../../functions/src/hearings/search" +import "../../functions/src/testimony/search" +import type { QueryDocumentSnapshot } from "../../functions/src/firebase" +import { configForAlias } from "../../functions/src/search/config" +import { corpusDir } from "../search-eval/corpus" +import { ExportedDoc, writeCorpus } from "../search-eval/corpusFiles" +import { Script } from "./types" + +const batchSize = 500 + +/** Extracts a search corpus from the emulator, seeded from the committed test + * fixture, using the production search converter — so the eval harness indexes + * exactly what the app's search pipeline would. + * + * This is the bills path (`yarn search-eval:corpus`). It uses the admin SDK, + * which bypasses security rules; an unauthenticated collectionGroup("bills") + * read is denied because the bills rule is path-scoped to generalCourts/**. + * + * Hearings and testimony come from a live project instead — the fixture holds + * ~21 hearings and lorem-ipsum testimony — and need no credentials at all: + * + * yarn search-eval corpus --env prod --alias hearings + * + * See tests/search-eval/README.md. + */ +export const script: Script = async ({ args }) => { + const alias: string = args.argv?.[0] + if (!alias) + throw Error( + "Pass the collection alias, e.g. `run-script exportSearchCorpus --env local bills`" + ) + + const config = configForAlias(alias) + + const docs: ExportedDoc[] = [] + let failures = 0 + + // A snapshot cursor, not the trailing id value: a snapshot carries the + // document-name tiebreak, where a value cursor positions after ALL documents + // equal to it — and bill numbers repeat across courts in the bills + // collection group, so a value cursor would silently drop the rest of a + // boundary-straddling group (the same hazard SearchIndexer.listPage + // documents). + let cursor: QueryDocumentSnapshot | undefined + for (;;) { + let query = config.sourceCollection.orderBy(config.idField).limit(batchSize) + if (cursor) query = query.startAfter(cursor) + const batch = await query.get() + for (const d of batch.docs) { + try { + const data = d.data() + if (config.filter && !config.filter(data)) continue + docs.push(config.convert(data) as ExportedDoc) + } catch (error: any) { + failures++ + console.error(`Failed to convert ${d.ref.path}: ${error.message}`) + } + } + cursor = last(batch.docs) + if (batch.size < batchSize) break + } + + const count = writeCorpus({ + alias, + source: "tests/integration/exportedTestData", + docs, + schema: config.schema + }) + + console.log( + `Wrote ${count} docs to ${corpusDir( + alias + )} (${failures} conversion failures)` + ) +} diff --git a/scripts/firebase-admin/searchUpgradeStatus.ts b/scripts/firebase-admin/searchUpgradeStatus.ts new file mode 100644 index 000000000..3229b0a62 --- /dev/null +++ b/scripts/firebase-admin/searchUpgradeStatus.ts @@ -0,0 +1,58 @@ +import { + aliasForUpgradeId, + UpgradeRun +} from "../../functions/src/search/backfillRun" +import { Script } from "./types" + +/** Prints the state of every search index upgrade run. + * + * The Firestore-side complement to `yarn typesense-admin status`, which sees + * only what Typesense holds. A backfill is a chain of chunk documents, so an + * in-flight run has a cursor and a batch count here long before the alias + * moves — which is the only way to tell "still working" from "stuck". + * + * Enumerates `/search` rather than taking the aliases from the config registry, + * so a run left behind by an alias that has since been renamed or removed still + * shows up — that run is exactly why an orphaned collection exists. + */ +const count = (n: number, one: string, many = `${one}s`) => + `${n} ${n === 1 ? one : many}` + +export const script: Script = async ({ db }) => { + const upgrades = await db.collection("search").listDocuments() + + for (const ref of upgrades) { + const alias = aliasForUpgradeId(ref.id) + if (!alias) continue + + const data = (await ref.get()).data() ?? {} + const run = UpgradeRun.partial().parse(data) + if (!run.runId || !run.totals) { + console.log(`${alias}: scheduled, not started`) + continue + } + + const { totals } = run + // Timestamps are written for operators and are not part of the schema. + const started = data.startedAt?.toDate() + const ended = data.finishedAt?.toDate() ?? data.updatedAt?.toDate() + const elapsed = + started && ended + ? `${Math.round((ended.getTime() - started.getTime()) / 1000)}s` + : "?" + console.log( + `${alias}: ${run.status} -> ${run.collectionName}\n` + + ` ${count(totals.documents, "document")} in ` + + `${count(totals.batches, "batch", "batches")} over ` + + `${count(run.chunks ?? 1, "chunk")}, ${elapsed}\n` + + ` cursor ${run.cursor ?? "(source exhausted)"}` + + (totals.convertFailures + ? `, ${count(totals.convertFailures, "convert failure")}` + : "") + + (run.numBatches ? `, limited to ${run.numBatches} batches` : "") + ) + if (data.lastError) console.log(` last error: ${data.lastError}`) + } + + if (!upgrades.length) console.log("no upgrades scheduled") +} diff --git a/scripts/search-eval/collections.ts b/scripts/search-eval/collections.ts new file mode 100644 index 000000000..f95003744 --- /dev/null +++ b/scripts/search-eval/collections.ts @@ -0,0 +1,111 @@ +/** Per-collection eval configuration. The search parameters come straight from + * the app (components/search/searchParams.ts) so ranking changes there flow + * into the eval without edits here. + */ +import type { SearchParameters } from "../../components/search/searchParams" +import { + billsRelevanceSort, + billsSearchParams, + hearingsRelevanceSort, + hearingsSearchParams, + testimonyRelevanceSort, + testimonySearchParams +} from "../../components/search/searchParams" + +export type EvalCollection = { + /** CollectionConfig.alias — also the corpus directory and golden set name. */ + alias: string + /** Local collection the corpus is seeded into, kept distinct from the + * dev-workflow alias so `yarn dev:up` data is never touched. */ + evalCollection: string + searchParams: SearchParameters + /** The app's relevance sort option. A golden set's defaults.sort_by must + * match it, or the eval is measuring an ordering no user can select. */ + relevanceSort: string + /** Pinned include_fields: the id, every field golden rules resolve against + * so judgments can be checked against a live index, and any field a human + * needs in the labeling sheet to judge relevance — its columns come from + * this list. bills' `pinslip` and `summary` are there for the second reason + * only; no rule names either, deliberately (see the README on why widening + * rules to the field under test rewards it by construction). */ + includeFields: string + categories: readonly string[] + /** How to rebuild this corpus, in order. Printed when the committed md5 does + * not match — which is exactly when a contributor is looking for the recipe, + * so it has to be the recipe the README documents and not an approximation of + * it. bills is two steps because its documents come from the emulator fixture + * while `summary` is joined in from a live project. + */ + regenerate: readonly string[] +} + +export const evalCollections: Record = { + bills: { + alias: "bills", + evalCollection: "bills_eval", + searchParams: billsSearchParams, + relevanceSort: billsRelevanceSort, + includeFields: + "id,court,number,title,pinslip,summary,primarySponsor,currentCommittee", + categories: [ + "exact-bill-id", + "topic", + "synonym", + "misspelling", + "member-committee", + "hyphenation", + "plain-language" + ], + regenerate: [ + "yarn search-eval:corpus", + "yarn search-eval enrich --env prod --alias bills --field summary" + ] + }, + hearings: { + alias: "hearings", + evalCollection: "hearings_eval", + searchParams: hearingsSearchParams, + relevanceSort: hearingsRelevanceSort, + includeFields: + "id,court,title,committeeName,chairNames,billNumbers,agendaTopics,locationName,locationCity", + categories: [ + "committee", + "chair", + "bill-on-agenda", + "agenda-topic", + "location", + "synonym", + "misspelling" + ], + regenerate: ["yarn search-eval corpus --env prod --alias hearings"] + }, + publishedTestimony: { + alias: "publishedTestimony", + evalCollection: "publishedTestimony_eval", + searchParams: testimonySearchParams, + relevanceSort: testimonyRelevanceSort, + includeFields: "id,court,billId,authorDisplayName,authorRole,position", + categories: [ + "bill-id", + "author", + "topic", + "synonym", + "misspelling", + "role" + ], + regenerate: [ + "yarn search-eval corpus --env prod --alias publishedTestimony --order-by publishedAt:desc" + ] + } +} + +export const aliases = Object.keys(evalCollections) + +export function resolveEvalCollection(alias: string): EvalCollection { + const collection = evalCollections[alias] + if (!collection) + throw Error( + `Unknown eval collection "${alias}". Known: ${aliases.join(", ")}` + ) + return collection +} diff --git a/scripts/search-eval/corpus.ts b/scripts/search-eval/corpus.ts new file mode 100644 index 000000000..064e42afd --- /dev/null +++ b/scripts/search-eval/corpus.ts @@ -0,0 +1,139 @@ +import { createHash } from "crypto" +import { existsSync, readFileSync } from "fs" +import { join } from "path" +import { gunzipSync } from "zlib" +import { createClient } from "../../functions/src/search/client" +import { resolveEvalCollection } from "./collections" + +export const corpusRoot = join(__dirname, "../../tests/search-eval/corpus") + +export const corpusDir = (alias: string) => join(corpusRoot, alias) + +/** A corpus document: whatever the collection's converter emits. Only `id` and + * `court` are read by the harness itself; golden rules resolve the rest by name. + */ +export type CorpusDoc = { id: string; court: number } & Record + +export type CorpusMeta = { + alias: string + source: string + count: number + courts: Record + jsonlMd5: string + /** Present when the export capped the document count. */ + limit?: number + /** The Firestore ordering the export paged in, e.g. "publishedAt:desc". */ + orderBy?: string + /** Fields overwritten at export time; never fields the eval searches. */ + redacted?: string[] + /** Fields copied in from a live project after export, for fields the corpus's + * own source predates. One entry per join. See the `enrich` subcommand. + */ + enriched?: { field: string; source: string; count: number }[] +} + +const regenerate = (alias: string) => + `${resolveEvalCollection(alias).regenerate.join( + " && " + )} (see tests/search-eval/README.md)` + +export function readMeta(alias: string): CorpusMeta { + return JSON.parse(readFileSync(join(corpusDir(alias), "meta.json"), "utf8")) +} + +/** The court the eval filters on, mirroring the app's court refinement: + * the court with the most documents in the corpus. + */ +export function majorityCourt(meta: CorpusMeta): number { + const courts = Object.entries(meta.courts).sort((a, b) => b[1] - a[1]) + if (!courts.length) throw Error("corpus meta lists no courts") + return Number(courts[0][0]) +} + +export function readCorpus(alias: string): { + docs: CorpusDoc[] + jsonl: string +} { + const path = join(corpusDir(alias), "docs.jsonl.gz") + if (!existsSync(path)) { + throw Error(`${path} not found. Generate it with: ${regenerate(alias)}`) + } + const jsonl = gunzipSync(readFileSync(path)).toString("utf8") + + const meta = readMeta(alias) + const md5 = createHash("md5").update(jsonl).digest("hex") + if (md5 !== meta.jsonlMd5) { + throw Error( + `Corpus hash mismatch: ${alias}/docs.jsonl.gz has md5 ${md5} but meta.json expects ${ + meta.jsonlMd5 + }. Regenerate with: ${regenerate(alias)}` + ) + } + + const docs = jsonl + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as CorpusDoc) + if (docs.length !== meta.count) + throw Error( + `Corpus has ${docs.length} docs, meta.json expects ${meta.count}` + ) + return { docs, jsonl } +} + +/** Drops and recreates `collection` from the committed schema, then imports + * the corpus. Uses a dedicated collection (default _eval) so the + * dev-workflow aliases are never touched. + */ +export async function seedCollection( + client: ReturnType, + alias: string, + collection: string +): Promise { + const { docs, jsonl } = readCorpus(alias) + const schema = JSON.parse( + readFileSync(join(corpusDir(alias), "schema.json"), "utf8") + ) + + const existing = await client.collections().retrieve() + if (existing.some((c: { name: string }) => c.name === collection)) { + await client.collections(collection).delete() + } + await client.collections().create({ name: collection, ...schema }) + + // Sliced by serialized bytes, not a fixed count: bills carry full `body` + // text, and a fixed-count slice can overflow the 10 MB API-gateway payload + // cap when seeding a remote collection. Keep in sync with + // IMPORT_BYTE_BUDGET in functions/src/search/SearchIndexer.ts — not + // imported, because that module would pull firebase-admin into every seed + // run. Raw JSONL string form works identically on typesense client 1.x and + // 3.x. + const importByteBudget = 6 * 1024 * 1024 + const lines = jsonl.split("\n").filter(Boolean) + let slice: string[] = [] + let bytes = 0 + const flush = async () => { + if (!slice.length) return + await client + .collections(collection) + .documents() + .import(slice.join("\n"), { action: "create" }) + slice = [] + bytes = 0 + } + for (const line of lines) { + const size = Buffer.byteLength(line) + 1 + if (slice.length && bytes + size > importByteBudget) await flush() + slice.push(line) + bytes += size + } + await flush() + + const created = await client.collections(collection).retrieve() + if (created.num_documents !== docs.length) { + throw Error( + `Seeded ${created.num_documents} documents, expected ${docs.length}` + ) + } + console.log(`Seeded ${created.num_documents} docs into "${collection}"`) +} diff --git a/scripts/search-eval/corpusFiles.ts b/scripts/search-eval/corpusFiles.ts new file mode 100644 index 000000000..a0bee7cd0 --- /dev/null +++ b/scripts/search-eval/corpusFiles.ts @@ -0,0 +1,102 @@ +/** Writing the corpus artifacts. Shared by the two ways a corpus is produced: + * the emulator export (scripts/firebase-admin/exportSearchCorpus.ts, admin SDK, + * rules bypassed) and the credential-free remote export (./remoteCorpus.ts, + * client SDK, subject to security rules). + */ +import { createHash } from "crypto" +import { mkdirSync, writeFileSync } from "fs" +import { join } from "path" +import { gzipSync } from "zlib" +import type { Schema } from "../../functions/src/search/config" +import { corpusDir, CorpusDoc } from "./corpus" + +/** One corpus document as an exporter writes it — the same shape the harness + * reads back, so writer and reader cannot drift. */ +export type ExportedDoc = CorpusDoc + +/** Fields rewritten before a document is written to the corpus. None of them + * are in the collection's query_by (components/search/searchParams.ts), so + * redacting them changes nothing the eval measures — it keeps identifiers the + * app never exposes out of a committed artifact. + */ +export const redactions: Record< + string, + { fields: string[]; apply: (doc: ExportedDoc) => ExportedDoc } +> = { + publishedTestimony: { + fields: ["authorUid", "fullName"], + apply: doc => ({ + ...doc, + authorUid: createHash("sha1") + .update(String(doc.authorUid)) + .digest("hex") + .slice(0, 16), + fullName: String(doc.authorDisplayName ?? "") + }) + } +} + +export function writeCorpus({ + alias, + source, + docs, + schema, + limit, + orderBy, + enriched +}: { + alias: string + /** Where the documents came from: an env name, or the fixture path. */ + source: string + docs: ExportedDoc[] + schema: Schema + limit?: number + orderBy?: string + /** Set by the `enrich` subcommand: one entry per field joined in from a live + * project after export. */ + enriched?: { field: string; source: string; count: number }[] +}): number { + const redact = redactions[alias] + + const byId = new Map() + for (const doc of docs) byId.set(doc.id, redact ? redact.apply(doc) : doc) + + // Sorted by id so the corpus — and its md5 — is stable across exports that + // paged in a different order. + const ids = Array.from(byId.keys()).sort() + const jsonl = ids.map(id => JSON.stringify(byId.get(id))).join("\n") + "\n" + + const courts: Record = {} + for (const id of ids) { + const court = String(byId.get(id)!.court) + courts[court] = (courts[court] ?? 0) + 1 + } + + const outDir = corpusDir(alias) + mkdirSync(outDir, { recursive: true }) + writeFileSync(join(outDir, "docs.jsonl.gz"), gzipSync(jsonl)) + writeFileSync( + join(outDir, "schema.json"), + JSON.stringify(schema, null, 2) + "\n" + ) + writeFileSync( + join(outDir, "meta.json"), + JSON.stringify( + { + alias, + source, + count: ids.length, + courts, + jsonlMd5: createHash("md5").update(jsonl).digest("hex"), + ...(limit === undefined ? {} : { limit }), + ...(orderBy === undefined ? {} : { orderBy }), + ...(redact === undefined ? {} : { redacted: redact.fields }), + ...(enriched === undefined ? {} : { enriched }) + }, + null, + 2 + ) + "\n" + ) + + return ids.length +} diff --git a/scripts/search-eval/goldens.ts b/scripts/search-eval/goldens.ts new file mode 100644 index 000000000..466ab4c52 --- /dev/null +++ b/scripts/search-eval/goldens.ts @@ -0,0 +1,178 @@ +import { readFileSync } from "fs" +import { join } from "path" +import { + Array as A, + Boolean as B, + Number as N, + Optional, + Record as R, + Static, + String as S +} from "runtypes" +import { resolveEvalCollection } from "./collections" +import { CorpusDoc } from "./corpus" +import { Judgments } from "./metrics" + +export const goldensPath = (alias: string) => + join(__dirname, `../../tests/search-eval/goldens/${alias}.json`) + +const Rule = R({ + field: Optional(S), + anyField: Optional(A(S)), + equals: Optional(S), + contains: Optional(S) +}) + +const RelevantEntry = R({ + docId: Optional(S), + rule: Optional(Rule), + grade: N +}) + +const GoldenQuery = R({ + id: S, + /** Validated against the collection's allowlist in loadGoldens — the + * meaningful categories differ per collection. */ + category: S, + query: S, + relevant: Optional(A(RelevantEntry)), + sameAs: Optional(S), + courtFilter: Optional(B) +}) + +const GoldenSet = R({ + collection: S, + defaults: R({ sort_by: S, courtFilter: B }), + queries: A(GoldenQuery) +}) + +export type GoldenQuery = Static +export type GoldenSet = Static +export type Rule = Static + +export function loadGoldens(alias: string): GoldenSet { + const collection = resolveEvalCollection(alias) + const goldens = GoldenSet.check( + JSON.parse(readFileSync(goldensPath(alias), "utf8")) + ) + + // A golden set scored under a sort the app cannot select measures an ordering + // no user ever sees. Hearings is the cautionary case: every sort option there + // was date-ordered, so nDCG measured recency. + if (goldens.defaults.sort_by !== collection.relevanceSort) + throw Error( + `${alias} goldens sort "${goldens.defaults.sort_by}" does not match the app's relevance sort "${collection.relevanceSort}" (components/search/searchParams.ts)` + ) + + const ids = new Set(goldens.queries.map(q => q.id)) + if (ids.size !== goldens.queries.length) + throw Error("Duplicate query ids in golden set") + for (const q of goldens.queries) { + if (!collection.categories.includes(q.category)) + throw Error( + `Query ${q.id} has unknown category "${ + q.category + }" for ${alias}. Known: ${collection.categories.join(", ")}` + ) + if (!q.relevant && !q.sameAs) + throw Error(`Query ${q.id} has neither relevant labels nor sameAs`) + if (q.sameAs && !ids.has(q.sameAs)) + throw Error(`Query ${q.id} sameAs references unknown id ${q.sameAs}`) + } + return goldens +} + +/** Normalization applied to rule values and doc fields (NOT to the query text + * sent to Typesense — tokenizer behavior on raw input is part of what's + * measured): uppercase, drop everything but letters and digits. + */ +export const normalize = (value: string) => + value.toUpperCase().replace(/[^A-Z0-9]/g, "") + +function ruleFields(rule: Rule): string[] { + const fields = rule.anyField ?? (rule.field ? [rule.field] : []) + if (!fields.length) throw Error("Rule specifies no field") + return fields +} + +function ruleMatches(rule: Rule, doc: CorpusDoc): boolean { + const values = ruleFields(rule) + .flatMap(field => { + const value = (doc as Record)[field] + return Array.isArray(value) ? value : [value] + }) + .filter((v): v is string => typeof v === "string") + .map(normalize) + + if (rule.equals !== undefined) { + const expected = normalize(rule.equals) + return values.some(v => v === expected) + } + if (rule.contains !== undefined) { + const expected = normalize(rule.contains) + return values.some(v => v.includes(expected)) + } + throw Error("Rule specifies neither equals nor contains") +} + +export const labels = (query: GoldenQuery, goldens: GoldenSet) => { + const source = query.sameAs + ? goldens.queries.find(q => q.id === query.sameAs)! + : query + if (!source.relevant) throw Error(`Query ${source.id} has no relevant labels`) + return source.relevant +} + +/** Resolves a query's judgments against the corpus. Docs outside `court` are + * excluded when the query uses the court filter, matching what the search + * itself can return. + */ +export function resolveJudgments( + query: GoldenQuery, + goldens: GoldenSet, + docs: CorpusDoc[], + court: number | undefined +): Judgments { + const judgments: Judgments = new Map() + const inScope = + court === undefined ? docs : docs.filter(d => d.court === court) + + for (const entry of labels(query, goldens)) { + if (entry.docId) { + judgments.set(entry.docId, entry.grade) + } else if (entry.rule) { + for (const doc of inScope.filter(d => ruleMatches(entry.rule!, d))) { + judgments.set(doc.id, Math.max(judgments.get(doc.id) ?? 0, entry.grade)) + } + } else { + throw Error(`Query ${query.id} has a label with neither docId nor rule`) + } + } + return judgments +} + +/** Rule resolution for live (dev/prod) targets where the local corpus is not + * authoritative: search the rule's fields for the rule value and keep the + * results the rule actually matches. + */ +export async function resolveRuleViaSearch( + rule: Rule, + grade: number, + search: (params: { + q: string + query_by: string + per_page: number + }) => Promise +): Promise { + const value = rule.equals ?? rule.contains + if (value === undefined) + throw Error("Rule specifies neither equals nor contains") + const found = await search({ + q: value, + query_by: ruleFields(rule).join(","), + per_page: 250 + }) + return new Map( + found.filter(doc => ruleMatches(rule, doc)).map(doc => [doc.id, grade]) + ) +} diff --git a/scripts/search-eval/index.ts b/scripts/search-eval/index.ts new file mode 100644 index 000000000..74415219f --- /dev/null +++ b/scripts/search-eval/index.ts @@ -0,0 +1,581 @@ +import { execSync } from "child_process" +import { writeFileSync } from "fs" +import { join } from "path" +import type { SearchParams } from "typesense" +import yargs, { Arguments } from "yargs" +import { hideBin } from "yargs/helpers" +import { supportedGeneralCourts } from "../../functions/src/shared/constants" +import { TypesenseConnectionArgs, resolveClient } from "../typesense-env" +import { EvalCollection, aliases, resolveEvalCollection } from "./collections" +import type { RemoteEnv } from "./remoteCorpus" +import { + CorpusDoc, + corpusDir, + majorityCourt, + readCorpus, + readMeta, + seedCollection +} from "./corpus" +import { + GoldenQuery, + GoldenSet, + labels, + loadGoldens, + resolveJudgments, + resolveRuleViaSearch +} from "./goldens" +import { Judgments, computeMetrics } from "./metrics" +import { + QueryResult, + buildResults, + compare, + printScorecard, + writeResults +} from "./report" + +type Args = Arguments< + TypesenseConnectionArgs & { + alias?: string + collection?: string + out?: string + courtFilter?: boolean + only?: string + threshold?: number + limit?: number + orderBy?: string + court?: number[] + field?: string + } +> + +const resolveTarget = (args: Args) => { + const evalCollection = resolveEvalCollection(args.alias ?? "bills") + const collection = + args.collection ?? + (args.env === "local" + ? evalCollection.evalCollection + : evalCollection.alias) + return { evalCollection, collection } +} + +/** The typesense client's own search params, with the fields every eval + * search must pin made required. New ranking knobs added to + * components/search/searchParams.ts flow through without edits here. + */ +type EvalSearchParams = SearchParams & { + q: string + query_by: string + per_page: number +} + +function createSearcher( + args: Args, + evalCollection: EvalCollection, + collection: string +) { + const client = resolveClient(args) + return { + client, + search: async (params: EvalSearchParams): Promise => { + const result = await client + .collections(collection) + .documents() + .search({ include_fields: evalCollection.includeFields, ...params }) + return (result.hits ?? []).map(h => h.document) + } + } +} + +/** The court filter applied to a query, mirroring the app's court refinement. */ +function courtFor( + query: GoldenQuery, + goldens: GoldenSet, + args: Args, + majority: number +): number | undefined { + const enabled = + args.courtFilter !== false && + (query.courtFilter ?? goldens.defaults.courtFilter) + return enabled ? majority : undefined +} + +/** The ranking params under test: everything the app sends except + * exclude_fields, which conflicts with the eval's pinned include_fields. The + * cast re-homes the adapter's parameter type, which is generic over its own + * DocumentSchema, onto the CorpusDoc the eval pins. + */ +const rankingParams = ({ searchParams }: EvalCollection) => { + const { exclude_fields: _, ...rest } = searchParams + return rest as Omit, "q" | "per_page"> & { + query_by: string + } +} + +const searchTop10 = ( + search: ReturnType["search"], + evalCollection: EvalCollection, + query: GoldenQuery, + goldens: GoldenSet, + court: number | undefined +) => + search({ + ...rankingParams(evalCollection), + q: query.query, + sort_by: goldens.defaults.sort_by, + filter_by: court === undefined ? undefined : `court:=${court}`, + per_page: 10 + }) + +async function judgmentsFor( + query: GoldenQuery, + goldens: GoldenSet, + args: Args, + court: number | undefined, + search: ReturnType["search"], + localDocs: CorpusDoc[] | undefined +): Promise { + if (localDocs) return resolveJudgments(query, goldens, localDocs, court) + + // Live (dev/prod) target: the local corpus is not authoritative there, so + // explicit labels pass through and rules resolve by searching the live index. + const judgments: Judgments = new Map() + for (const entry of labels(query, goldens)) { + if (entry.docId) { + judgments.set(entry.docId, entry.grade) + } else if (entry.rule) { + const resolved = await resolveRuleViaSearch(entry.rule, entry.grade, p => + search({ + ...p, + filter_by: court === undefined ? undefined : `court:=${court}` + }) + ) + resolved.forEach((grade, id) => + judgments.set(id, Math.max(judgments.get(id) ?? 0, grade)) + ) + } + } + return judgments.size ? judgments : undefined +} + +async function runEval(args: Args) { + const { evalCollection, collection } = resolveTarget(args) + const goldens = loadGoldens(evalCollection.alias) + const { client, search } = createSearcher(args, evalCollection, collection) + const localDocs = + args.env === "local" ? readCorpus(evalCollection.alias).docs : undefined + const meta = readMeta(evalCollection.alias) + const majority = majorityCourt(meta) + + const results: QueryResult[] = [] + const skipped: string[] = [] + for (const query of goldens.queries) { + const court = courtFor(query, goldens, args, majority) + const judgments = await judgmentsFor( + query, + goldens, + args, + court, + search, + localDocs + ) + if (!judgments || judgments.size === 0) { + skipped.push(query.id) + continue + } + + const top10 = await searchTop10( + search, + evalCollection, + query, + goldens, + court + ) + const topIds = top10.map(d => d.id) + results.push({ + id: query.id, + category: query.category, + query: query.query, + metrics: computeMetrics(topIds, judgments), + top10: topIds, + relevantCount: judgments.size + }) + } + + const serverVersion = await client.debug + .retrieve() + .then((d: { version?: string }) => d.version ?? "unknown") + .catch(() => "unknown") + const gitSha = execSync("git rev-parse HEAD").toString().trim() + + const evalResults = buildResults( + { + env: args.env ?? "custom", + url: args.url ?? "", + alias: evalCollection.alias, + collection, + serverVersion, + corpusMd5: meta.jsonlMd5, + gitSha, + timestamp: new Date().toISOString(), + sortBy: goldens.defaults.sort_by, + court: + args.courtFilter === false || !goldens.defaults.courtFilter + ? undefined + : majority, + skippedQueries: skipped + }, + results + ) + printScorecard(evalResults) + if (args.out) writeResults(evalResults, args.out) +} + +/** Renders one field of a hit for the labeling sheet: arrays joined, pipes + * swapped out so they cannot break the markdown table. + */ +const cell = (value: unknown) => + (Array.isArray(value) ? value.join("; ") : (value ?? "").toString()).replace( + /\|/g, + "/" + ) + +async function writeLabelingSheet(args: Args) { + const { evalCollection, collection } = resolveTarget(args) + const goldens = loadGoldens(evalCollection.alias) + const { search } = createSearcher(args, evalCollection, collection) + const queries = goldens.queries.filter( + q => !args.only || q.category === args.only + ) + + // The pinned include_fields minus the id, which gets its own column. + const columns = evalCollection.includeFields + .split(",") + .filter(field => field !== "id") + + const sections: string[] = [ + `# Search eval labeling sheet — ${evalCollection.alias}`, + "", + `Top 10 results per golden query (collection "${collection}", sort "${goldens.defaults.sort_by}").`, + "Grade each result: 3 = exactly what the searcher wanted, 1 = relevant, 0/blank = not relevant.", + `Merge grades into tests/search-eval/goldens/${evalCollection.alias}.json as explicit docId entries.`, + "" + ] + const majority = majorityCourt(readMeta(evalCollection.alias)) + for (const query of queries) { + const court = courtFor(query, goldens, args, majority) + const top10 = await searchTop10( + search, + evalCollection, + query, + goldens, + court + ) + sections.push( + `## ${query.id} (${query.category}): "${query.query}"`, + "", + `| grade | rank | docId | ${columns.join(" | ")} |`, + `| --- | --- | --- | ${columns.map(() => "---").join(" | ")} |`, + ...top10.map( + (d, i) => + `| | ${i + 1} | ${d.id} | ${columns + .map(field => cell(d[field])) + .join(" | ")} |` + ), + "" + ) + } + + const path = join( + __dirname, + `../../tests/search-eval/labeling-sheet-${evalCollection.alias}.md` + ) + writeFileSync(path, sections.join("\n")) + console.log(`Wrote labeling sheet for ${queries.length} queries to ${path}`) +} + +/** Everything the two subcommands that read a live project need. Imports are + * deferred because pulling in the search configs loads firebase-admin, which + * every other subcommand does without — and `require` rather than `import()`, + * because under ts-node --swc a dynamic import survives as a native ESM import + * and fails to resolve the extensionless path. + */ +function remoteSetup(args: Args, command: string) { + // Args.env is a bare string (see TypesenseConnectionArgs), so the guard is + // what pins it to a project the remote export can actually read. + const env: RemoteEnv | undefined = + args.env === "dev" || args.env === "prod" ? args.env : undefined + if (!env) throw Error(`${command} needs --env dev or --env prod`) + + const { evalCollection } = resolveTarget(args) + const remoteCorpus = + require("./remoteCorpus") as typeof import("./remoteCorpus") + const corpusFiles = require("./corpusFiles") as typeof import("./corpusFiles") + const { configForAlias } = + require("../../functions/src/search/config") as typeof import("../../functions/src/search/config") + // Loading a collection's search module is what registers its config. + require("../../functions/src/bills/search") + require("../../functions/src/hearings/search") + require("../../functions/src/testimony/search") + + return { + env, + alias: evalCollection.alias, + config: configForAlias(evalCollection.alias), + fetchRemoteDocs: remoteCorpus.fetchRemoteDocs, + projects: remoteCorpus.projects, + writeCorpus: corpusFiles.writeCorpus, + redactions: corpusFiles.redactions + } +} + +/** Only bills' trigger nests under `{court}`; for every other collection the + * list is unused, so passing a default costs nothing. + */ +const courtsFor = (args: Args, fallback: readonly number[]) => + args.court?.length ? args.court : fallback + +/** Builds a corpus from a live project. */ +async function buildRemoteCorpus(args: Args) { + const { env, config, fetchRemoteDocs, projects, writeCorpus } = remoteSetup( + args, + "corpus" + ) + + const [orderByField, direction = "asc"] = (args.orderBy ?? "__name__").split( + ":" + ) + if (direction !== "asc" && direction !== "desc") + throw Error(`--order-by direction must be asc or desc, got "${direction}"`) + + const courts = courtsFor(args, supportedGeneralCourts) + console.log( + `Reading ${config.alias} from ${projects[env]} (no credentials — see firestore.rules)` + ) + const { docs, failures } = await fetchRemoteDocs({ + env, + config, + courts, + limit: args.limit, + orderByField, + direction, + onProgress: (fetched, kept) => + console.log(` fetched ${fetched}, kept ${kept}`) + }) + + const count = writeCorpus({ + alias: config.alias, + source: env, + docs, + schema: config.schema, + limit: args.limit, + orderBy: args.orderBy + }) + console.log( + `Wrote ${count} docs to ${corpusDir( + config.alias + )} (${failures} conversion failures)` + ) +} + +/** Copies one field from a live project onto an existing corpus, for a field + * the corpus's own source predates. + * + * The bills corpus is the committed emulator fixture, captured before the LLM + * `summary` trigger in llm/ existed — all 7,337 documents have no summary, and + * prod has one for 97.9% of them. Re-exporting bills from prod instead would + * bring four years of unrelated drift with it: court 192 closed in 2022, so + * prod's copy has drained out of its policy committees (36 distinct + * `currentCommittee` values in the fixture against 21 in prod, and none left in + * Revenue at all), which silently rewrites the member-committee goldens. + * Joining the one missing field keeps every other value frozen, so a control + * run reproduces the previous baseline exactly and the field under test is the + * only thing that moved. + */ +async function enrichCorpus(args: Args) { + const field = args.field + if (!field) throw Error("enrich needs --field, e.g. --field summary") + + const { + env, + alias, + config, + fetchRemoteDocs, + projects, + writeCorpus, + redactions + } = remoteSetup(args, "enrich") + + // writeCorpus redacts on the way out, and redaction is not idempotent — + // publishedTestimony's authorUid would be hashed a second time. + if (redactions[alias]) + throw Error( + `Cannot enrich "${alias}": its export redacts fields, and rewriting the corpus here would redact them twice.` + ) + + const meta = readMeta(alias) + const { docs: corpusDocs } = readCorpus(alias) + // The corpus records which courts it holds, and a court it does not hold can + // only produce documents the id lookup below throws away — so read those and + // no more. Defaulting to every supported court downloads the current one in + // full for nothing. + const courts = courtsFor(args, Object.keys(meta.courts).map(Number)) + + console.log( + `Reading ${config.alias} from ${projects[env]} for "${field}" (no credentials — see firestore.rules)` + ) + const { docs: liveDocs, failures } = await fetchRemoteDocs({ + env, + config, + courts, + orderByField: "__name__", + direction: "asc", + onProgress: (fetched, kept) => + console.log(` fetched ${fetched}, kept ${kept}`) + }) + + const values = new Map( + liveDocs + .filter( + d => d[field] !== undefined && d[field] !== null && d[field] !== "" + ) + .map(d => [d.id, d[field]]) + ) + + let enriched = 0 + const merged = corpusDocs.map(doc => { + const value = values.get(doc.id) + if (value === undefined) return doc + enriched++ + return { ...doc, [field]: value } + }) + + const count = writeCorpus({ + alias, + source: meta.source, + docs: merged, + schema: config.schema, + limit: meta.limit, + orderBy: meta.orderBy, + // Replace this field's entry, keep every other join's. A corpus that has + // been enriched twice has to say so, or it stops describing how it was + // built and the md5 can only tell you the bytes moved. + enriched: [ + ...(meta.enriched ?? []).filter(e => e.field !== field), + { field, source: env, count: enriched } + ] + }) + console.log( + `Set "${field}" on ${enriched} of ${count} docs from ${projects[env]} (${failures} conversion failures)` + ) +} + +/** Shared by `corpus` and `enrich`; the default differs per command, so the + * describe text names the flag's purpose rather than a specific fallback. */ +const courtOption = { + array: true, + number: true, + describe: + "general court(s) to read, for collections nested under one (bills). Defaults to every supported court for `corpus`, and to the courts the corpus already holds for `enrich`" +} as const + +yargs(hideBin(process.argv)) + .scriptName("search-eval") + .command( + "corpus", + "build a corpus from a live project (dev/prod, no credentials needed)", + yargs => + yargs.options({ + limit: { number: true, describe: "cap the number of documents" }, + "order-by": { + string: true, + describe: + "field[:asc|desc] to page in, e.g. publishedAt:desc (default: __name__, which needs no index)" + }, + court: courtOption + }), + (args: Args) => buildRemoteCorpus(args) + ) + .command( + "enrich", + "copy one field from a live project onto the existing corpus", + yargs => + yargs.options({ + field: { + string: true, + demandOption: true, + describe: "the field to copy, e.g. summary" + }, + court: courtOption + }), + (args: Args) => enrichCorpus(args) + ) + .command( + "seed", + "create and import the eval collection from the corpus snapshot", + {}, + async (args: Args) => { + const { evalCollection, collection } = resolveTarget(args) + await seedCollection( + resolveClient(args), + evalCollection.alias, + collection + ) + } + ) + .command( + "run", + "run the golden queries and print the relevance scorecard", + yargs => + yargs.options({ + out: { string: true, describe: "write results JSON to this path" }, + "court-filter": { + boolean: true, + default: true, + describe: "filter to the corpus majority court, like the app does" + } + }), + (args: Args) => runEval(args) + ) + .command( + "label", + "write a labeling sheet with each query's top 10 results", + yargs => + yargs.options({ + only: { string: true, describe: "restrict to one category" } + }), + (args: Args) => writeLabelingSheet(args) + ) + .command( + "compare ", + "print metric deltas between two results files", + yargs => + yargs.options({ + threshold: { + number: true, + default: 0.05, + describe: "flag per-query and overall nDCG@10 drops above this" + } + }), + (args: Args & { baseline?: string; candidate?: string }) => { + const ok = compare(args.baseline!, args.candidate!, args.threshold!) + if (!ok) process.exitCode = 1 + } + ) + .options({ + url: { string: true, alias: "u" }, + key: { string: true, alias: "k" }, + env: { choices: ["local", "dev", "prod"], alias: "e" }, + alias: { + string: true, + choices: aliases, + default: "bills", + describe: "which search collection to evaluate" + }, + collection: { + string: true, + describe: + "target collection (default: _eval locally, on dev/prod)" + } + }) + .demandCommand(1) + .strictCommands().argv diff --git a/scripts/search-eval/metrics.ts b/scripts/search-eval/metrics.ts new file mode 100644 index 000000000..e9a6f725f --- /dev/null +++ b/scripts/search-eval/metrics.ts @@ -0,0 +1,69 @@ +/** Relevance judgments for one query: docId -> grade (grade > 0 = relevant). */ +export type Judgments = Map + +export type QueryMetrics = { + recallAt10: number + mrr: number + ndcgAt10: number +} + +const K = 10 + +/** Capped recall@10: relevant results in the top 10 over min(|relevant|, 10), + * so a query with more than 10 relevant docs can still score 1.0. + */ +export function recallAtK(resultIds: string[], judgments: Judgments): number { + const denominator = Math.min(judgments.size, K) + if (denominator === 0) return 0 + const hits = resultIds + .slice(0, K) + .filter(id => (judgments.get(id) ?? 0) > 0).length + return hits / denominator +} + +/** Reciprocal rank of the first relevant result, 0 if none appear. */ +export function mrr(resultIds: string[], judgments: Judgments): number { + const rank = resultIds.findIndex(id => (judgments.get(id) ?? 0) > 0) + return rank === -1 ? 0 : 1 / (rank + 1) +} + +/** nDCG@10 with exponential gain (2^grade - 1). */ +export function ndcgAtK(resultIds: string[], judgments: Judgments): number { + const gain = (grade: number) => Math.pow(2, grade) - 1 + const discount = (rank: number) => 1 / Math.log2(rank + 2) + + const dcg = resultIds + .slice(0, K) + .reduce((sum, id, i) => sum + gain(judgments.get(id) ?? 0) * discount(i), 0) + + const idealDcg = Array.from(judgments.values()) + .sort((a, b) => b - a) + .slice(0, K) + .reduce((sum, grade, i) => sum + gain(grade) * discount(i), 0) + + return idealDcg === 0 ? 0 : dcg / idealDcg +} + +export function computeMetrics( + resultIds: string[], + judgments: Judgments +): QueryMetrics { + return { + recallAt10: recallAtK(resultIds, judgments), + mrr: mrr(resultIds, judgments), + ndcgAt10: ndcgAtK(resultIds, judgments) + } +} + +export function averageMetrics(all: QueryMetrics[]): QueryMetrics & { + queryCount: number +} { + const avg = (select: (m: QueryMetrics) => number) => + all.length ? all.reduce((sum, m) => sum + select(m), 0) / all.length : 0 + return { + recallAt10: avg(m => m.recallAt10), + mrr: avg(m => m.mrr), + ndcgAt10: avg(m => m.ndcgAt10), + queryCount: all.length + } +} diff --git a/scripts/search-eval/remoteCorpus.ts b/scripts/search-eval/remoteCorpus.ts new file mode 100644 index 000000000..63b89fee9 --- /dev/null +++ b/scripts/search-eval/remoteCorpus.ts @@ -0,0 +1,181 @@ +/** Builds a corpus from a live MAPLE project without any credentials. + * + * firestore.rules grants unauthenticated read on every collection a corpus is + * built from — `events` and `generalCourts/**` are "public, read-only", and + * publishedTestimony has an explicit `match /{path=**}/publishedTestimony/{id}` + * rule so it can be read as a collection group. So the web client SDK reads it + * with nothing but a project id, which is what makes every corpus reachable to + * contributors who have no service account. + * + * Bills needs one extra step. Its rule is path-scoped to `generalCourts/**` and + * does not grant collection-group scope, so `collectionGroup("bills")` is + * denied — but `generalCourts/192/bills` is a plain collection read inside that + * scope and is allowed. Substituting each general court for the trigger's + * `{court}` wildcard turns the one denied read into three permitted ones. + */ +import { initializeApp } from "firebase/app" +import { + DocumentData, + QueryDocumentSnapshot, + Timestamp as ClientTimestamp, + collection, + collectionGroup, + documentId, + getDocs, + getFirestore, + limit as limitTo, + orderBy, + query, + startAfter +} from "firebase/firestore" +// The very same Timestamp the converters' runtypes check against. Importing it +// from "firebase-admin/firestore" here instead resolves to the ROOT copy of the +// package while functions/src/firebase resolves to functions/node_modules — two +// distinct classes, and every InstanceOf check fails with the memorable +// "Expected Timestamp, but was Timestamp". +import { Timestamp as AdminTimestamp } from "../../functions/src/firebase" +import type { CollectionConfig } from "../../functions/src/search/config" +import { ExportedDoc } from "./corpusFiles" + +export const projects = { + dev: "digital-testimony-dev", + prod: "digital-testimony-prod" +} as const + +export type RemoteEnv = keyof typeof projects + +const batchSize = 500 + +/** The converters validate timestamps with `InstanceOf(Timestamp)` against the + * firebase-admin class (functions/src/testimony/types.ts:41, + * functions/src/events/types.ts:33). The client SDK's Timestamp is a different + * class, so every document fails validation unless it is rebuilt as the admin + * one. Seconds and nanos are carried across rather than millis so nothing is + * rounded away. + */ +const toAdminTypes = (value: unknown): unknown => { + if (value instanceof ClientTimestamp) + return new AdminTimestamp(value.seconds, value.nanoseconds) + if (Array.isArray(value)) return value.map(toAdminTypes) + if (value !== null && typeof value === "object") + return Object.fromEntries( + Object.entries(value as Record).map(([k, v]) => [ + k, + toAdminTypes(v) + ]) + ) + return value +} + +/** One readable Firestore source. For a collection group the collection id is + * the path, so one field covers both. + */ +type RemoteSource = { path: string; isCollectionGroup?: boolean } + +/** The sources to read, derived from the config's document trigger so they + * cannot drift from the indexer's own source. + * + * - `events/{eventId}` is a top-level collection. + * - `users/{uid}/publishedTestimony/{id}` is nested under a wildcard, so it is + * read as a collection group — permitted by its own `{path=**}` rule. + * - `generalCourts/{court}/bills/{id}` is nested too, but no rule grants it + * group scope. `courts` is substituted for `{court}` to give one concrete, + * readable path per general court. + * + * Any `where` on config.sourceCollection is deliberately not reproduced — + * config.filter is the authority on which documents belong in the index, and + * it is applied to every document below. + */ +export function sourcesFromTrigger( + documentTrigger: string, + courts: readonly number[] +): RemoteSource[] { + const segments = documentTrigger.split("/").filter(Boolean) + if (segments.length < 2 || segments.length % 2 !== 0) + throw Error(`Cannot read a collection out of trigger "${documentTrigger}"`) + + // Drop the trailing document-id wildcard; what remains is the collection. + const path = segments.slice(0, -1) + const collectionId = path[path.length - 1] + + if (path.length === 1) return [{ path: collectionId }] + if (path.length === 3 && path[0] === "generalCourts") + return courts.map(court => ({ + path: `generalCourts/${court}/${collectionId}` + })) + return [{ path: collectionId, isCollectionGroup: true }] +} + +export async function fetchRemoteDocs({ + env, + config, + courts, + limit, + orderByField, + direction, + onProgress +}: { + env: RemoteEnv + config: CollectionConfig + courts: readonly number[] + limit?: number + orderByField: string + direction: "asc" | "desc" + onProgress?: (fetched: number, kept: number) => void +}): Promise<{ docs: ExportedDoc[]; failures: number }> { + const projectId = projects[env] + const sources = sourcesFromTrigger(config.documentTrigger, courts) + + const app = initializeApp({ projectId }, `search-eval-${env}`) + const db = getFirestore(app) + + const docs: ExportedDoc[] = [] + let failures = 0 + let fetched = 0 + + // `limit` caps the corpus as a whole, not each source, so a capped export of + // a multi-court collection is the first N documents overall. + for (const source of sources) { + const ref = source.isCollectionGroup + ? collectionGroup(db, source.path) + : collection(db, source.path) + let cursor: QueryDocumentSnapshot | undefined + + while (limit === undefined || docs.length < limit) { + const page = query( + ref, + // Paging needs a total order, not a meaningful one: writeCorpus sorts + // by id before writing, so this never reaches the corpus. "__name__" + // is the default because document name is the one ordering Firestore + // can always serve — prod carries a single-field index exemption on + // bills' `id`, and ordering by it fails with failed-precondition. + orderBy( + orderByField === "__name__" ? documentId() : orderByField, + direction + ), + ...(cursor ? [startAfter(cursor)] : []), + limitTo(batchSize) + ) + const snap = await getDocs(page) + if (snap.empty) break + cursor = snap.docs[snap.docs.length - 1] + fetched += snap.size + + for (const d of snap.docs) { + const data = toAdminTypes(d.data()) as DocumentData + try { + if (config.filter && !config.filter(data)) continue + docs.push(config.convert(data) as ExportedDoc) + if (limit !== undefined && docs.length >= limit) break + } catch (error: any) { + failures++ + console.error(`Failed to convert ${d.ref.path}: ${error.message}`) + } + } + onProgress?.(fetched, docs.length) + if (snap.size < batchSize) break + } + } + + return { docs, failures } +} diff --git a/scripts/search-eval/report.ts b/scripts/search-eval/report.ts new file mode 100644 index 000000000..774adf47b --- /dev/null +++ b/scripts/search-eval/report.ts @@ -0,0 +1,147 @@ +import { readFileSync, writeFileSync } from "fs" +import { QueryMetrics, averageMetrics } from "./metrics" + +export type QueryResult = { + id: string + category: string + query: string + metrics: QueryMetrics + top10: string[] + relevantCount: number +} + +export type EvalResults = { + meta: { + env: string + url: string + alias: string + collection: string + serverVersion: string + corpusMd5: string + gitSha: string + timestamp: string + sortBy: string + court: number | undefined + skippedQueries: string[] + } + overall: QueryMetrics & { queryCount: number } + categories: Record + queries: QueryResult[] +} + +export function buildResults( + meta: EvalResults["meta"], + queries: QueryResult[] +): EvalResults { + const categories: EvalResults["categories"] = {} + const categoryNames = Array.from(new Set(queries.map(q => q.category))).sort() + for (const category of categoryNames) { + categories[category] = averageMetrics( + queries.filter(q => q.category === category).map(q => q.metrics) + ) + } + return { + meta, + overall: averageMetrics(queries.map(q => q.metrics)), + categories, + queries + } +} + +const row = (label: string, m: QueryMetrics & { queryCount: number }) => ({ + category: label, + queries: m.queryCount, + "recall@10": m.recallAt10.toFixed(3), + MRR: m.mrr.toFixed(3), + "nDCG@10": m.ndcgAt10.toFixed(3) +}) + +export function printScorecard(results: EvalResults) { + console.log( + `\nSearch relevance scorecard — ${results.meta.alias}, ` + + `server ${results.meta.serverVersion}, ` + + `collection ${results.meta.collection}, sort "${results.meta.sortBy}"` + + (results.meta.court !== undefined ? `, court ${results.meta.court}` : "") + ) + console.table([ + ...Object.entries(results.categories).map(([category, m]) => + row(category, m) + ), + row("OVERALL", results.overall) + ]) + if (results.meta.skippedQueries.length) + console.log( + `Skipped ${results.meta.skippedQueries.length} queries: ` + + results.meta.skippedQueries.join(", ") + ) +} + +export function writeResults(results: EvalResults, path: string) { + writeFileSync(path, JSON.stringify(results, null, 2) + "\n") + console.log(`Wrote results to ${path}`) +} + +export function compare( + baselinePath: string, + candidatePath: string, + threshold: number +): boolean { + const baseline: EvalResults = JSON.parse(readFileSync(baselinePath, "utf8")) + const candidate: EvalResults = JSON.parse(readFileSync(candidatePath, "utf8")) + + console.log( + `\nComparing ${candidatePath} (server ${candidate.meta.serverVersion})` + + ` against ${baselinePath} (server ${baseline.meta.serverVersion})` + ) + if (baseline.meta.alias !== candidate.meta.alias) + console.warn( + `WARNING: comparing different collections (${baseline.meta.alias} vs ${candidate.meta.alias})` + ) + if (baseline.meta.corpusMd5 !== candidate.meta.corpusMd5) + console.warn("WARNING: corpus hashes differ — not an apples-to-apples run") + + const delta = (b: QueryMetrics, c: QueryMetrics) => ({ + "Δrecall@10": (c.recallAt10 - b.recallAt10).toFixed(3), + ΔMRR: (c.mrr - b.mrr).toFixed(3), + "ΔnDCG@10": (c.ndcgAt10 - b.ndcgAt10).toFixed(3) + }) + + const categories = Array.from( + new Set([ + ...Object.keys(baseline.categories), + ...Object.keys(candidate.categories) + ]) + ).sort() + console.table([ + ...categories + .filter(c => baseline.categories[c] && candidate.categories[c]) + .map(c => ({ + category: c, + ...delta(baseline.categories[c], candidate.categories[c]) + })), + { category: "OVERALL", ...delta(baseline.overall, candidate.overall) } + ]) + + const baselineQueries = new Map(baseline.queries.map(q => [q.id, q])) + const regressions = candidate.queries.filter(q => { + const before = baselineQueries.get(q.id) + return before && before.metrics.ndcgAt10 - q.metrics.ndcgAt10 > threshold + }) + if (regressions.length) { + console.log(`\nPer-query nDCG@10 regressions > ${threshold}:`) + console.table( + regressions.map(q => ({ + id: q.id, + query: q.query, + before: baselineQueries.get(q.id)!.metrics.ndcgAt10.toFixed(3), + after: q.metrics.ndcgAt10.toFixed(3) + })) + ) + } + + const overallRegressed = + baseline.overall.ndcgAt10 - candidate.overall.ndcgAt10 > threshold + if (overallRegressed) + console.error(`Overall nDCG@10 regressed by more than ${threshold}`) + return !overallRegressed +} diff --git a/scripts/typesense-admin.ts b/scripts/typesense-admin.ts index be9eaa67e..7bf50df75 100644 --- a/scripts/typesense-admin.ts +++ b/scripts/typesense-admin.ts @@ -1,26 +1,23 @@ -import { execSync } from "child_process" import repl from "repl" import yargs, { Arguments } from "yargs" import { hideBin } from "yargs/helpers" import { createClient } from "../functions/src/search/client" +import { + SYNONYM_SET_NAME, + upsertLegislativeSynonyms +} from "../functions/src/search/synonyms" +import { aliases, evalCollections } from "./search-eval/collections" +import { + TypesenseConnectionArgs, + resolveClient, + typesenseEnvs +} from "./typesense-env" declare global { var client: ReturnType } -const envs: Record = { - local: { url: "http://localhost:8108", key: "test-api-key" }, - dev: { - url: "https://o89yhjf824.execute-api.us-east-1.amazonaws.com/search", - alias: "default" - }, - prod: { - url: "https://yyd73lsw3h.execute-api.us-east-1.amazonaws.com/search", - alias: "prod" - } -} - -type Args = Arguments<{ url?: string; key?: string; env?: string }> +type Args = Arguments yargs(hideBin(process.argv)) .scriptName("typesense-admin") .command( @@ -46,10 +43,115 @@ yargs(hideBin(process.argv)) console.log("Created", key.value) } ) + .command( + "status", + "print the server version, every alias with its live collection and document count, and any orphaned collections", + {}, + async (args: Args) => { + const client = resolveClient(args) + // Independent of the alias walk below; starts now, printed at the end. + // The catch is attached immediately: a fast rejection (a pre-cutover + // 0.24 gateway with no /synonym_sets route, an unreachable host) would + // otherwise crash the process as an unhandled rejection before any of + // the output below prints. + const synonymSetsPromise: Promise = client + .synonymSets() + .retrieve() + .catch((e: unknown) => e) + + const { version } = await client.debug.retrieve() + console.log(`server: ${version}`) + + const { aliases } = await client.aliases().retrieve() + const live = new Set(aliases.map(a => a.collection_name)) + + // One listing serves both the alias rows and the orphan check, instead + // of a round trip per alias plus the listing. + const collections = await client.collections().retrieve() + const documentCounts = new Map( + collections.map(c => [c.name, c.num_documents]) + ) + + if (aliases.length === 0) { + console.log("aliases: none — nothing is being served") + } + for (const { name, collection_name } of aliases) { + console.log( + `${name} -> ${collection_name} ${ + documentCounts.get(collection_name) ?? "?" + } docs` + ) + } + + /** A collection with no alias pointing at it is a backfill that never + * reached `upgradeAlias` — a run that failed, or one superseded by a + * later deploy before it finished. The backfill resumes from its cursor + * rather than from the first batch, so an orphan that is still growing is + * a run in progress; one that is not is wreckage. `yarn firebase-admin + * run-script searchUpgradeStatus` says which, and why. + */ + const orphans = collections.filter(c => !live.has(c.name)) + for (const { name, num_documents } of orphans) { + console.log(`(orphan) ${name} ${num_documents} docs`) + } + + /** Server state the searches reference by name; a missing set means + * every `synonym_sets` search runs with synonyms silently off. */ + const synonymSets = await synonymSetsPromise + if (!Array.isArray(synonymSets)) { + console.log(`synonym sets unavailable: ${synonymSets}`) + return + } + for (const { name, items } of synonymSets) { + console.log(`synonyms ${name} ${items.length} items`) + } + if (!synonymSets.some(s => s.name === SYNONYM_SET_NAME)) { + console.log(`(missing) synonym set "${SYNONYM_SET_NAME}"`) + } + } + ) + .command( + "preview-eval-corpus [alias]", + "point the local app alias(es) at their _eval collection, so `yarn dev:up`'s running app shows the frozen search-eval corpus instead of the dev-workflow backfill (run `yarn search-eval seed -e local` first). Kept separate from `search-eval seed` so eval runs never touch app aliases as a side effect.", + yargs => + yargs.positional("alias", { + string: true, + choices: aliases, + describe: "restrict to one alias; defaults to every eval collection" + }), + async (args: Args & { alias?: string }) => { + const client = resolveClient(args) + const targets = args.alias + ? [evalCollections[args.alias]] + : Object.values(evalCollections) + for (const { alias, evalCollection } of targets) { + const exists = await client.collections(evalCollection).exists() + if (!exists) { + throw Error( + `"${evalCollection}" doesn't exist yet — run: yarn search-eval seed -e local --alias ${alias}` + ) + } + await client + .aliases() + .upsert(alias, { collection_name: evalCollection }) + console.log(`${alias} -> ${evalCollection}`) + } + } + ) .command("list-keys", "list keys", {}, async (args: Args) => { const client = resolveClient(args) console.log(await client.keys().retrieve()) }) + .command( + "upsert-synonyms", + "upsert the legislative synonym set from functions/src/search/synonyms.ts (deploys also do this, via checkSearchIndexVersion)", + {}, + async (args: Args) => { + const client = resolveClient(args) + const { name, items } = await upsertLegislativeSynonyms(client) + console.log(`Upserted synonym set "${name}" with ${items} items`) + } + ) .command( "delete-key ", "list keys", @@ -62,7 +164,7 @@ yargs(hideBin(process.argv)) .options({ url: { string: true, alias: "u" }, key: { string: true, alias: "k" }, - env: { choices: Object.keys(envs), alias: "e" } + env: { choices: Object.keys(typesenseEnvs), alias: "e" } }) .check(argv => { if (!argv.env && !argv.url) return "Must specify env or url" @@ -70,29 +172,3 @@ yargs(hideBin(process.argv)) return true }) .strictCommands().argv - -function resolveClient(args: Args) { - let key: string | undefined, url: string | undefined - if (args.env) { - const env = envs[args.env] - if (!env) throw Error(`Invalid env, allowed values: ${Object.keys(envs)}`) - url = env.url - if (env.key) { - key = env.key - } else if (env.alias) { - key = execSync( - `yarn -s firebase --project ${env.alias} functions:secrets:access TYPESENSE_API_KEY` - ) - .toString() - .trim() - } else { - throw Error("Couldn't resolve env") - } - } - if (args.url) url = args.url - if (args.key) key = args.key - - if (!url || !key) throw new Error("Couldn't resolve url or key") - - return createClient({ apiUrl: url, apiKey: key }) -} diff --git a/scripts/typesense-env.ts b/scripts/typesense-env.ts new file mode 100644 index 000000000..e3862a10d --- /dev/null +++ b/scripts/typesense-env.ts @@ -0,0 +1,50 @@ +import { execSync } from "child_process" +import { createClient } from "../functions/src/search/client" + +export const typesenseEnvs: Record< + string, + { url: string; key?: string; alias?: string } +> = { + local: { url: "http://localhost:8108", key: "test-api-key" }, + dev: { + url: "https://o89yhjf824.execute-api.us-east-1.amazonaws.com/search", + alias: "default" + }, + prod: { + url: "https://yyd73lsw3h.execute-api.us-east-1.amazonaws.com/search", + alias: "prod" + } +} + +export type TypesenseConnectionArgs = { + url?: string + key?: string + env?: string +} + +export function resolveClient(args: TypesenseConnectionArgs) { + let key: string | undefined, url: string | undefined + if (args.env) { + const env = typesenseEnvs[args.env] + if (!env) + throw Error(`Invalid env, allowed values: ${Object.keys(typesenseEnvs)}`) + url = env.url + if (env.key) { + key = env.key + } else if (env.alias) { + key = execSync( + `yarn -s firebase --project ${env.alias} functions:secrets:access TYPESENSE_API_KEY` + ) + .toString() + .trim() + } else { + throw Error("Couldn't resolve env") + } + } + if (args.url) url = args.url + if (args.key) key = args.key + + if (!url || !key) throw new Error("Couldn't resolve url or key") + + return createClient({ apiUrl: url, apiKey: key }) +} diff --git a/tests/integration/search.test.ts b/tests/integration/search.test.ts index 71431113b..87e34a30e 100644 --- a/tests/integration/search.test.ts +++ b/tests/integration/search.test.ts @@ -7,7 +7,7 @@ import { nanoid } from "nanoid" import { CollectionAliasSchema } from "typesense/lib/Typesense/Aliases" import { Timestamp } from "../../functions/src/firebase" import { createClient } from "../../functions/src/search/client" -import { SearchIndexer } from "../../functions/src/search/SearchIndexer" +import { upgradePath } from "../../functions/src/search/backfillRun" import { testDb } from "../testUtils" import { createFakeBill } from "./common" @@ -75,9 +75,7 @@ describe.skip("Upgrades", () => { } async function lastUpgradeTime(): Promise { - const existingUpgrade = await testDb - .doc(SearchIndexer.upgradePath(testAlias)) - .get() + const existingUpgrade = await testDb.doc(upgradePath(testAlias)).get() expect(existingUpgrade.exists).toBeTruthy() return existingUpgrade.data()!.createdAt } diff --git a/tests/search-eval/README.md b/tests/search-eval/README.md new file mode 100644 index 000000000..e35bee65c --- /dev/null +++ b/tests/search-eval/README.md @@ -0,0 +1,379 @@ +# Search relevance eval + +Measures how well Typesense answers real queries, per collection, so search +changes ship with a number attached instead of a hunch. + +Scored with capped recall@10, MRR and nDCG@10 (exponential gain) over a golden +query set, against a frozen corpus snapshot seeded into a dedicated local +collection. Harness lives in `scripts/search-eval`. Shipping a measured +change: see [`docs/search-deployment.md`](../../docs/search-deployment.md). + +## The loop + +The repo needs node 22; the usual machine default is 24 and plain `yarn` fails +the engine check. + +```bash +export PATH="$HOME/.nvm/versions/node/v22.23.2/bin:$PATH" +yarn dev:up # local Typesense on :8108 +yarn typesense-admin upsert-synonyms --env local # the "legislative" synonym set + +yarn search-eval seed -e local --alias +yarn search-eval run -e local --alias --out /tmp/candidate.json +yarn search-eval compare tests/search-eval/baselines/.json /tmp/candidate.json +``` + +`--alias` is one of `bills` (the default), `hearings`, `publishedTestimony`. +`compare` exits non-zero when overall nDCG@10 drops by more than `--threshold` +(default 0.05) and lists per-query regressions. + +To author or revise a golden set, `yarn search-eval label -e local --alias ` +writes `labeling-sheet-.md` — every query's top 10 as a gradeable table. +Grade 3 = exactly what the searcher wanted, 1 = relevant, 0/blank = not, then +merge the graded rows back as explicit `docId` entries. + +Run `yarn check-types` as well as `yarn test`: jest transpiles differently and +hides errors the root tsconfig catches. + +## Corpora + +Each lives in `corpus//` as `docs.jsonl.gz` + `schema.json` + +`meta.json`. `meta.json` pins the gzip's md5, and the harness refuses to run +against a corpus that does not match it. Documents are produced by the +**production converter** (`functions/src/*/search.ts`), so the eval indexes +exactly what the app's search pipeline would. + +**bills** is built in two steps. The documents come from the committed emulator +fixture; the LLM `summary` field is joined in from a live project afterwards, +because the fixture was captured before the trigger in `llm/` existed and has a +summary for exactly 0 of its 7,337 bills: + +```bash +yarn search-eval:corpus # 7,337 bills, ~15s +yarn search-eval enrich --env prod --alias bills --field summary +``` + +Re-exporting bills from prod wholesale is the obvious alternative and it is +wrong. Court 192 closed in 2022, so prod's copy of it has drained out of its +policy committees — 36 distinct `currentCommittee` values in the fixture against +21 in prod, with none left in Revenue at all, which deletes `mc-013` outright and +costs member-committee 0.864 → 0.739. Joining the one missing field keeps every +other value frozen, so the control run reproduces the previous baseline exactly +and the field under test is the only thing that moved. + +**hearings** and **publishedTestimony** cannot. The fixture holds ~21 hearings, +and its testimony content is lorem ipsum (`tests/seed/seedTestimony.test.ts` +generates it with `loremIpsum()` for 4 test users) — there is no real language +to match against. They come from a live project instead, and need **no +credentials**: + +```bash +yarn search-eval corpus --env prod --alias hearings +yarn search-eval corpus --env prod --alias publishedTestimony --order-by publishedAt:desc +``` + +`firestore.rules` grants unauthenticated read on both sources — `events` is +"public, read-only" and publishedTestimony has an explicit +`match /{path=**}/publishedTestimony/{id}` rule so it can be read as a +collection group — so the web client SDK reads them with nothing but a project +id. `--env dev` works the same way against `digital-testimony-dev`. + +Bills needs one extra step. Its rule is path-scoped to `generalCourts/**` and +does not grant collection-group scope, so an unauthenticated +`collectionGroup("bills")` read is denied — but `generalCourts/192/bills` is a +plain collection read inside that scope and is allowed. `--court` substitutes +each general court for the trigger's `{court}` wildcard, turning the one denied +read into three permitted ones; `enrich` defaults it to the courts the corpus +already holds, since any other court's documents would only be discarded. Paging orders by `__name__` rather than the +config's id field, because prod carries a single-field index exemption on bills' +`id` and ordering by it fails with `failed-precondition`. + +All three corpora are **committed**, because none can be rebuilt byte-for-byte +from what the repo holds: they are snapshots of a moving source, and +re-exporting tomorrow gives a different md5. Committing pins the eval to one +snapshot, and refreshing it becomes a deliberate act. hearings and +publishedTestimony are small (126 KB and 992 KB gzipped); bills is 10 MB, the +price of carrying `body` for 7,337 bills. + +Re-export when a converter changes what it emits. The corpus md5 moves with it, +and `compare` will warn that the run is not apples-to-apples. To keep such a +change measurable, re-run the _previous_ search config against the new corpus +first and use that as the control. + +Two things the testimony corpus is not: + +- **Complete history.** It is every published testimony prod holds (1,056 at + time of writing), not a sample — but that is small next to bills' 7,337, and + the court refinement narrows each query to one court's share. Absolute scores + read higher than prod's full index would give; run-to-run deltas stay valid. +- **Unredacted.** `authorUid` becomes a stable hash and `fullName` is replaced + by `authorDisplayName`. Neither is in any `query_by` + (`components/search/searchParams.ts`), so nothing measured changes; + `authorDisplayName` and `content` are kept verbatim because they are already + published publicly and are what the eval scores. `meta.json` records which + fields were rewritten. + +## Golden sets + +`goldens/.json`. Each query carries either `relevant` labels or a +`sameAs` pointer to a query it shares labels with (how misspellings reuse their +correctly-spelled twin). A label is an explicit `docId` or a `rule` resolved +against the corpus: + +```json +{ + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "relevant": [ + { "rule": { "field": "billId", "equals": "S531" }, "grade": 3 }, + { "rule": { "field": "content", "contains": "ranked choice" }, "grade": 1 } + ] +} +``` + +Grades: **3** when the query names the thing exactly — a bill number, a +committee, an author, a location, a specific policy — and **1** when the +document is merely on the topic. A query whose labels are all one grade only +measures whether the right documents are found; mixing grades makes their +_order_ matter too, which is what the example above does: testimony on the +ranked-choice bill should outrank testimony that name-drops the phrase. + +Rule values and document values are compared with everything but letters and +digits stripped, so `contains: "health care"` also matches "healthcare", and +`equals: "Cindy F. Friedman"` needs the indexed form, not the form a searcher +types. Two traps when writing rules: + +- **Short `contains` values match inside words.** `contains: "gun"` matches + "begun" — 54 false positives in the testimony corpus. Prefer a stem long + enough to be unambiguous (`"incarcerat"`, `"disabilit"`). +- **A rule that resolves to nothing silently skips the query.** Check a + candidate rule against the corpus before committing it. + +`defaults.sort_by` must equal the collection's relevance sort exported from +`components/search/searchParams.ts`; `loadGoldens` throws otherwise. A golden +set scored under a sort no user can select measures an ordering nobody sees — +hearings was exactly that case until a relevance option was added, with every +sort date-ordered so nDCG measured recency. + +`defaults.courtFilter` mirrors the app's court refinement, pinned to the +corpus's majority court — 192 for the bills fixture, 194 for both prod corpora. +It is the _only_ app refinement the eval mirrors: hearings also defaults +`hasVideo: ["true"]` in its initial UI state, which the eval ignores because it +shrinks the candidate pool without being a relevance knob. + +### Prefer queries that can fail + +Broad topics saturate. An early draft of the testimony `topic` category asked +for "housing", "climate", "education" and scored a flat **1.000** on all ten +queries: with 100+ relevant documents each, the top ten fills with relevant +documents whatever the ranking does, so the category could not have detected a +regression. Narrow queries with 3-25 relevant documents discriminate, because +then recall@10's denominator is the set size and a missed document costs real +score. Prefer "solitary confinement" and "same day voter registration" over +"prisons" and "voting". + +## Baselines + +`baselines/*.json` are committed scorecards, one per landed search change, each +recording the collection, server version, corpus md5 and git sha it was produced +from. Current: + +| collection | queries | nDCG@10 | weakest categories | +| ------------------ | ------- | ------- | --------------------------------- | +| bills | 82 | 0.752 | topic 0.500, misspelling 0.507 | +| hearings | 48 | 0.957 | synonym 0.810, agenda-topic 0.842 | +| publishedTestimony | 39 | 0.980 | misspelling 0.943, topic 0.971 | + +Landed and measured: the legislative synonym set on hearings and testimony +(synonym 0.000 → 0.810 and 0.673 → 0.968), bill-number variants on testimony +(bill-id 0.848 → 1.000), hyphen tokenization on all three (bills +hyphenation 0.574 → 0.955, testimony topic +0.083), synonym head terms +for `alcohol`/`climate`/`vehicles` (#90) on all three (bills 0.766 → 0.754, +a known and accepted cost — see below; hearings and testimony unaffected), +bill pinslip at weight 3 (#87, bills 0.753 → 0.755, topic recall@10 +0.025), +the LLM bill summary at weight 2 (#76, bills 0.737 → 0.739 with recall@10 +0.860 → 0.880 and the new `plain-language` category 0.508 → 0.569), and sorting +procedural orders below everything else (#95, bills 0.739 → 0.752, +member-committee 0.864 → 0.934, nothing else moved at all). + +The bills row keeps dropping while bills search keeps improving, because each +change also adds queries the previous number never had to answer. Compare +baselines, not rows: the honest "before" for #76 is +`bills-summary-control.json`, the previous ranking config re-run against the new +corpus and the new goldens. `bills-pinslip.json` and everything older is not +comparable to it. Any change that moves the corpus md5 or the goldens needs the +same treatment. + +### What these goldens have already settled + +- **Weights are not a lever on either new collection.** Testimony's are inert: + inverting them entirely (billId 10→1, content 3→10) changes 2 of 39 queries' + top-10 and moves the score not at all, because its queries are field-disjoint + in practice — a bill number only appears in `billId`, an author name only in + `authorDisplayName`, topic words only in `content`. Hearings' are already + optimal: every small perturbation scores exactly 0.957, and the only vectors + that change anything drop `title` to or below `agendaTopics`, trading + committee accuracy (0.978 → 0.903) for agenda-topic (0.842 → 0.924) at a net + loss. +- **Hearings must not index bill-number variants.** Typesense pools tokens + across a field's array elements, so on a multi-valued `billNumbers` the query + "H 2391" matches a hearing whose agenda happens to list both an H-bill and an + S2391 — `hearing-5226`, with 46 unrelated bills, becomes a false positive. + Bills and testimony are safe: one variant per document, so tokens cannot + combine across neighbours. +- **A category cannot reward what it never asks for.** Hyphen tokenization first + measured at −0.002 on bills, because none of the 70 bills queries contained a + hyphenated phrase: the goldens could see the rank reshuffle it caused but not + the matches it unlocked. Six `hyphenation` queries later, the same change + measured +0.028 overall and +0.381 on that category. When a change scores + slightly negative, check whether anything in the set actually exercises it + before believing the number. +- **Typo tolerance masks synonym defects.** `liquor` appeared to reach the + "Alcohol" hearings, but only because the expansion "alcoholic" is two edits + from "alcohol"; at `num_typos=0` it found nothing. The actual defect (#90): + legislative synonym sets omitted their own head term (`alcohol`, `climate`, + `vehicles` were never in their own `synonyms` array), so a query for the + head term never triggered the group at all. Fixed by adding the head term + to each array. Verify synonym work with typos off, or a broken set will + look healthy. +- **Mixing single-word and multi-word entries in one synonym group inflates + single-word matches.** Fixing #90 above dropped bills' `liquor` golden + (`syn-009`) from 1.000 to near-zero nDCG, and it isn't a golden-coverage + gap like the hyphenation case — it's a real Typesense scoring quirk, traced + to source. For a single-token query, `words_present` credit for a + synonym-matched document is `syn_orig_num_tokens` — the _maximum_ token + length across every member of the resolved synonym group, not the length + of what actually matched (`index.cpp`, `score_results2` / + `syn_orig_num_tokens` computation). The `alcohol` group has two 2-word + phrases (`"alcoholic beverages"`, `"alcoholic beverage"`), so every + single-word match in that group — e.g. "alcohol" when you search "liquor" + — is credited as if 2 words matched, while the literal query match stays + correctly at 1. That component sits in the highest bits of `_text_match`, + so it dominates: general "alcohol" bills outrank literal "liquor" bills + wholesale, confirmed by decoding the raw scores and by reproducing/clearing + the effect with a stripped-down synonym set. `demote_synonym_match` does + not fix this — it operates on the lowest bits of the same score and never + gets a chance to matter. There's no clean fix: dropping the phrase entries + removes the inflation, but 62 of the 63 bills that say "alcoholic + beverage(s)" don't separately say "alcohol" or "liquor" anywhere, so that + would trade a scoring quirk for real lost recall. Kept the group as-is and + updated `syn-009` to grade literal `liquor`/`alcoholic beverage` matches at + 3 and general `alcohol` matches at 1, reflecting genuine relevance rather + than the old (accidentally-narrow) expectation — nDCG on that query stays + low (0.143) because the _ranking_ is still quirky, but recall and MRR + correctly show the fix works. `vehicles` mixes `"motor vehicle(s)"` the + same way, but that regression (`hyp-005`, a 3-token query) isn't this bug — + the inflation formula only fires for single-token queries — and doesn't have + a known fix either; documented as an accepted cost. +- **Rule-based judgments are biased against any field they don't name.** Every + bills `topic` golden grades relevance with `title`-contains rules plus one or + two hand-picked `docId`s, so a bill with a thin title and a topical pinslip + is grade 0 — outside both the numerator and the denominator. When #87 added + pinslip to `query_by`, promoting exactly those bills _displaced_ title-matching + grade-1 docs, and the score fell where the change was working. This is the + hyphenation bullet's evil twin: there the goldens were blind to the change, + here they penalise it. The remedy is to measure before grading — run the + candidate against the unchanged goldens, diff the top-10s against a control, + and hand-grade only what actually moved. Widening the rules instead (say to + `anyField: ["title","pinslip"]`) is the tempting shortcut and the wrong one: + it rewards the field under test by construction. +- **`misspelling` is not an independent category on bills.** 8 of its 10 + queries score _identically_ to the `topic` query they `sameAs`, because typo + tolerance fully recovers the query before ranking ever happens. It measures + topic's ordering a second time rather than testing typo handling, so a topic + movement double-counts into the overall number. Read the two together, and + don't describe a change as fixing two weak categories when it fixed one. +- **A new field's damage can be a document problem, not a ranking problem.** + Indexing the LLM `summary` cost member-committee 0.864 → 0.831, and `mc-014` + ("Education Committee") collapsed 0.558 → 0.064. None of it was the summaries. + The six documents it promoted were all procedural — "Extension Order - + Education", "Order relative to authorizing the joint committee on Education to + make an investigation and study" — and they already matched a committee query + in title, pinslip and body. A fourth matching field was enough, because + `fields_matched` sits in the lowest bits of `_text_match` and breaks ties. + Withholding `summary` from Orders puts the member-committee delta at exactly + 0.000 and leaves `mc-014` identical to the control, with the whole + plain-language gain intact. Two rounds of weight tuning and a ranker-mode + change all failed to buy back what one line in `convert` fixed. Before tuning + weights against a regression, look at _which documents_ moved. + + `content.LegislationTypeName` is the marker: a court is roughly 10% non-Bill + (prod's 192 is 7,780 Bill, 332 Order, 185 Extension Order, 166 Amendment, 74 + Resolve, and nine rarer types). Only Order and Extension Order are withheld — + Resolves and constitutional amendment proposals are legislation people search + for, so "Bill or nothing" would be the wrong cut. Dropping procedural + documents from the index altogether is worth a further +0.010 before any + ranking change, and member-committee 0.864 → 0.920, but it makes them + unfindable by number: a product decision, tracked separately. + +- **Demote procedural documents; do not drop them.** The follow-up to the + bullet above. `billsRelevanceSort` opens with + `` _eval(legislationType:!=`Order` && legislationType:!=`Extension Order`):desc `` + (two `!=` clauses, not the equivalent `!=[...]` list — the string doubles as + the InstantSearch index name, and qs parses square brackets in a URL key as + nesting, corrupting the routed search state), which sorts every + procedural document below every other one. Measured against dropping them from + the index outright: demotion 0.752 overall / member-committee 0.934, deletion + 0.749 / 0.920. Demotion wins on the number _and_ keeps the documents + reachable, so there is no case left for a `config.filter` here. Three queries + moved, all up — mc-015 +0.727, mc-013 +0.220, mc-014 +0.107 — and the other 79 + are byte-identical. + + Two things the goldens cannot tell you about it, both checked by hand. Lookup + by number survives: no bill shares an order's number, so `H4394` still returns + the Extension Order at rank 1 even though its whole class is demoted. Searching + for the type by name does not: "Extension Order Education" puts bills first and + the real orders at rank 26 of 32. Every `exact-bill-id` golden names a real + Bill, so the set could not have caught either. That second cost is why the + `legislationType` refinement ships with the sort rather than after it — + refining to the type lifts the demoted class back. + +- **`text_match_type: max_weight` is a dead end here, and the reason is + instructive.** It looks like the fix for the tiebreak above, and on "zero + emission vehicles" it is. But it scores a document off the highest-_weighted_ + matching field even when a lower-weighted field matched far better. Read off + `text_match_info` for "Education Committee": a bill in that committee matches + `currentCommittee` with `best_field_score` 2211897868289, while a study order + matches `title` more loosely at 2211897802753 — the committee match is 65,536 + (2^16, one step in the positional component) better. `max_score` keys on that + score and ranks the bill first; `max_weight` keys on the weight index (title + 14, currentCommittee 11) and ranks the order first. Raising `currentCommittee` + above `title` to compensate breaks the opposite case: "renewable energy" then + scores every bill in the Energy committee off the words "and Energy" in its + committee name, 24 documents share one score, and `testimonyCount` decides the + top of the list. The same field pair needs opposite priority for committee and + topic queries, so no weight ordering satisfies both — only a mode that asks + which field actually matched better can, which is what `max_score` does. + +- **The `plain-language` category exists because the other 76 queries could not + ask the question.** Summaries restate a bill in everyday words; every other + bills category is phrased in the legislature's vocabulary, so the change they + measured was dilution, not benefit. Six plain-language queries later, the same + change reads +0.061 nDCG, +0.100 recall@10 and +0.111 MRR on the category. + The gain is concentrated: `pl-001` "tenants facing eviction" goes 0.247 → + 0.613 because the control only _retrieves_ four documents for it — two of them + clean-energy bills — while summaries bring the real eviction bills into the + candidate set. That is a retrieval win, not a reranking one, which is why + recall@10 moves further than nDCG@10 everywhere in this change. Their ground + truth was identified from `title`/`pinslip`/`body` only, never from `summary`, + so the relevant sets are not derived from the field under test. + + `pl-004` "property tax break for solar" scores 0.000 in every configuration + measured, including the control. "property tax exemption solar" returns both + target bills at ranks 1 and 2. That is a _tax break_/_tax exemption_ + vocabulary gap and a candidate for the legislative synonym set — the category + found a defect it was not looking for, which is the argument for keeping + queries that fail. + +- **Long fields are topic-agnostic magnets.** Twenty bills (0.3%) are + procedural study orders whose `Pinslip` is not a description but a + concatenated docket of every petition referred to a committee — up to 28,049 + characters against a median of 224. Indexed whole, they matched nearly any + topical query: a search for "eviction" surfaced the Judiciary study order at + rank 6. `convert` now drops a pinslip over 2000 characters, which restored + real bills to those slots ("An Act promoting housing stability and + homelessness prevention" for eviction, "An Act creating a youth wage" for + minimum wage). The headline metric barely moved — both the noise and its + replacement were ungraded — so this one is visible in the top-10 diffs and + almost invisible in the score. Read the diffs. diff --git a/tests/search-eval/baselines/bills-order-demotion-control.json b/tests/search-eval/baselines/bills-order-demotion-control.json new file mode 100644 index 000000000..1e2b7f178 --- /dev/null +++ b/tests/search-eval/baselines/bills-order-demotion-control.json @@ -0,0 +1,1811 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "daf353d4bb28eec428b433d37ccf6d8a", + "gitSha": "572e3c5112f13e3a91fdbf5c79baaaa2ff0a12e9", + "timestamp": "2026-08-20T17:32:24.968Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8803571428571427, + "mrr": 0.9319589624467673, + "ndcgAt10": 0.7388146101520783, + "queryCount": 82 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 0.9166666666666666, + "ndcgAt10": 0.8912567800687374, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.9299999999999999, + "mrr": 1, + "ndcgAt10": 0.5068494239756252, + "queryCount": 10 + }, + "plain-language": { + "recallAt10": 0.5190476190476191, + "mrr": 0.75, + "ndcgAt10": 0.5686766029112228, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.8300000000000001, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.5000503759412055, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2580", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S874", + "192-S861", + "192-S623", + "192-S303", + "192-S2494", + "192-S2304", + "192-S2260", + "192-S2226", + "192-S2027", + "192-S1578" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H3711" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4303", + "192-H4282" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1984", + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806", + "192-S798", + "192-S791" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S1997", + "192-S1972" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2755", + "192-S2756", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2711", + "192-S2715", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.22593989791556013 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H2439", + "192-H1529", + "192-S942", + "192-S547", + "192-S1618", + "192-S1009", + "192-H2476", + "192-H1734" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2305", + "192-S2225", + "192-S2135" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.646574964942732 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4348", + "192-H3301" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6594071201332128 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-H656", + "192-H4110", + "192-H676", + "192-H598" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.4110138070100193 + }, + "top10": [ + "192-H3637", + "192-H2071", + "192-S2", + "192-H2086", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H3842", + "192-H3841" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.7276545800191442 + }, + "top10": [ + "192-S678", + "192-H1101", + "192-H125", + "192-S753", + "192-S679", + "192-H1293", + "192-H1212", + "192-S770", + "192-S766", + "192-S682" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S468", + "192-S451", + "192-S484", + "192-H834", + "192-H805", + "192-H3950", + "192-S459" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H547", + "192-S349", + "192-H564", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3335", + "192-H3330", + "192-H2159", + "192-S2157" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2258", + "192-H2124", + "192-H2051" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H373", + "192-H157", + "192-S197", + "192-S179", + "192-H454", + "192-H422", + "192-H4196" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-S1912", + "192-H1590", + "192-H3104", + "192-S358", + "192-S349", + "192-S2401", + "192-S1207" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H3722", + "192-H3526" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H2309", + "192-H1828", + "192-S984", + "192-S362", + "192-S1085", + "192-H605", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H272", + "192-H201", + "192-H119", + "192-H4512", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 0.5, + "ndcgAt10": 0.3475406804124245 + }, + "top10": [ + "192-S2253", + "192-H3434", + "192-S2230", + "192-S2229", + "192-S2151", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-H3888" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2456", + "192-H2112", + "192-H2091", + "192-H2726", + "192-H3635", + "192-S2462" + ], + "relevantCount": 8 + }, + { + "id": "pl-001", + "category": "plain-language", + "query": "tenants facing eviction", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.612554580191019 + }, + "top10": [ + "192-S2467", + "192-S2170", + "192-H3372", + "192-S891", + "192-H1436", + "192-H1434", + "192-S874", + "192-S889", + "192-H1440", + "192-H4148" + ], + "relevantCount": 17 + }, + { + "id": "pl-002", + "category": "plain-language", + "query": "cap insulin costs", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.4039439270866917 + }, + "top10": ["192-H4034"], + "relevantCount": 10 + }, + { + "id": "pl-003", + "category": "plain-language", + "query": "lead in school drinking water", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S580", + "192-H906", + "192-S1056", + "192-H1725", + "192-H939", + "192-S2580" + ], + "relevantCount": 5 + }, + { + "id": "pl-004", + "category": "plain-language", + "query": "property tax break for solar", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2226", "192-H3365", "192-S2708"], + "relevantCount": 2 + }, + { + "id": "pl-005", + "category": "plain-language", + "query": "background checks for gun sales", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.9173194127129571 + }, + "top10": ["192-H3729", "192-H1570"], + "relevantCount": 2 + }, + { + "id": "pl-006", + "category": "plain-language", + "query": "limit rent increases", + "metrics": { + "recallAt10": 0.7142857142857143, + "mrr": 0.5, + "ndcgAt10": 0.47824169747666917 + }, + "top10": [ + "192-S1808", + "192-H1440", + "192-S889", + "192-H3721", + "192-H1378", + "192-S886", + "192-S2225", + "192-H3838" + ], + "relevantCount": 7 + } + ] +} diff --git a/tests/search-eval/baselines/bills-order-demotion.json b/tests/search-eval/baselines/bills-order-demotion.json new file mode 100644 index 000000000..19afa2a8e --- /dev/null +++ b/tests/search-eval/baselines/bills-order-demotion.json @@ -0,0 +1,1811 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "daf353d4bb28eec428b433d37ccf6d8a", + "gitSha": "572e3c5112f13e3a91fdbf5c79baaaa2ff0a12e9", + "timestamp": "2026-08-20T17:55:35.005Z", + "sortBy": "_eval(legislationType:!=[`Order`,`Extension Order`]):desc,_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8901132404181182, + "mrr": 0.948509485094851, + "ndcgAt10": 0.7516737797578698, + "queryCount": 82 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 0.9166666666666666, + "ndcgAt10": 0.8912567800687374, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.94, + "mrr": 0.9444444444444444, + "ndcgAt10": 0.9344239443664147, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.9299999999999999, + "mrr": 1, + "ndcgAt10": 0.5068494239756252, + "queryCount": 10 + }, + "plain-language": { + "recallAt10": 0.5190476190476191, + "mrr": 0.75, + "ndcgAt10": 0.5686766029112228, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.8300000000000001, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.5000503759412055, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2580", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S874", + "192-S861", + "192-S623", + "192-S303", + "192-S2494", + "192-S2304", + "192-S2260", + "192-S2226", + "192-S2027", + "192-S1578" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H3711" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4303", + "192-H4282" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1984", + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806", + "192-S798", + "192-S791" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S1997", + "192-S1972", + "192-S1938" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6652907408280685 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319", + "192-S317", + "192-S2751", + "192-S2750" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237", + "192-S2230", + "192-S2222", + "192-S2202", + "192-S2198", + "192-S2191", + "192-S2184" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.22593989791556013 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H2439", + "192-H1529", + "192-S942", + "192-S547", + "192-S1618", + "192-S1009", + "192-H2476", + "192-H1734" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2305", + "192-S2225", + "192-S2135" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.646574964942732 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4348", + "192-H3301" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6594071201332128 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-H656", + "192-H4110", + "192-H676", + "192-H598" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.4110138070100193 + }, + "top10": [ + "192-H3637", + "192-H2071", + "192-S2", + "192-H2086", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H3842", + "192-H3841" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.7276545800191442 + }, + "top10": [ + "192-S678", + "192-H1101", + "192-H125", + "192-S753", + "192-S679", + "192-H1293", + "192-H1212", + "192-S770", + "192-S766", + "192-S682" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S468", + "192-S451", + "192-S484", + "192-H834", + "192-H805", + "192-H3950", + "192-S459" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H547", + "192-S349", + "192-H564", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3335", + "192-H3330", + "192-H2159", + "192-S2157" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2258", + "192-H2124", + "192-H2051" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H373", + "192-H157", + "192-S197", + "192-S179", + "192-H454", + "192-H422", + "192-H4196" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-S1912", + "192-H1590", + "192-H3104", + "192-S358", + "192-S349", + "192-S2401", + "192-S1207" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H3722", + "192-H3526" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H2309", + "192-H1828", + "192-S984", + "192-S362", + "192-S1085", + "192-H605", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H272", + "192-H201", + "192-H119", + "192-H4512", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 0.5, + "ndcgAt10": 0.3475406804124245 + }, + "top10": [ + "192-S2253", + "192-H3434", + "192-S2230", + "192-S2229", + "192-S2151", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-H3888" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2456", + "192-H2112", + "192-H2091", + "192-H2726", + "192-H3635", + "192-S2462" + ], + "relevantCount": 8 + }, + { + "id": "pl-001", + "category": "plain-language", + "query": "tenants facing eviction", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.612554580191019 + }, + "top10": [ + "192-S2467", + "192-S2170", + "192-H3372", + "192-S891", + "192-H1436", + "192-H1434", + "192-S874", + "192-S889", + "192-H1440", + "192-H4148" + ], + "relevantCount": 17 + }, + { + "id": "pl-002", + "category": "plain-language", + "query": "cap insulin costs", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.4039439270866917 + }, + "top10": ["192-H4034"], + "relevantCount": 10 + }, + { + "id": "pl-003", + "category": "plain-language", + "query": "lead in school drinking water", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S580", + "192-H906", + "192-S1056", + "192-H1725", + "192-H939", + "192-S2580" + ], + "relevantCount": 5 + }, + { + "id": "pl-004", + "category": "plain-language", + "query": "property tax break for solar", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2226", "192-H3365", "192-S2708"], + "relevantCount": 2 + }, + { + "id": "pl-005", + "category": "plain-language", + "query": "background checks for gun sales", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.9173194127129571 + }, + "top10": ["192-H3729", "192-H1570"], + "relevantCount": 2 + }, + { + "id": "pl-006", + "category": "plain-language", + "query": "limit rent increases", + "metrics": { + "recallAt10": 0.7142857142857143, + "mrr": 0.5, + "ndcgAt10": 0.47824169747666917 + }, + "top10": [ + "192-S1808", + "192-H1440", + "192-S889", + "192-H3721", + "192-H1378", + "192-S886", + "192-S2225", + "192-H3838" + ], + "relevantCount": 7 + } + ] +} diff --git a/tests/search-eval/baselines/bills-pinslip-control.json b/tests/search-eval/baselines/bills-pinslip-control.json new file mode 100644 index 000000000..e01320d56 --- /dev/null +++ b/tests/search-eval/baselines/bills-pinslip-control.json @@ -0,0 +1,1706 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "c4ba2f5615bcf2199d2515c07e16ca47", + "gitSha": "b90d5e22fa071ddef3f370da32176ad00dca19de", + "timestamp": "2026-08-20T15:04:56.402Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8851973684210526, + "mrr": 0.9594820384294069, + "ndcgAt10": 0.7532666164499898, + "queryCount": 76 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 1, + "ndcgAt10": 0.9231671885460636, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.86, + "mrr": 1, + "ndcgAt10": 0.4928253138091646, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.775, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.5013655302410447, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.17571068025797132 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615", + "192-S1130" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2522", + "192-S2305", + "192-S2225" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8752656443235947 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2318", + "192-H2258" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374", + "192-S900" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H723", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H4196", + "192-H373", + "192-H157", + "192-S200", + "192-S197", + "192-S179", + "192-S1278" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349", + "192-S2580" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8752656443235947 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S852", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H4481" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S362", + "192-S1085", + "192-H605", + "192-H2309", + "192-H1828", + "192-S984", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H4512", + "192-H272", + "192-H201", + "192-H1779", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 1, + "ndcgAt10": 0.539003131276382 + }, + "top10": [ + "192-S2151", + "192-H3434", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-S2230", + "192-S2229", + "192-H3888", + "192-H3415" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2726", + "192-H2456", + "192-H2112", + "192-H2091", + "192-S70", + "192-S2761" + ], + "relevantCount": 8 + } + ] +} diff --git a/tests/search-eval/baselines/bills-pinslip.json b/tests/search-eval/baselines/bills-pinslip.json new file mode 100644 index 000000000..1ab722f8d --- /dev/null +++ b/tests/search-eval/baselines/bills-pinslip.json @@ -0,0 +1,1706 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "c4ba2f5615bcf2199d2515c07e16ca47", + "gitSha": "b90d5e22fa071ddef3f370da32176ad00dca19de", + "timestamp": "2026-08-20T15:05:15.155Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.894407894736842, + "mrr": 0.9594820384294069, + "ndcgAt10": 0.755003880355015, + "queryCount": 76 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 1, + "ndcgAt10": 0.9231671885460636, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8800000000000001, + "mrr": 1, + "ndcgAt10": 0.48969440043315426, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.8, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.5095325897681456, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2580", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S874", + "192-S861", + "192-S623", + "192-S303", + "192-S2494", + "192-S2304", + "192-S2260", + "192-S2226", + "192-S2027", + "192-S1578" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H3711" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4303", + "192-H4282" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1984", + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806", + "192-S798", + "192-S795" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S1997", + "192-S1972" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2755", + "192-S2756", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2711", + "192-S2715", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.2036279870679677 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-H2439", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2522", + "192-S2305", + "192-S2225" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.646574964942732 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4348", + "192-H3301" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.8989054358937958 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1959", + "192-H2013", + "192-H1964" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6594071201332128 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-H656", + "192-H4110", + "192-H676", + "192-H598" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.564174087041131 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H2101", + "192-H3796" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H3842", + "192-H3841" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6003787094494394 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.5137267312977696 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.7276545800191442 + }, + "top10": [ + "192-S678", + "192-H1101", + "192-H125", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H547", + "192-H564", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3708", + "192-H3335", + "192-H3330", + "192-H2159" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2258", + "192-H2124" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H373", + "192-H157", + "192-S200", + "192-S197", + "192-S179", + "192-S1278", + "192-H454" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-S1912", + "192-H1590", + "192-H3104", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.564174087041131 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6003787094494394 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.5137267312977696 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.8989054358937958 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1959", + "192-H2013", + "192-H1964" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H3722", + "192-H3526" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S362", + "192-S1085", + "192-H605", + "192-H2309", + "192-H1828", + "192-S984", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H272", + "192-H201", + "192-H119", + "192-H4512", + "192-H1779" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 1, + "ndcgAt10": 0.539003131276382 + }, + "top10": [ + "192-S2151", + "192-H3434", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-S2230", + "192-S2229", + "192-H3888", + "192-H3415" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2726", + "192-H2456", + "192-H2112", + "192-H2091", + "192-H3635", + "192-S70" + ], + "relevantCount": 8 + } + ] +} diff --git a/tests/search-eval/baselines/bills-summary-control.json b/tests/search-eval/baselines/bills-summary-control.json new file mode 100644 index 000000000..b5013073f --- /dev/null +++ b/tests/search-eval/baselines/bills-summary-control.json @@ -0,0 +1,1800 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "96afb294d3334c14221b3a891b6961cc", + "gitSha": "ca5708ac50cb1b08f252ef3025454dc00e493d54", + "timestamp": "2026-08-20T16:41:48.003Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8596254355400695, + "mrr": 0.9360240030971738, + "ndcgAt10": 0.7369126389734455, + "queryCount": 82 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 1, + "ndcgAt10": 0.9231671885460636, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8800000000000001, + "mrr": 1, + "ndcgAt10": 0.48969440043315426, + "queryCount": 10 + }, + "plain-language": { + "recallAt10": 0.41904761904761906, + "mrr": 0.6388888888888888, + "ndcgAt10": 0.5077569148068996, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.8, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.5095325897681456, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2580", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S874", + "192-S861", + "192-S623", + "192-S303", + "192-S2494", + "192-S2304", + "192-S2260", + "192-S2226", + "192-S2027", + "192-S1578" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H3711" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4303", + "192-H4282" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1984", + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806", + "192-S798", + "192-S795" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S1997", + "192-S1972" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2755", + "192-S2756", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2711", + "192-S2715", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.2036279870679677 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-H2439", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2522", + "192-S2305", + "192-S2225" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.646574964942732 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4348", + "192-H3301" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.8989054358937958 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1959", + "192-H2013", + "192-H1964" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6594071201332128 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-H656", + "192-H4110", + "192-H676", + "192-H598" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.564174087041131 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H2101", + "192-H3796" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H3842", + "192-H3841" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6003787094494394 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.5137267312977696 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.7276545800191442 + }, + "top10": [ + "192-S678", + "192-H1101", + "192-H125", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H547", + "192-H564", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3708", + "192-H3335", + "192-H3330", + "192-H2159" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2258", + "192-H2124" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H373", + "192-H157", + "192-S200", + "192-S197", + "192-S179", + "192-S1278", + "192-H454" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-S1912", + "192-H1590", + "192-H3104", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.564174087041131 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6003787094494394 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.5137267312977696 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.8989054358937958 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1959", + "192-H2013", + "192-H1964" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H3722", + "192-H3526" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S362", + "192-S1085", + "192-H605", + "192-H2309", + "192-H1828", + "192-S984", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H272", + "192-H201", + "192-H119", + "192-H4512", + "192-H1779" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 1, + "ndcgAt10": 0.539003131276382 + }, + "top10": [ + "192-S2151", + "192-H3434", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-S2230", + "192-S2229", + "192-H3888", + "192-H3415" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2726", + "192-H2456", + "192-H2112", + "192-H2091", + "192-H3635", + "192-S70" + ], + "relevantCount": 8 + }, + { + "id": "pl-001", + "category": "plain-language", + "query": "tenants facing eviction", + "metrics": { + "recallAt10": 0.2, + "mrr": 0.3333333333333333, + "ndcgAt10": 0.2470364515650797 + }, + "top10": ["192-S2170", "192-H3372", "192-S889", "192-H1440"], + "relevantCount": 17 + }, + { + "id": "pl-002", + "category": "plain-language", + "query": "cap insulin costs", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.4039439270866917 + }, + "top10": ["192-H4034"], + "relevantCount": 10 + }, + { + "id": "pl-003", + "category": "plain-language", + "query": "lead in school drinking water", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S580", + "192-H906", + "192-S1056", + "192-H1725", + "192-H939", + "192-S2580" + ], + "relevantCount": 5 + }, + { + "id": "pl-004", + "category": "plain-language", + "query": "property tax break for solar", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2226", "192-H3365", "192-S2708"], + "relevantCount": 2 + }, + { + "id": "pl-005", + "category": "plain-language", + "query": "background checks for gun sales", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.9173194127129571 + }, + "top10": ["192-H3729"], + "relevantCount": 2 + }, + { + "id": "pl-006", + "category": "plain-language", + "query": "limit rent increases", + "metrics": { + "recallAt10": 0.7142857142857143, + "mrr": 0.5, + "ndcgAt10": 0.47824169747666917 + }, + "top10": [ + "192-S1808", + "192-H1440", + "192-S889", + "192-H3721", + "192-H1378", + "192-S886", + "192-S2225", + "192-H3838" + ], + "relevantCount": 7 + } + ] +} diff --git a/tests/search-eval/baselines/bills-summary.json b/tests/search-eval/baselines/bills-summary.json new file mode 100644 index 000000000..36a47b94c --- /dev/null +++ b/tests/search-eval/baselines/bills-summary.json @@ -0,0 +1,1811 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "256fe0c51980064a1a308cefcde070ad", + "gitSha": "ca5708ac50cb1b08f252ef3025454dc00e493d54", + "timestamp": "2026-08-20T17:02:56.900Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8803571428571427, + "mrr": 0.9319589624467673, + "ndcgAt10": 0.7388146101520783, + "queryCount": 82 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 0.9166666666666666, + "ndcgAt10": 0.8912567800687374, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.9299999999999999, + "mrr": 1, + "ndcgAt10": 0.5068494239756252, + "queryCount": 10 + }, + "plain-language": { + "recallAt10": 0.5190476190476191, + "mrr": 0.75, + "ndcgAt10": 0.5686766029112228, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.8300000000000001, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.5000503759412055, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2580", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S874", + "192-S861", + "192-S623", + "192-S303", + "192-S2494", + "192-S2304", + "192-S2260", + "192-S2226", + "192-S2027", + "192-S1578" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H3711" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4303", + "192-H4282" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1984", + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806", + "192-S798", + "192-S791" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S1997", + "192-S1972" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2755", + "192-S2756", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2711", + "192-S2715", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.22593989791556013 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H2439", + "192-H1529", + "192-S942", + "192-S547", + "192-S1618", + "192-S1009", + "192-H2476", + "192-H1734" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2305", + "192-S2225", + "192-S2135" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.646574964942732 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4348", + "192-H3301" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6594071201332128 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-H656", + "192-H4110", + "192-H676", + "192-H598" + ], + "relevantCount": 15 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.4110138070100193 + }, + "top10": [ + "192-H3637", + "192-H2071", + "192-S2", + "192-H2086", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H3842", + "192-H3841" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.7276545800191442 + }, + "top10": [ + "192-S678", + "192-H1101", + "192-H125", + "192-S753", + "192-S679", + "192-H1293", + "192-H1212", + "192-S770", + "192-S766", + "192-S682" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S468", + "192-S451", + "192-S484", + "192-H834", + "192-H805", + "192-H3950", + "192-S459" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H547", + "192-S349", + "192-H564", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3335", + "192-H3330", + "192-H2159", + "192-S2157" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S1614", + "192-S1568", + "192-S1563", + "192-S1561", + "192-S1537", + "192-H43", + "192-H4236", + "192-H3847", + "192-H3846" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681", + "192-H594" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2660", + "192-S77", + "192-S76", + "192-S74", + "192-S73", + "192-S69", + "192-S64", + "192-S63" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2258", + "192-H2124", + "192-H2051" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H1374", + "192-S900", + "192-S399" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H373", + "192-H157", + "192-S197", + "192-S179", + "192-H454", + "192-H422", + "192-H4196" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-S1912", + "192-H1590", + "192-H3104", + "192-S358", + "192-S349", + "192-S2401", + "192-S1207" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5752285066691645 + }, + "top10": [ + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H229", + "192-H1436", + "192-H1385", + "192-S874" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1023", + "192-S1015", + "192-H3842" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.6019750269668604 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H1808", + "192-H4505", + "192-H1911", + "192-S891", + "192-S886", + "192-S874", + "192-S865", + "192-S2467" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.6700894585557804 + }, + "top10": [ + "192-H2394", + "192-H2411", + "192-S2467", + "192-S1755", + "192-S1515", + "192-H2370", + "192-S1517", + "192-S1426", + "192-S1424", + "192-H498" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.901442206915039 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-H1971", + "192-H2013", + "192-H1964", + "192-S121", + "192-S1184" + ], + "relevantCount": 5 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H3722", + "192-H3526" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H2309", + "192-H1828", + "192-S984", + "192-S362", + "192-S1085", + "192-H605", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H272", + "192-H201", + "192-H119", + "192-H4512", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 0.5, + "ndcgAt10": 0.3475406804124245 + }, + "top10": [ + "192-S2253", + "192-H3434", + "192-S2230", + "192-S2229", + "192-S2151", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-H3888" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2456", + "192-H2112", + "192-H2091", + "192-H2726", + "192-H3635", + "192-S2462" + ], + "relevantCount": 8 + }, + { + "id": "pl-001", + "category": "plain-language", + "query": "tenants facing eviction", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.612554580191019 + }, + "top10": [ + "192-S2467", + "192-S2170", + "192-H3372", + "192-S891", + "192-H1436", + "192-H1434", + "192-S874", + "192-S889", + "192-H1440", + "192-H4148" + ], + "relevantCount": 17 + }, + { + "id": "pl-002", + "category": "plain-language", + "query": "cap insulin costs", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.4039439270866917 + }, + "top10": ["192-H4034"], + "relevantCount": 10 + }, + { + "id": "pl-003", + "category": "plain-language", + "query": "lead in school drinking water", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S580", + "192-H906", + "192-S1056", + "192-H1725", + "192-H939", + "192-S2580" + ], + "relevantCount": 5 + }, + { + "id": "pl-004", + "category": "plain-language", + "query": "property tax break for solar", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2226", "192-H3365", "192-S2708"], + "relevantCount": 2 + }, + { + "id": "pl-005", + "category": "plain-language", + "query": "background checks for gun sales", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.9173194127129571 + }, + "top10": ["192-H3729", "192-H1570"], + "relevantCount": 2 + }, + { + "id": "pl-006", + "category": "plain-language", + "query": "limit rent increases", + "metrics": { + "recallAt10": 0.7142857142857143, + "mrr": 0.5, + "ndcgAt10": 0.47824169747666917 + }, + "top10": [ + "192-S1808", + "192-H1440", + "192-S889", + "192-H3721", + "192-H1378", + "192-S886", + "192-S2225", + "192-H3838" + ], + "relevantCount": 7 + } + ] +} diff --git a/tests/search-eval/baselines/bills-synonym-head-terms.json b/tests/search-eval/baselines/bills-synonym-head-terms.json new file mode 100644 index 000000000..59d8751f9 --- /dev/null +++ b/tests/search-eval/baselines/bills-synonym-head-terms.json @@ -0,0 +1,1706 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "6041d1fed1399eece7aa523428e97397", + "gitSha": "9c9059a3f59913ac86a6231efc064573fc6a2d6c", + "timestamp": "2026-08-20T14:31:23.263Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8904605263157894, + "mrr": 0.9594820384294069, + "ndcgAt10": 0.7539882788671833, + "queryCount": 76 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.8958333333333334, + "mrr": 1, + "ndcgAt10": 0.9231671885460636, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8800000000000001, + "mrr": 1, + "ndcgAt10": 0.49556763099450035, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.879178871818899, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.7849999999999999, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.5027366888337126, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.17571068025797132 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615", + "192-S1130" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.5812727842509424 + }, + "top10": [ + "192-H614", + "192-S9", + "192-S7", + "192-S571", + "192-S501", + "192-S311", + "192-S2737", + "192-S2522", + "192-S2305", + "192-S2225" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2318", + "192-H2258" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374", + "192-S900" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H723", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.14285714285714285 + }, + "top10": [ + "192-S168", + "192-S1001", + "192-S1000", + "192-H4196", + "192-H373", + "192-H157", + "192-S200", + "192-S197", + "192-S179", + "192-S1278" + ], + "relevantCount": 106 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349", + "192-S2580" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S852", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H4481" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S362", + "192-S1085", + "192-H605", + "192-H2309", + "192-H1828", + "192-S984", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H4512", + "192-H272", + "192-H201", + "192-H1779", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.375, + "mrr": 1, + "ndcgAt10": 0.539003131276382 + }, + "top10": [ + "192-S2151", + "192-H3434", + "192-H3347", + "192-S9", + "192-S2351", + "192-S2265", + "192-S2230", + "192-S2229", + "192-H3888", + "192-H3415" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2726", + "192-H2456", + "192-H2112", + "192-H2091", + "192-S70", + "192-S2761" + ], + "relevantCount": 8 + } + ] +} diff --git a/tests/search-eval/baselines/bills-token-separators.json b/tests/search-eval/baselines/bills-token-separators.json new file mode 100644 index 000000000..b7bbe833b --- /dev/null +++ b/tests/search-eval/baselines/bills-token-separators.json @@ -0,0 +1,1706 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "6041d1fed1399eece7aa523428e97397", + "gitSha": "c4a843c12cd9c72061ddae203bff02aa4ae2db18", + "timestamp": "2026-08-19T20:58:48.706Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8937499999999999, + "mrr": 0.9594820384294069, + "ndcgAt10": 0.766011702009288, + "queryCount": 76 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.9375, + "mrr": 1, + "ndcgAt10": 0.9546223604094309, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8800000000000001, + "mrr": 1, + "ndcgAt10": 0.49556763099450035, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.95, + "mrr": 1, + "ndcgAt10": 0.9648931575331847, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.7849999999999999, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.49613200235755717, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.17571068025797132 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615", + "192-S1130" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2318", + "192-H2258" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374", + "192-S900" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H723", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-H1374" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S40", + "192-S2788", + "192-S2556", + "192-S2511", + "192-S2509", + "192-S2507", + "192-S2506", + "192-S2504", + "192-S2501", + "192-S2500" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.6489315753318465 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349", + "192-S2580" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-S1349", + "192-H4026" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6022386851026632 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2219", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.5713510976253698 + }, + "top10": [ + "192-S944", + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S97", + "192-S852", + "192-S2726", + "192-S2724", + "192-S2386", + "192-S1830", + "192-S1227", + "192-S121", + "192-H560", + "192-H4481" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S412", + "192-S1482", + "192-S1476", + "192-S133", + "192-S671", + "192-S659", + "192-S400", + "192-H754", + "192-H749", + "192-H736" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S362", + "192-S1085", + "192-H605", + "192-H2309", + "192-H1828", + "192-S984", + "192-S1442", + "192-S100", + "192-H3845", + "192-H1569" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S978", + "192-S753", + "192-S60", + "192-S2688", + "192-S124", + "192-H4512", + "192-H272", + "192-H201", + "192-H1779", + "192-H1497" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.625, + "mrr": 1, + "ndcgAt10": 0.7277341624565862 + }, + "top10": [ + "192-S2151", + "192-S2130", + "192-H3347", + "192-H4134", + "192-S2351", + "192-S2255", + "192-S2254", + "192-S2225", + "192-H3888", + "192-S2292" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2403", + "192-S1691", + "192-H3664", + "192-H3663", + "192-H2726", + "192-H2456", + "192-H2112", + "192-H2091", + "192-S70", + "192-S2761" + ], + "relevantCount": 8 + } + ] +} diff --git a/tests/search-eval/baselines/hearings-base.json b/tests/search-eval/baselines/hearings-base.json new file mode 100644 index 000000000..3430ab8f9 --- /dev/null +++ b/tests/search-eval/baselines/hearings-base.json @@ -0,0 +1,1004 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "hearings", + "collection": "hearings_eval", + "serverVersion": "30.2", + "corpusMd5": "fa6e9fe42e3da564da0bd72eef1e55aa", + "gitSha": "7e6529b622c9e1b2b45a53add525c355f96a1ba3", + "timestamp": "2026-08-19T18:32:18.185Z", + "sortBy": "_text_match:desc,startsAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.90625, + "mrr": 0.9270833333333334, + "ndcgAt10": 0.9065328757618745, + "queryCount": 48 + }, + "categories": { + "agenda-topic": { + "recallAt10": 0.825, + "mrr": 1, + "ndcgAt10": 0.8417087253585072, + "queryCount": 8 + }, + "bill-on-agenda": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 10 + }, + "chair": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 6 + }, + "committee": { + "recallAt10": 0.99, + "mrr": 0.95, + "ndcgAt10": 0.977990823370192, + "queryCount": 10 + }, + "location": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 5 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0, + "queryCount": 3 + } + }, + "queries": [ + { + "id": "hc-001", + "category": "committee", + "query": "Ways and Means", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5099", + "hearing-5594", + "hearing-5585", + "hearing-5577", + "hearing-5582", + "hearing-5584", + "hearing-5575", + "hearing-5583", + "hearing-5546", + "hearing-5100" + ], + "relevantCount": 19 + }, + { + "id": "hc-002", + "category": "committee", + "query": "Judiciary", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5639", + "hearing-5494", + "hearing-5478", + "hearing-5452", + "hearing-5434", + "hearing-5388", + "hearing-5358", + "hearing-5229", + "hearing-5227", + "hearing-5216" + ], + "relevantCount": 13 + }, + { + "id": "hc-003", + "category": "committee", + "query": "Financial Services", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5403", + "hearing-5721", + "hearing-5576", + "hearing-5524", + "hearing-5473", + "hearing-5457", + "hearing-5448", + "hearing-5439", + "hearing-5417", + "hearing-5374" + ], + "relevantCount": 21 + }, + { + "id": "hc-004", + "category": "committee", + "query": "Public Health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5532", + "hearing-5390", + "hearing-5239", + "hearing-5236", + "hearing-5222", + "hearing-5628", + "hearing-5360", + "hearing-5221", + "hearing-5211", + "hearing-5110" + ], + "relevantCount": 13 + }, + { + "id": "hc-005", + "category": "committee", + "query": "Transportation", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5431", + "hearing-5407", + "hearing-5738", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5505", + "hearing-5454", + "hearing-5400", + "hearing-5345" + ], + "relevantCount": 14 + }, + { + "id": "hc-006", + "category": "committee", + "query": "Housing", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019198 + }, + "top10": [ + "hearing-5366", + "hearing-5197", + "hearing-5750", + "hearing-5718", + "hearing-5526", + "hearing-5474", + "hearing-5485", + "hearing-5419", + "hearing-5367", + "hearing-5240" + ], + "relevantCount": 11 + }, + { + "id": "hc-007", + "category": "committee", + "query": "Election Laws", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5224", + "hearing-5113", + "hearing-5735", + "hearing-5627", + "hearing-5545", + "hearing-5508", + "hearing-5451", + "hearing-5426", + "hearing-5322", + "hearing-5151" + ], + "relevantCount": 10 + }, + { + "id": "hc-008", + "category": "committee", + "query": "Revenue", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5178", + "hearing-5748", + "hearing-5722", + "hearing-5687", + "hearing-5549", + "hearing-5535", + "hearing-5479", + "hearing-5458", + "hearing-5447", + "hearing-5427" + ], + "relevantCount": 19 + }, + { + "id": "hc-009", + "category": "committee", + "query": "Consumer Protection", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5147", + "hearing-5395", + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5433", + "hearing-5416", + "hearing-5357" + ], + "relevantCount": 13 + }, + { + "id": "hc-010", + "category": "committee", + "query": "Municipalities and Regional Government", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5684", + "hearing-5572", + "hearing-5212", + "hearing-5152", + "hearing-5741", + "hearing-5719", + "hearing-5637", + "hearing-5516", + "hearing-5503", + "hearing-5495" + ], + "relevantCount": 18 + }, + { + "id": "hch-001", + "category": "chair", + "query": "Rebecca Rausch", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5744", + "hearing-5741", + "hearing-5727", + "hearing-5719", + "hearing-5684", + "hearing-5637", + "hearing-5581", + "hearing-5579", + "hearing-5572", + "hearing-5238" + ], + "relevantCount": 15 + }, + { + "id": "hch-002", + "category": "chair", + "query": "Cindy Friedman", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 15 + }, + { + "id": "hch-003", + "category": "chair", + "query": "Alice Peisch", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 10 + }, + { + "id": "hch-004", + "category": "chair", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5237", + "hearing-5194", + "hearing-5171", + "hearing-5147" + ], + "relevantCount": 9 + }, + { + "id": "hch-005", + "category": "chair", + "query": "Brendan Crighton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5738", + "hearing-5734", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5226", + "hearing-5195", + "hearing-5174", + "hearing-5166" + ], + "relevantCount": 9 + }, + { + "id": "hch-006", + "category": "chair", + "query": "Daniel Ryan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5732", + "hearing-5701", + "hearing-5658", + "hearing-5586", + "hearing-5534", + "hearing-5206", + "hearing-5203", + "hearing-5192", + "hearing-5177", + "hearing-5163" + ], + "relevantCount": 10 + }, + { + "id": "hb-001", + "category": "bill-on-agenda", + "query": "H2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-002", + "category": "bill-on-agenda", + "query": "H 2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-003", + "category": "bill-on-agenda", + "query": "H.2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-004", + "category": "bill-on-agenda", + "query": "S1489", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-005", + "category": "bill-on-agenda", + "query": "S 1489", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-006", + "category": "bill-on-agenda", + "query": "H63", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5113", + "hearing-5396", + "hearing-5362", + "hearing-5220", + "hearing-5189", + "hearing-5168", + "hearing-5141" + ], + "relevantCount": 1 + }, + { + "id": "hb-007", + "category": "bill-on-agenda", + "query": "H 63", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5113"], + "relevantCount": 1 + }, + { + "id": "hb-008", + "category": "bill-on-agenda", + "query": "S6", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5113", + "hearing-5475", + "hearing-5448", + "hearing-5442", + "hearing-5439", + "hearing-5190" + ], + "relevantCount": 1 + }, + { + "id": "hb-009", + "category": "bill-on-agenda", + "query": "H56", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5458", + "hearing-5326", + "hearing-5376", + "hearing-5459", + "hearing-5396", + "hearing-5362", + "hearing-5189", + "hearing-5168" + ], + "relevantCount": 3 + }, + { + "id": "hb-010", + "category": "bill-on-agenda", + "query": "H.2424", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "ht-001", + "category": "agenda-topic", + "query": "home rule petitions", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5516", + "hearing-5503", + "hearing-5462", + "hearing-5438", + "hearing-5397", + "hearing-5368", + "hearing-5330", + "hearing-5572", + "hearing-5541", + "hearing-5474" + ], + "relevantCount": 19 + }, + { + "id": "ht-002", + "category": "agenda-topic", + "query": "budget hearing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5594", + "hearing-5546", + "hearing-5100", + "hearing-5099", + "hearing-5098", + "hearing-5097", + "hearing-5096", + "hearing-5095", + "hearing-5090", + "hearing-5088" + ], + "relevantCount": 10 + }, + { + "id": "ht-003", + "category": "agenda-topic", + "query": "miscellaneous matters", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.46900009332067477 + }, + "top10": ["hearing-5461", "hearing-5210", "hearing-5390"], + "relevantCount": 21 + }, + { + "id": "ht-004", + "category": "agenda-topic", + "query": "disabilities", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.3499667779514209 + }, + "top10": [ + "hearing-5169", + "hearing-5728", + "hearing-5477", + "hearing-5432", + "hearing-5377", + "hearing-5353", + "hearing-5334", + "hearing-5214", + "hearing-5747", + "hearing-5708" + ], + "relevantCount": 30 + }, + { + "id": "ht-005", + "category": "agenda-topic", + "query": "public safety", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5704", + "hearing-5638", + "hearing-5604", + "hearing-5440", + "hearing-5409", + "hearing-5347", + "hearing-5205", + "hearing-5167", + "hearing-5471", + "hearing-5378" + ], + "relevantCount": 12 + }, + { + "id": "ht-006", + "category": "agenda-topic", + "query": "late files", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5744", + "hearing-5743", + "hearing-5727", + "hearing-5719", + "hearing-5637", + "hearing-5639", + "hearing-5627", + "hearing-5581", + "hearing-5508", + "hearing-5524" + ], + "relevantCount": 40 + }, + { + "id": "ht-007", + "category": "agenda-topic", + "query": "health care cost growth benchmark", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5091"], + "relevantCount": 1 + }, + { + "id": "ht-008", + "category": "agenda-topic", + "query": "climate", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9147029315959613 + }, + "top10": [ + "hearing-5156", + "hearing-5112", + "hearing-5690", + "hearing-5418", + "hearing-5344", + "hearing-5300", + "hearing-5220", + "hearing-5190" + ], + "relevantCount": 6 + }, + { + "id": "hl-001", + "category": "location", + "query": "Gardner Auditorium", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5688", + "hearing-5671", + "hearing-5594", + "hearing-5546", + "hearing-5515", + "hearing-5471", + "hearing-5474", + "hearing-5459", + "hearing-5458", + "hearing-5465" + ], + "relevantCount": 33 + }, + { + "id": "hl-002", + "category": "location", + "query": "Virtual Hearing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5215", + "hearing-5733", + "hearing-5714", + "hearing-5713", + "hearing-5716", + "hearing-5712", + "hearing-5709", + "hearing-5703", + "hearing-5699", + "hearing-5698" + ], + "relevantCount": 60 + }, + { + "id": "hl-003", + "category": "location", + "query": "Written Testimony Only", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5506", + "hearing-5455", + "hearing-5704", + "hearing-5750", + "hearing-5743", + "hearing-5748", + "hearing-5738", + "hearing-5737", + "hearing-5726", + "hearing-5722" + ], + "relevantCount": 33 + }, + { + "id": "hl-004", + "category": "location", + "query": "UMass Amherst", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5575", "hearing-5095"], + "relevantCount": 2 + }, + { + "id": "hl-005", + "category": "location", + "query": "Northampton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5672", "hearing-5747"], + "relevantCount": 2 + }, + { + "id": "hs-001", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["hearing-5530", "hearing-5235", "hearing-5165"], + "relevantCount": 1 + }, + { + "id": "hs-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 3 + }, + { + "id": "hs-003", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 8 + }, + { + "id": "hm-001", + "category": "misspelling", + "query": "judicary", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5639", + "hearing-5494", + "hearing-5478", + "hearing-5452", + "hearing-5434", + "hearing-5388", + "hearing-5358", + "hearing-5229", + "hearing-5227", + "hearing-5216" + ], + "relevantCount": 13 + }, + { + "id": "hm-002", + "category": "misspelling", + "query": "transportaton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5431", + "hearing-5407", + "hearing-5738", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5505", + "hearing-5454", + "hearing-5400", + "hearing-5345" + ], + "relevantCount": 14 + }, + { + "id": "hm-003", + "category": "misspelling", + "query": "gardner auditorum", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5688", + "hearing-5671", + "hearing-5594", + "hearing-5546", + "hearing-5515", + "hearing-5471", + "hearing-5474", + "hearing-5459", + "hearing-5458", + "hearing-5465" + ], + "relevantCount": 33 + }, + { + "id": "hm-004", + "category": "misspelling", + "query": "municipalites", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5741", + "hearing-5719", + "hearing-5684", + "hearing-5637", + "hearing-5572", + "hearing-5516", + "hearing-5503", + "hearing-5495", + "hearing-5462", + "hearing-5326" + ], + "relevantCount": 18 + }, + { + "id": "hm-005", + "category": "misspelling", + "query": "consumer protecton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5147", + "hearing-5395", + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5433", + "hearing-5416", + "hearing-5357" + ], + "relevantCount": 13 + }, + { + "id": "hm-006", + "category": "misspelling", + "query": "cindy freidman", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 15 + } + ] +} diff --git a/tests/search-eval/baselines/hearings-synonyms.json b/tests/search-eval/baselines/hearings-synonyms.json new file mode 100644 index 000000000..a03b6da91 --- /dev/null +++ b/tests/search-eval/baselines/hearings-synonyms.json @@ -0,0 +1,1013 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "hearings", + "collection": "hearings_eval", + "serverVersion": "30.2", + "corpusMd5": "fa6e9fe42e3da564da0bd72eef1e55aa", + "gitSha": "7e6529b622c9e1b2b45a53add525c355f96a1ba3", + "timestamp": "2026-08-19T18:32:18.836Z", + "sortBy": "_text_match:desc,startsAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.96875, + "mrr": 0.9739583333333334, + "ndcgAt10": 0.9571719707217369, + "queryCount": 48 + }, + "categories": { + "agenda-topic": { + "recallAt10": 0.825, + "mrr": 1, + "ndcgAt10": 0.8417087253585072, + "queryCount": 8 + }, + "bill-on-agenda": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 10 + }, + "chair": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 6 + }, + "committee": { + "recallAt10": 0.99, + "mrr": 0.95, + "ndcgAt10": 0.977990823370192, + "queryCount": 10 + }, + "location": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 5 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 1, + "mrr": 0.75, + "ndcgAt10": 0.8102255193577976, + "queryCount": 3 + } + }, + "queries": [ + { + "id": "hc-001", + "category": "committee", + "query": "Ways and Means", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5099", + "hearing-5594", + "hearing-5585", + "hearing-5577", + "hearing-5582", + "hearing-5584", + "hearing-5575", + "hearing-5583", + "hearing-5546", + "hearing-5100" + ], + "relevantCount": 19 + }, + { + "id": "hc-002", + "category": "committee", + "query": "Judiciary", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5639", + "hearing-5494", + "hearing-5478", + "hearing-5452", + "hearing-5434", + "hearing-5388", + "hearing-5358", + "hearing-5229", + "hearing-5227", + "hearing-5216" + ], + "relevantCount": 13 + }, + { + "id": "hc-003", + "category": "committee", + "query": "Financial Services", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5403", + "hearing-5721", + "hearing-5576", + "hearing-5524", + "hearing-5473", + "hearing-5457", + "hearing-5448", + "hearing-5439", + "hearing-5417", + "hearing-5374" + ], + "relevantCount": 21 + }, + { + "id": "hc-004", + "category": "committee", + "query": "Public Health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5532", + "hearing-5390", + "hearing-5239", + "hearing-5236", + "hearing-5222", + "hearing-5628", + "hearing-5360", + "hearing-5221", + "hearing-5211", + "hearing-5110" + ], + "relevantCount": 13 + }, + { + "id": "hc-005", + "category": "committee", + "query": "Transportation", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5431", + "hearing-5407", + "hearing-5738", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5505", + "hearing-5454", + "hearing-5400", + "hearing-5345" + ], + "relevantCount": 14 + }, + { + "id": "hc-006", + "category": "committee", + "query": "Housing", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019198 + }, + "top10": [ + "hearing-5366", + "hearing-5197", + "hearing-5750", + "hearing-5718", + "hearing-5526", + "hearing-5474", + "hearing-5485", + "hearing-5419", + "hearing-5367", + "hearing-5240" + ], + "relevantCount": 11 + }, + { + "id": "hc-007", + "category": "committee", + "query": "Election Laws", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5224", + "hearing-5113", + "hearing-5735", + "hearing-5627", + "hearing-5545", + "hearing-5508", + "hearing-5451", + "hearing-5426", + "hearing-5322", + "hearing-5151" + ], + "relevantCount": 10 + }, + { + "id": "hc-008", + "category": "committee", + "query": "Revenue", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5178", + "hearing-5748", + "hearing-5722", + "hearing-5687", + "hearing-5549", + "hearing-5535", + "hearing-5479", + "hearing-5458", + "hearing-5447", + "hearing-5427" + ], + "relevantCount": 19 + }, + { + "id": "hc-009", + "category": "committee", + "query": "Consumer Protection", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5147", + "hearing-5395", + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5433", + "hearing-5416", + "hearing-5357" + ], + "relevantCount": 13 + }, + { + "id": "hc-010", + "category": "committee", + "query": "Municipalities and Regional Government", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5684", + "hearing-5572", + "hearing-5212", + "hearing-5152", + "hearing-5741", + "hearing-5719", + "hearing-5637", + "hearing-5516", + "hearing-5503", + "hearing-5495" + ], + "relevantCount": 18 + }, + { + "id": "hch-001", + "category": "chair", + "query": "Rebecca Rausch", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5744", + "hearing-5741", + "hearing-5727", + "hearing-5719", + "hearing-5684", + "hearing-5637", + "hearing-5581", + "hearing-5579", + "hearing-5572", + "hearing-5238" + ], + "relevantCount": 15 + }, + { + "id": "hch-002", + "category": "chair", + "query": "Cindy Friedman", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 15 + }, + { + "id": "hch-003", + "category": "chair", + "query": "Alice Peisch", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 10 + }, + { + "id": "hch-004", + "category": "chair", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5237", + "hearing-5194", + "hearing-5171", + "hearing-5147" + ], + "relevantCount": 9 + }, + { + "id": "hch-005", + "category": "chair", + "query": "Brendan Crighton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5738", + "hearing-5734", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5226", + "hearing-5195", + "hearing-5174", + "hearing-5166" + ], + "relevantCount": 9 + }, + { + "id": "hch-006", + "category": "chair", + "query": "Daniel Ryan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5732", + "hearing-5701", + "hearing-5658", + "hearing-5586", + "hearing-5534", + "hearing-5206", + "hearing-5203", + "hearing-5192", + "hearing-5177", + "hearing-5163" + ], + "relevantCount": 10 + }, + { + "id": "hb-001", + "category": "bill-on-agenda", + "query": "H2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-002", + "category": "bill-on-agenda", + "query": "H 2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-003", + "category": "bill-on-agenda", + "query": "H.2391", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-004", + "category": "bill-on-agenda", + "query": "S1489", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-005", + "category": "bill-on-agenda", + "query": "S 1489", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "hb-006", + "category": "bill-on-agenda", + "query": "H63", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5113", + "hearing-5396", + "hearing-5362", + "hearing-5220", + "hearing-5189", + "hearing-5168", + "hearing-5141" + ], + "relevantCount": 1 + }, + { + "id": "hb-007", + "category": "bill-on-agenda", + "query": "H 63", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5113"], + "relevantCount": 1 + }, + { + "id": "hb-008", + "category": "bill-on-agenda", + "query": "S6", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5113", + "hearing-5475", + "hearing-5448", + "hearing-5442", + "hearing-5439", + "hearing-5190" + ], + "relevantCount": 1 + }, + { + "id": "hb-009", + "category": "bill-on-agenda", + "query": "H56", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5458", + "hearing-5326", + "hearing-5376", + "hearing-5459", + "hearing-5396", + "hearing-5362", + "hearing-5189", + "hearing-5168" + ], + "relevantCount": 3 + }, + { + "id": "hb-010", + "category": "bill-on-agenda", + "query": "H.2424", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5111"], + "relevantCount": 1 + }, + { + "id": "ht-001", + "category": "agenda-topic", + "query": "home rule petitions", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5516", + "hearing-5503", + "hearing-5462", + "hearing-5438", + "hearing-5397", + "hearing-5368", + "hearing-5330", + "hearing-5572", + "hearing-5541", + "hearing-5474" + ], + "relevantCount": 19 + }, + { + "id": "ht-002", + "category": "agenda-topic", + "query": "budget hearing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5594", + "hearing-5546", + "hearing-5100", + "hearing-5099", + "hearing-5098", + "hearing-5097", + "hearing-5096", + "hearing-5095", + "hearing-5090", + "hearing-5088" + ], + "relevantCount": 10 + }, + { + "id": "ht-003", + "category": "agenda-topic", + "query": "miscellaneous matters", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.46900009332067477 + }, + "top10": ["hearing-5461", "hearing-5210", "hearing-5390"], + "relevantCount": 21 + }, + { + "id": "ht-004", + "category": "agenda-topic", + "query": "disabilities", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.3499667779514209 + }, + "top10": [ + "hearing-5169", + "hearing-5728", + "hearing-5477", + "hearing-5432", + "hearing-5377", + "hearing-5353", + "hearing-5334", + "hearing-5214", + "hearing-5264", + "hearing-5234" + ], + "relevantCount": 30 + }, + { + "id": "ht-005", + "category": "agenda-topic", + "query": "public safety", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5704", + "hearing-5638", + "hearing-5604", + "hearing-5440", + "hearing-5409", + "hearing-5347", + "hearing-5205", + "hearing-5167", + "hearing-5471", + "hearing-5378" + ], + "relevantCount": 12 + }, + { + "id": "ht-006", + "category": "agenda-topic", + "query": "late files", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5744", + "hearing-5743", + "hearing-5727", + "hearing-5719", + "hearing-5637", + "hearing-5639", + "hearing-5627", + "hearing-5581", + "hearing-5508", + "hearing-5524" + ], + "relevantCount": 40 + }, + { + "id": "ht-007", + "category": "agenda-topic", + "query": "health care cost growth benchmark", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5091"], + "relevantCount": 1 + }, + { + "id": "ht-008", + "category": "agenda-topic", + "query": "climate", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9147029315959613 + }, + "top10": [ + "hearing-5156", + "hearing-5112", + "hearing-5690", + "hearing-5418", + "hearing-5344", + "hearing-5300", + "hearing-5220", + "hearing-5190" + ], + "relevantCount": 6 + }, + { + "id": "hl-001", + "category": "location", + "query": "Gardner Auditorium", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5688", + "hearing-5671", + "hearing-5594", + "hearing-5546", + "hearing-5515", + "hearing-5471", + "hearing-5474", + "hearing-5459", + "hearing-5458", + "hearing-5465" + ], + "relevantCount": 33 + }, + { + "id": "hl-002", + "category": "location", + "query": "Virtual Hearing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5215", + "hearing-5733", + "hearing-5714", + "hearing-5713", + "hearing-5716", + "hearing-5712", + "hearing-5709", + "hearing-5703", + "hearing-5699", + "hearing-5698" + ], + "relevantCount": 60 + }, + { + "id": "hl-003", + "category": "location", + "query": "Written Testimony Only", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5506", + "hearing-5455", + "hearing-5704", + "hearing-5750", + "hearing-5743", + "hearing-5748", + "hearing-5738", + "hearing-5737", + "hearing-5726", + "hearing-5722" + ], + "relevantCount": 33 + }, + { + "id": "hl-004", + "category": "location", + "query": "UMass Amherst", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5575", "hearing-5095"], + "relevantCount": 2 + }, + { + "id": "hl-005", + "category": "location", + "query": "Northampton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5672", "hearing-5747"], + "relevantCount": 2 + }, + { + "id": "hs-001", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 0.25, + "ndcgAt10": 0.43067655807339306 + }, + "top10": ["hearing-5530", "hearing-5235", "hearing-5165", "hearing-5602"], + "relevantCount": 1 + }, + { + "id": "hs-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["hearing-5440", "hearing-5321", "hearing-5223"], + "relevantCount": 3 + }, + { + "id": "hs-003", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5705", + "hearing-5692", + "hearing-5630", + "hearing-5433", + "hearing-5395", + "hearing-5237", + "hearing-5171", + "hearing-5416" + ], + "relevantCount": 8 + }, + { + "id": "hm-001", + "category": "misspelling", + "query": "judicary", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5639", + "hearing-5494", + "hearing-5478", + "hearing-5452", + "hearing-5434", + "hearing-5388", + "hearing-5358", + "hearing-5229", + "hearing-5227", + "hearing-5216" + ], + "relevantCount": 13 + }, + { + "id": "hm-002", + "category": "misspelling", + "query": "transportaton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5431", + "hearing-5407", + "hearing-5738", + "hearing-5694", + "hearing-5634", + "hearing-5573", + "hearing-5505", + "hearing-5454", + "hearing-5400", + "hearing-5345" + ], + "relevantCount": 14 + }, + { + "id": "hm-003", + "category": "misspelling", + "query": "gardner auditorum", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5688", + "hearing-5671", + "hearing-5594", + "hearing-5546", + "hearing-5515", + "hearing-5471", + "hearing-5474", + "hearing-5459", + "hearing-5458", + "hearing-5465" + ], + "relevantCount": 33 + }, + { + "id": "hm-004", + "category": "misspelling", + "query": "municipalites", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5741", + "hearing-5719", + "hearing-5684", + "hearing-5637", + "hearing-5572", + "hearing-5516", + "hearing-5503", + "hearing-5495", + "hearing-5462", + "hearing-5326" + ], + "relevantCount": 18 + }, + { + "id": "hm-005", + "category": "misspelling", + "query": "consumer protecton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5147", + "hearing-5395", + "hearing-5720", + "hearing-5692", + "hearing-5656", + "hearing-5630", + "hearing-5541", + "hearing-5433", + "hearing-5416", + "hearing-5357" + ], + "relevantCount": 13 + }, + { + "id": "hm-006", + "category": "misspelling", + "query": "cindy freidman", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "hearing-5632", + "hearing-5603", + "hearing-5631", + "hearing-5602", + "hearing-5601", + "hearing-5598", + "hearing-5589", + "hearing-5597", + "hearing-5595", + "hearing-5590" + ], + "relevantCount": 15 + } + ] +} diff --git a/tests/search-eval/baselines/main-typesense-0.24.json b/tests/search-eval/baselines/main-typesense-0.24.json new file mode 100644 index 000000000..678d56d42 --- /dev/null +++ b/tests/search-eval/baselines/main-typesense-0.24.json @@ -0,0 +1,1474 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "0.24.0", + "corpusMd5": "a0499d7bb8597f88dbd737893bcd8a4d", + "gitSha": "6495fdbb614c9de0c04188cc4ddda29bb92556d8", + "timestamp": "2026-08-18T18:34:49.611Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.6514285714285716, + "mrr": 0.7673015873015874, + "ndcgAt10": 0.5251301729907821, + "queryCount": 70 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 0.8, + "mrr": 0.8, + "ndcgAt10": 0.8, + "queryCount": 15 + }, + "member-committee": { + "recallAt10": 0.09333333333333334, + "mrr": 0.2733333333333333, + "ndcgAt10": 0.11884984315846131, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8699999999999999, + "mrr": 1, + "ndcgAt10": 0.4985867229196039, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.7899999999999999, + "mrr": 0.9, + "ndcgAt10": 0.8262463153675897, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.78, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.4864017039552942, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S904", + "192-S9", + "192-S898", + "192-S873", + "192-S805", + "192-S804", + "192-S782", + "192-S771", + "192-S766", + "192-S675" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S995", + "192-S971", + "192-S955", + "192-S951", + "192-S922", + "192-S913", + "192-S911", + "192-S91", + "192-S905" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S995", + "192-S971", + "192-S955", + "192-S951", + "192-S922", + "192-S913", + "192-S911", + "192-S91", + "192-S905" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4352"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2170", "192-S2158", "192-H3372", "192-H3302"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["192-S1698"], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4316", "192-S2018", "192-H3166"], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S1"], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S733", + "192-S571", + "192-S2737", + "192-S2523", + "192-S1796", + "192-S143", + "192-H877", + "192-H4433", + "192-H2908", + "192-H2833" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S44", "192-H3765"], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H2756"], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["192-S15", "192-S2580"], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4341", "192-S7", "192-S2544", "192-H3476"], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2544", "192-H3476"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H629", + "192-H3415", + "192-H1968", + "192-S82", + "192-S785", + "192-S771", + "192-S364", + "192-S2626" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8415908474314976 + }, + "top10": [ + "192-S798", + "192-S791", + "192-S787", + "192-S785", + "192-S782", + "192-S779", + "192-S770", + "192-S758", + "192-S743", + "192-S675" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2743", + "192-S2070", + "192-S1866", + "192-S1731", + "192-S1730", + "192-H4518", + "192-H4513", + "192-H4429", + "192-H4428", + "192-H3803" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.43735247915031 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S846", + "192-S387", + "192-S381", + "192-S378", + "192-S365" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.1, + "mrr": 0.1, + "ndcgAt10": 0.06362078819895171 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S568", + "192-S566", + "192-S2229", + "192-S2222" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.12111401772315426 + }, + "top10": [ + "192-H2527", + "192-S942", + "192-H1734", + "192-S939", + "192-S514", + "192-S1130", + "192-S1009", + "192-H43", + "192-H308", + "192-H2635" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S703", + "192-S675", + "192-S636", + "192-S292", + "192-S2790", + "192-S2584", + "192-S2572", + "192-S2528", + "192-S2514" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S868", + "192-S867", + "192-S2025", + "192-S1985", + "192-S1853", + "192-S1808" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2174", + "192-S2168", + "192-S2145", + "192-S1896", + "192-S1326", + "192-H4044", + "192-H3346" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3330", + "192-H3269", + "192-H2159", + "192-H2145", + "192-S2157", + "192-H3964" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2762", + "192-S1568", + "192-S1563", + "192-S1561", + "192-H43", + "192-H3847", + "192-H3846", + "192-H2635", + "192-H2533", + "192-H2505" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8643145546088337 + }, + "top10": [ + "192-H2470", + "192-S1540", + "192-H2874", + "192-H2491", + "192-H2488", + "192-H2439", + "192-S942", + "192-S939", + "192-S1615", + "192-S1562" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8701249883466593 + }, + "top10": [ + "192-S366", + "192-S2748", + "192-H682", + "192-H642", + "192-H127", + "192-S285", + "192-H675", + "192-H1620", + "192-S384", + "192-S377" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-H628", + "192-H594", + "192-H2759", + "192-H2688", + "192-H2655", + "192-H2620" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S77", + "192-S69", + "192-S63", + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-H2056" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2600", + "192-S2079", + "192-S1458", + "192-S1436", + "192-S1420", + "192-S1399", + "192-S1134", + "192-S1009", + "192-H4510", + "192-H437" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.800693766409882 + }, + "top10": [ + "192-H723", + "192-S1961", + "192-H734", + "192-H732", + "192-H3734", + "192-H2903", + "192-H2947", + "192-S766", + "192-S252", + "192-H500" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2215", + "192-H353", + "192-S2566", + "192-S2477", + "192-S246", + "192-S228", + "192-S227", + "192-S199", + "192-H473", + "192-H472" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H2884", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S703", + "192-S675", + "192-S636", + "192-S292", + "192-S2790", + "192-S2584", + "192-S2572", + "192-S2528", + "192-S2514" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S868", + "192-S867", + "192-S2025", + "192-S1985", + "192-S1853", + "192-S1808" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6104531109861133 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071", + "192-H2068" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/testimony-base.json b/tests/search-eval/baselines/testimony-base.json new file mode 100644 index 000000000..dabb5b1c0 --- /dev/null +++ b/tests/search-eval/baselines/testimony-base.json @@ -0,0 +1,888 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "publishedTestimony", + "collection": "publishedTestimony_eval", + "serverVersion": "30.2", + "corpusMd5": "bd1aa76b61b9803ea9ca90827539d688", + "gitSha": "2e037d0c8b1a61d3fe09b8c37a9de94d9c4a076f", + "timestamp": "2026-08-19T18:30:11.594Z", + "sortBy": "_text_match:desc,publishedAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8658119658119657, + "mrr": 0.9102564102564102, + "ndcgAt10": 0.8556474089088215, + "queryCount": 39 + }, + "categories": { + "author": { + "recallAt10": 0.9833333333333334, + "mrr": 1, + "ndcgAt10": 0.9816590194751601, + "queryCount": 6 + }, + "bill-id": { + "recallAt10": 0.8125, + "mrr": 1, + "ndcgAt10": 0.8479850279985381, + "queryCount": 8 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9429187326594431, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.6523809523809524, + "mrr": 0.7142857142857143, + "ndcgAt10": 0.6730753520197963, + "queryCount": 7 + }, + "topic": { + "recallAt10": 0.9, + "mrr": 0.875, + "ndcgAt10": 0.8606145622091282, + "queryCount": 12 + } + }, + "queries": [ + { + "id": "tb-001", + "category": "bill-id", + "query": "S531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-002", + "category": "bill-id", + "query": "S 531", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.5637884576902256 + }, + "top10": [ + "1fSJjeB4F2hKx7xktRO00", + "O8Q64gYX42olvz7w9NczZ", + "0wpvjt7NwI_wHGE47nKbC", + "j95dKA76AwuBFdXhKz1Vu" + ], + "relevantCount": 108 + }, + { + "id": "tb-003", + "category": "bill-id", + "query": "S.531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-004", + "category": "bill-id", + "query": "H4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG", + "5V9bzQe2iZNmTALY2Quso", + "d2E2LxE3r03TgpgjMG5NX", + "9-awIt_tkxdiaPXPhQBB7" + ], + "relevantCount": 28 + }, + { + "id": "tb-005", + "category": "bill-id", + "query": "H 4351", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["sSa8ZLDXwVOR5PRBLzOF0"], + "relevantCount": 28 + }, + { + "id": "tb-006", + "category": "bill-id", + "query": "S505", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "q7FABYQzOvfmjegGbYa0-", + "Kjme82E2J3xh_iavVpoyc", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "g_CMBoU8VJyXAB_0n_Hx5", + "65VMrIqujcAENO9dZP5Gg", + "n214rkLkVmTQngHYGShYa", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz" + ], + "relevantCount": 7 + }, + { + "id": "tb-007", + "category": "bill-id", + "query": "H2057", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "p-Fj1pOglGAmu18SEZVis", + "qA3IwUwvx5MxvFbqGc7sd", + "uXdD5bXYgaRAla6i9UAxz", + "1iLkOnpKnIyMPZ0EJ6q-8", + "zQ0790mAoAXDi3wH5EysT", + "byQC1IHIabnkVGSLxeQ46" + ], + "relevantCount": 6 + }, + { + "id": "tb-008", + "category": "bill-id", + "query": "H.1405", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Dc6RugqP0HRxcujz5pLDF", + "Fhw90ohV0b4d6-_oJjixg", + "c7CaVvPRtM6WP-4puuIL9", + "nthww4idwGAivhUavH1KO", + "4UdhIwSdZQKYBsulWHTie", + "uiCjZ0jMjwTOnF8hRKFNH", + "L_gM_T02MsULa-W3BXldU", + "biQVm39ND9kHg9rGlmphQ", + "QfqfGXsYn0K4Iu_38SGF_" + ], + "relevantCount": 6 + }, + { + "id": "ta-001", + "category": "author", + "query": "Progressive Massachusetts", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "ta-002", + "category": "author", + "query": "League of Women Voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "ta-003", + "category": "author", + "query": "T Stephen Jones", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "9-ITBjvC_UUWw-8HhQa9k", + "-01jqkUE2o1lPjijGhV3V", + "0ANKQpAgWF-1sSLPY7aq_", + "7erqVBGptvNi_KUs-EOp2" + ], + "relevantCount": 51 + }, + { + "id": "ta-004", + "category": "author", + "query": "Elaine Almquist", + "metrics": { + "recallAt10": 0.9, + "mrr": 1, + "ndcgAt10": 0.8899541168509599 + }, + "top10": [ + "qkCngGUEqPpW56990vrXS", + "zCw3UiFfn7NpEADJF4pFE", + "7yrt6_JiCsG7N34SdUssW", + "DXt9s5U1LgPRcoT3iaAtb", + "Dc6RugqP0HRxcujz5pLDF", + "Ns08eB38sg4fXiUVyPIS-", + "izBjYVA4sX_hv2LqJNQ7G", + "-sYilJcb7FYgM-xPApD-E", + "JwZRV5xz-g7VAI7AbHSFS", + "ML-T7WWVi7L4Moz961IyO" + ], + "relevantCount": 23 + }, + { + "id": "ta-005", + "category": "author", + "query": "Partners in Democracy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1lOiHvfe3gdqKkfVyUX5P", + "rxS5QAxVif3dC-cLVF7Bb", + "n0cd5-Ga0RBro4CUrGwtZ", + "LiqNtrg-sc03SCuN6Z0py", + "Pw4c_r2EUikJuKCxrUqah", + "NYPkatkLuretCPsXulkdI", + "HEQ9nYz14M8UWESrjHJmo", + "HhPnj15sNI5XTk_G2DIp4", + "hkrlPHtg0Iz-k5bii2aLr", + "IayRrND2zy-f2XSd9cjJS" + ], + "relevantCount": 20 + }, + { + "id": "ta-006", + "category": "author", + "query": "Commission on the Status of Women", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "TvuAMRkXLh2xY7zFkUKzt", + "Y5DsjTHY-u7Mbf4L7B6JS", + "fR53JFa8xQfPTsWmaTzy5", + "Fhw90ohV0b4d6-_oJjixg", + "NFxoUtF2UXQrkCao7y1mg", + "Nc59-_Ck2zDzW7N7LoA8L", + "A-9hxql_cZ8vl_duOB8v8", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0" + ], + "relevantCount": 6 + }, + { + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tt-002", + "category": "topic", + "query": "rent control", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "-l-iQxshYz8DF4nrW0dy4", + "FeBsBGdvWUVRQIKV6pfsF", + "Y5DsjTHY-u7Mbf4L7B6JS" + ], + "relevantCount": 6 + }, + { + "id": "tt-003", + "category": "topic", + "query": "same day voter registration", + "metrics": { + "recallAt10": 0.8, + "mrr": 0.5, + "ndcgAt10": 0.6698623505528797 + }, + "top10": [ + "n214rkLkVmTQngHYGShYa", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "65VMrIqujcAENO9dZP5Gg", + "kWt58RStbQMUB7VAqXNCV", + "g_CMBoU8VJyXAB_0n_Hx5", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz", + "c4TkesyGcrZWBcn07LsTu" + ], + "relevantCount": 14 + }, + { + "id": "tt-004", + "category": "topic", + "query": "vote by mail", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "U7M6KOrOk3HxXp3HMtxga", + "wUCZhFjmgD-gQf1rwlZBO", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0", + "s7YU0Lr8siUL6DZsczrVe", + "rZOoRP_eVkKCsB1EQ64tR", + "ZFMH6QiRVy_ay0TfHLu_u", + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "0f-XBB35ZN3aY4qYD3aIq" + ], + "relevantCount": 3 + }, + { + "id": "tt-005", + "category": "topic", + "query": "solitary confinement", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + }, + { + "id": "tt-006", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tt-007", + "category": "topic", + "query": "zoning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tt-008", + "category": "topic", + "query": "fossil fuels", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "l1I8kGowVtz2WPrYovaPV", + "GDrFCpA4hAdsIw_Rm5X3X", + "-glcyOYtYYPv-IQCOLTu4", + "dUExko-64U-IHhp8lyPri" + ], + "relevantCount": 14 + }, + { + "id": "tt-009", + "category": "topic", + "query": "gender affirming care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Fhw90ohV0b4d6-_oJjixg", + "QddFVr819ct-AL2R1mpSt", + "mC9cec9DPZmJh-h0k59pL", + "YoMcbg3flFzl2ZuNDBsG4", + "qj9B-cAH0m6Zc1tenFcSN", + "7C300VIQg4-XGPdTM9dTq" + ], + "relevantCount": 6 + }, + { + "id": "tt-010", + "category": "topic", + "query": "language access", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "JwZRV5xz-g7VAI7AbHSFS", + "wJk41fu1FiVBJ1xC8GFBE", + "VwGDFqblM0NpNdygg5jj3", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "YZR3tRTH_AxEbmjFrQ1vc", + "gtnHu_C21R-d1YXD04drW", + "Bs8eththvqbwf63jWoIyz", + "WnfgfrLKQDAiUHcNswyT5" + ], + "relevantCount": 6 + }, + { + "id": "tt-011", + "category": "topic", + "query": "parole", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "G2hQLzmZ5GVbZw0IVaWmT", + "B0ZfwvCJxhgQPhi1qUWBE", + "HWYONHILNhIg1YogNtBis", + "tH5mbM__T0n2uCu5qLdPZ", + "xWyIDifmJf0c2Qqz08Wu5", + "SCSHutdvYlIgGvCAMLGop", + "evQA35IoJkYpR0X4mNbcq", + "1e-dFpB_PC6uL_lXhALKT", + "JWXz3tBMSthD1hbfJlC7a", + "VyQaykWaAosNzTi5ZMXVF" + ], + "relevantCount": 21 + }, + { + "id": "tt-012", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "-01jqkUE2o1lPjijGhV3V", + "jyGFy24gK9DJU4A5nb_34", + "tzuzCzguXwJXggnIRpjH8", + "dJZJPWdoBPU9DonfEUVta" + ], + "relevantCount": 25 + }, + { + "id": "ts-001", + "category": "synonym", + "query": "renters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "lsZSRKcHwgsW_scJjO480", + "Kjme82E2J3xh_iavVpoyc", + "Bu6Tz_JnL6Qjfyd-pY_-W", + "AHXLF18qIoNN9O75snH7F", + "jO8-5-ckBJKvogjIleQYa", + "-l-iQxshYz8DF4nrW0dy4", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "1lOiHvfe3gdqKkfVyUX5P" + ], + "relevantCount": 35 + }, + { + "id": "ts-002", + "category": "synonym", + "query": "tenants", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F" + ], + "relevantCount": 35 + }, + { + "id": "ts-003", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "751FZTubP1RtXBZqhCbA9", + "C8n-9BfBnF0oODvYkH7BW", + "t_edzv3GrkA1_XCF0Ri1-", + "jycTZ6dvmsbDtAXhY_52F", + "j8eAKtpVbb9Vd05l6TIiX", + "2aDdN2OCct1-sOQsBogkd", + "JIIYdlaFK2FxxjDA7wg4c", + "PqLuEfw8zCdTdaEAumWvT", + "Mu2UQm0U3Dr54IOFp8Ldj", + "wyrhNVcf3Z4G4JLJWrjUh" + ], + "relevantCount": 26 + }, + { + "id": "ts-004", + "category": "synonym", + "query": "doctor", + "metrics": { + "recallAt10": 0.9, + "mrr": 1, + "ndcgAt10": 0.9363792118010483 + }, + "top10": [ + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi", + "klgcdRRi8wbTeCV7VZRYE", + "YZR3tRTH_AxEbmjFrQ1vc", + "gtnHu_C21R-d1YXD04drW", + "ks5e6tOhqWkvobyCnqqki", + "jO8-5-ckBJKvogjIleQYa", + "fR53JFa8xQfPTsWmaTzy5", + "CwGMam-vtxDaat2fEROIM" + ], + "relevantCount": 42 + }, + { + "id": "ts-005", + "category": "synonym", + "query": "servicemember", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 16 + }, + { + "id": "ts-006", + "category": "synonym", + "query": "opiate", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 6 + }, + { + "id": "ts-007", + "category": "synonym", + "query": "clean energy", + "metrics": { + "recallAt10": 0.6666666666666666, + "mrr": 1, + "ndcgAt10": 0.7751482523375255 + }, + "top10": [ + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "of8VK0lewRoYWyDgfNykB", + "dTVPgcuqJZgIhFobes1v-", + "LrWaFn2nhuFwxIPk-GuKj" + ], + "relevantCount": 6 + }, + { + "id": "tm-001", + "category": "misspelling", + "query": "progresive massachusets", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "tm-002", + "category": "misspelling", + "query": "leage of women voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "tm-003", + "category": "misspelling", + "query": "rankd choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tm-004", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tm-005", + "category": "misspelling", + "query": "zonning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tm-006", + "category": "misspelling", + "query": "solitary confinment", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/testimony-number-variants.json b/tests/search-eval/baselines/testimony-number-variants.json new file mode 100644 index 000000000..59102ab6b --- /dev/null +++ b/tests/search-eval/baselines/testimony-number-variants.json @@ -0,0 +1,922 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "publishedTestimony", + "collection": "publishedTestimony_eval", + "serverVersion": "30.2", + "corpusMd5": "2f6c3a70929df36f4ffa373a44486cf7", + "gitSha": "c4a843c12cd9c72061ddae203bff02aa4ae2db18", + "timestamp": "2026-08-19T20:55:33.624Z", + "sortBy": "_text_match:desc,publishedAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.9632478632478632, + "mrr": 0.9743589743589743, + "ndcgAt10": 0.9482083887462002, + "queryCount": 39 + }, + "categories": { + "author": { + "recallAt10": 0.9833333333333334, + "mrr": 1, + "ndcgAt10": 0.9816590194751601, + "queryCount": 6 + }, + "bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 8 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9429187326594431, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.9523809523809524, + "mrr": 1, + "ndcgAt10": 0.9678783217625037, + "queryCount": 7 + }, + "topic": { + "recallAt10": 0.9166666666666666, + "mrr": 0.9166666666666666, + "ndcgAt10": 0.8881260329963881, + "queryCount": 12 + } + }, + "queries": [ + { + "id": "tb-001", + "category": "bill-id", + "query": "S531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-002", + "category": "bill-id", + "query": "S 531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1fSJjeB4F2hKx7xktRO00", + "O8Q64gYX42olvz7w9NczZ", + "0wpvjt7NwI_wHGE47nKbC", + "j95dKA76AwuBFdXhKz1Vu", + "bsp6UU1XycOMBjsL5B6oK", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw" + ], + "relevantCount": 108 + }, + { + "id": "tb-003", + "category": "bill-id", + "query": "S.531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-004", + "category": "bill-id", + "query": "H4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG", + "5V9bzQe2iZNmTALY2Quso", + "d2E2LxE3r03TgpgjMG5NX", + "9-awIt_tkxdiaPXPhQBB7" + ], + "relevantCount": 28 + }, + { + "id": "tb-005", + "category": "bill-id", + "query": "H 4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9-awIt_tkxdiaPXPhQBB7", + "sSa8ZLDXwVOR5PRBLzOF0", + "IN1SeVROwhkoyerBYC4fo", + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG" + ], + "relevantCount": 28 + }, + { + "id": "tb-006", + "category": "bill-id", + "query": "S505", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "q7FABYQzOvfmjegGbYa0-", + "Kjme82E2J3xh_iavVpoyc", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "g_CMBoU8VJyXAB_0n_Hx5", + "65VMrIqujcAENO9dZP5Gg", + "n214rkLkVmTQngHYGShYa", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz" + ], + "relevantCount": 7 + }, + { + "id": "tb-007", + "category": "bill-id", + "query": "H2057", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "p-Fj1pOglGAmu18SEZVis", + "qA3IwUwvx5MxvFbqGc7sd", + "uXdD5bXYgaRAla6i9UAxz", + "1iLkOnpKnIyMPZ0EJ6q-8", + "zQ0790mAoAXDi3wH5EysT", + "byQC1IHIabnkVGSLxeQ46" + ], + "relevantCount": 6 + }, + { + "id": "tb-008", + "category": "bill-id", + "query": "H.1405", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Dc6RugqP0HRxcujz5pLDF", + "Fhw90ohV0b4d6-_oJjixg", + "c7CaVvPRtM6WP-4puuIL9", + "nthww4idwGAivhUavH1KO", + "4UdhIwSdZQKYBsulWHTie", + "uiCjZ0jMjwTOnF8hRKFNH", + "L_gM_T02MsULa-W3BXldU", + "biQVm39ND9kHg9rGlmphQ", + "QfqfGXsYn0K4Iu_38SGF_" + ], + "relevantCount": 6 + }, + { + "id": "ta-001", + "category": "author", + "query": "Progressive Massachusetts", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "ta-002", + "category": "author", + "query": "League of Women Voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "ta-003", + "category": "author", + "query": "T Stephen Jones", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "9-ITBjvC_UUWw-8HhQa9k", + "-01jqkUE2o1lPjijGhV3V", + "0ANKQpAgWF-1sSLPY7aq_", + "7erqVBGptvNi_KUs-EOp2" + ], + "relevantCount": 51 + }, + { + "id": "ta-004", + "category": "author", + "query": "Elaine Almquist", + "metrics": { + "recallAt10": 0.9, + "mrr": 1, + "ndcgAt10": 0.8899541168509599 + }, + "top10": [ + "qkCngGUEqPpW56990vrXS", + "zCw3UiFfn7NpEADJF4pFE", + "7yrt6_JiCsG7N34SdUssW", + "DXt9s5U1LgPRcoT3iaAtb", + "Dc6RugqP0HRxcujz5pLDF", + "Ns08eB38sg4fXiUVyPIS-", + "izBjYVA4sX_hv2LqJNQ7G", + "-sYilJcb7FYgM-xPApD-E", + "JwZRV5xz-g7VAI7AbHSFS", + "ML-T7WWVi7L4Moz961IyO" + ], + "relevantCount": 23 + }, + { + "id": "ta-005", + "category": "author", + "query": "Partners in Democracy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1lOiHvfe3gdqKkfVyUX5P", + "rxS5QAxVif3dC-cLVF7Bb", + "n0cd5-Ga0RBro4CUrGwtZ", + "LiqNtrg-sc03SCuN6Z0py", + "Pw4c_r2EUikJuKCxrUqah", + "NYPkatkLuretCPsXulkdI", + "HEQ9nYz14M8UWESrjHJmo", + "HhPnj15sNI5XTk_G2DIp4", + "hkrlPHtg0Iz-k5bii2aLr", + "IayRrND2zy-f2XSd9cjJS" + ], + "relevantCount": 20 + }, + { + "id": "ta-006", + "category": "author", + "query": "Commission on the Status of Women", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "TvuAMRkXLh2xY7zFkUKzt", + "Y5DsjTHY-u7Mbf4L7B6JS", + "fR53JFa8xQfPTsWmaTzy5", + "Fhw90ohV0b4d6-_oJjixg", + "NFxoUtF2UXQrkCao7y1mg", + "Nc59-_Ck2zDzW7N7LoA8L", + "A-9hxql_cZ8vl_duOB8v8", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0" + ], + "relevantCount": 6 + }, + { + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tt-002", + "category": "topic", + "query": "rent control", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "-l-iQxshYz8DF4nrW0dy4", + "FeBsBGdvWUVRQIKV6pfsF", + "Y5DsjTHY-u7Mbf4L7B6JS" + ], + "relevantCount": 6 + }, + { + "id": "tt-003", + "category": "topic", + "query": "same day voter registration", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "n214rkLkVmTQngHYGShYa", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "65VMrIqujcAENO9dZP5Gg", + "kWt58RStbQMUB7VAqXNCV", + "g_CMBoU8VJyXAB_0n_Hx5", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz", + "c4TkesyGcrZWBcn07LsTu" + ], + "relevantCount": 18 + }, + { + "id": "tt-004", + "category": "topic", + "query": "vote by mail", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "U7M6KOrOk3HxXp3HMtxga", + "wUCZhFjmgD-gQf1rwlZBO", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0", + "s7YU0Lr8siUL6DZsczrVe", + "rZOoRP_eVkKCsB1EQ64tR", + "ZFMH6QiRVy_ay0TfHLu_u", + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "0f-XBB35ZN3aY4qYD3aIq" + ], + "relevantCount": 3 + }, + { + "id": "tt-005", + "category": "topic", + "query": "solitary confinement", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + }, + { + "id": "tt-006", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tt-007", + "category": "topic", + "query": "zoning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tt-008", + "category": "topic", + "query": "fossil fuels", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "l1I8kGowVtz2WPrYovaPV", + "GDrFCpA4hAdsIw_Rm5X3X", + "-glcyOYtYYPv-IQCOLTu4", + "dUExko-64U-IHhp8lyPri" + ], + "relevantCount": 14 + }, + { + "id": "tt-009", + "category": "topic", + "query": "gender affirming care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Fhw90ohV0b4d6-_oJjixg", + "QddFVr819ct-AL2R1mpSt", + "mC9cec9DPZmJh-h0k59pL", + "YoMcbg3flFzl2ZuNDBsG4", + "qj9B-cAH0m6Zc1tenFcSN", + "7C300VIQg4-XGPdTM9dTq" + ], + "relevantCount": 6 + }, + { + "id": "tt-010", + "category": "topic", + "query": "language access", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "JwZRV5xz-g7VAI7AbHSFS", + "wJk41fu1FiVBJ1xC8GFBE", + "VwGDFqblM0NpNdygg5jj3", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "YZR3tRTH_AxEbmjFrQ1vc", + "gtnHu_C21R-d1YXD04drW", + "Bs8eththvqbwf63jWoIyz", + "WnfgfrLKQDAiUHcNswyT5" + ], + "relevantCount": 6 + }, + { + "id": "tt-011", + "category": "topic", + "query": "parole", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "G2hQLzmZ5GVbZw0IVaWmT", + "B0ZfwvCJxhgQPhi1qUWBE", + "HWYONHILNhIg1YogNtBis", + "tH5mbM__T0n2uCu5qLdPZ", + "xWyIDifmJf0c2Qqz08Wu5", + "SCSHutdvYlIgGvCAMLGop", + "evQA35IoJkYpR0X4mNbcq", + "1e-dFpB_PC6uL_lXhALKT", + "JWXz3tBMSthD1hbfJlC7a", + "VyQaykWaAosNzTi5ZMXVF" + ], + "relevantCount": 21 + }, + { + "id": "tt-012", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "-01jqkUE2o1lPjijGhV3V", + "jyGFy24gK9DJU4A5nb_34", + "tzuzCzguXwJXggnIRpjH8", + "dJZJPWdoBPU9DonfEUVta" + ], + "relevantCount": 25 + }, + { + "id": "ts-001", + "category": "synonym", + "query": "renters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-002", + "category": "synonym", + "query": "tenants", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-003", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "751FZTubP1RtXBZqhCbA9", + "L-3FT2B_7oofbiiOiRexN", + "3FtHbKJu9pAUcAttyGHvN", + "C8n-9BfBnF0oODvYkH7BW", + "Ri_EnpBWxNRBNfub-Jdn8", + "jbagjzjw6Wh1ciFuVSb_S", + "KWMM39oYqore5zm4A384g" + ], + "relevantCount": 26 + }, + { + "id": "ts-004", + "category": "synonym", + "query": "doctor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi", + "klgcdRRi8wbTeCV7VZRYE", + "-01jqkUE2o1lPjijGhV3V" + ], + "relevantCount": 42 + }, + { + "id": "ts-005", + "category": "synonym", + "query": "servicemember", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "L-3FT2B_7oofbiiOiRexN", + "4lMlyTqg1oh8lE4VDSHjs", + "9-awIt_tkxdiaPXPhQBB7", + "3FtHbKJu9pAUcAttyGHvN", + "Y1OMEyu2Tp1GkKMwLk3Se", + "dGmpznN-Ug5JADMuKFcC8", + "JFgq7bV_BWj0QQIGN2qKx", + "V2WhRLPdA4LHRYSki_wBL" + ], + "relevantCount": 16 + }, + { + "id": "ts-006", + "category": "synonym", + "query": "opiate", + "metrics": { + "recallAt10": 0.6666666666666666, + "mrr": 1, + "ndcgAt10": 0.7751482523375254 + }, + "top10": [ + "SBcIo8CsixaeEhBTYBR6W", + "stwHXrRVbe9vJYIDzI6CW", + "j8eAKtpVbb9Vd05l6TIiX", + "2aDdN2OCct1-sOQsBogkd" + ], + "relevantCount": 6 + }, + { + "id": "ts-007", + "category": "synonym", + "query": "clean energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "GDrFCpA4hAdsIw_Rm5X3X", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "of8VK0lewRoYWyDgfNykB" + ], + "relevantCount": 6 + }, + { + "id": "tm-001", + "category": "misspelling", + "query": "progresive massachusets", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "tm-002", + "category": "misspelling", + "query": "leage of women voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "tm-003", + "category": "misspelling", + "query": "rankd choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tm-004", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tm-005", + "category": "misspelling", + "query": "zonning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tm-006", + "category": "misspelling", + "query": "solitary confinment", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/testimony-synonyms.json b/tests/search-eval/baselines/testimony-synonyms.json new file mode 100644 index 000000000..80c3b774b --- /dev/null +++ b/tests/search-eval/baselines/testimony-synonyms.json @@ -0,0 +1,905 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "publishedTestimony", + "collection": "publishedTestimony_eval", + "serverVersion": "30.2", + "corpusMd5": "bd1aa76b61b9803ea9ca90827539d688", + "gitSha": "7e6529b622c9e1b2b45a53add525c355f96a1ba3", + "timestamp": "2026-08-19T18:34:13.783Z", + "sortBy": "_text_match:desc,publishedAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.9196581196581198, + "mrr": 0.9615384615384616, + "ndcgAt10": 0.9085607624523845, + "queryCount": 39 + }, + "categories": { + "author": { + "recallAt10": 0.9833333333333334, + "mrr": 1, + "ndcgAt10": 0.9816590194751601, + "queryCount": 6 + }, + "bill-id": { + "recallAt10": 0.8125, + "mrr": 1, + "ndcgAt10": 0.8479850279985381, + "queryCount": 8 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9429187326594431, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 0.9523809523809524, + "mrr": 1, + "ndcgAt10": 0.9678783217625037, + "queryCount": 7 + }, + "topic": { + "recallAt10": 0.9, + "mrr": 0.875, + "ndcgAt10": 0.8606145622091282, + "queryCount": 12 + } + }, + "queries": [ + { + "id": "tb-001", + "category": "bill-id", + "query": "S531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-002", + "category": "bill-id", + "query": "S 531", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.5637884576902256 + }, + "top10": [ + "1fSJjeB4F2hKx7xktRO00", + "O8Q64gYX42olvz7w9NczZ", + "0wpvjt7NwI_wHGE47nKbC", + "j95dKA76AwuBFdXhKz1Vu" + ], + "relevantCount": 108 + }, + { + "id": "tb-003", + "category": "bill-id", + "query": "S.531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-004", + "category": "bill-id", + "query": "H4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG", + "5V9bzQe2iZNmTALY2Quso", + "d2E2LxE3r03TgpgjMG5NX", + "9-awIt_tkxdiaPXPhQBB7" + ], + "relevantCount": 28 + }, + { + "id": "tb-005", + "category": "bill-id", + "query": "H 4351", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["sSa8ZLDXwVOR5PRBLzOF0"], + "relevantCount": 28 + }, + { + "id": "tb-006", + "category": "bill-id", + "query": "S505", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "q7FABYQzOvfmjegGbYa0-", + "Kjme82E2J3xh_iavVpoyc", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "g_CMBoU8VJyXAB_0n_Hx5", + "65VMrIqujcAENO9dZP5Gg", + "n214rkLkVmTQngHYGShYa", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz" + ], + "relevantCount": 7 + }, + { + "id": "tb-007", + "category": "bill-id", + "query": "H2057", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "p-Fj1pOglGAmu18SEZVis", + "qA3IwUwvx5MxvFbqGc7sd", + "uXdD5bXYgaRAla6i9UAxz", + "1iLkOnpKnIyMPZ0EJ6q-8", + "zQ0790mAoAXDi3wH5EysT", + "byQC1IHIabnkVGSLxeQ46" + ], + "relevantCount": 6 + }, + { + "id": "tb-008", + "category": "bill-id", + "query": "H.1405", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Dc6RugqP0HRxcujz5pLDF", + "Fhw90ohV0b4d6-_oJjixg", + "c7CaVvPRtM6WP-4puuIL9", + "nthww4idwGAivhUavH1KO", + "4UdhIwSdZQKYBsulWHTie", + "uiCjZ0jMjwTOnF8hRKFNH", + "L_gM_T02MsULa-W3BXldU", + "biQVm39ND9kHg9rGlmphQ", + "QfqfGXsYn0K4Iu_38SGF_" + ], + "relevantCount": 6 + }, + { + "id": "ta-001", + "category": "author", + "query": "Progressive Massachusetts", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "ta-002", + "category": "author", + "query": "League of Women Voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "ta-003", + "category": "author", + "query": "T Stephen Jones", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "9-ITBjvC_UUWw-8HhQa9k", + "-01jqkUE2o1lPjijGhV3V", + "0ANKQpAgWF-1sSLPY7aq_", + "7erqVBGptvNi_KUs-EOp2" + ], + "relevantCount": 51 + }, + { + "id": "ta-004", + "category": "author", + "query": "Elaine Almquist", + "metrics": { + "recallAt10": 0.9, + "mrr": 1, + "ndcgAt10": 0.8899541168509599 + }, + "top10": [ + "qkCngGUEqPpW56990vrXS", + "zCw3UiFfn7NpEADJF4pFE", + "7yrt6_JiCsG7N34SdUssW", + "DXt9s5U1LgPRcoT3iaAtb", + "Dc6RugqP0HRxcujz5pLDF", + "Ns08eB38sg4fXiUVyPIS-", + "izBjYVA4sX_hv2LqJNQ7G", + "-sYilJcb7FYgM-xPApD-E", + "JwZRV5xz-g7VAI7AbHSFS", + "ML-T7WWVi7L4Moz961IyO" + ], + "relevantCount": 23 + }, + { + "id": "ta-005", + "category": "author", + "query": "Partners in Democracy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1lOiHvfe3gdqKkfVyUX5P", + "rxS5QAxVif3dC-cLVF7Bb", + "n0cd5-Ga0RBro4CUrGwtZ", + "LiqNtrg-sc03SCuN6Z0py", + "Pw4c_r2EUikJuKCxrUqah", + "NYPkatkLuretCPsXulkdI", + "HEQ9nYz14M8UWESrjHJmo", + "HhPnj15sNI5XTk_G2DIp4", + "hkrlPHtg0Iz-k5bii2aLr", + "IayRrND2zy-f2XSd9cjJS" + ], + "relevantCount": 20 + }, + { + "id": "ta-006", + "category": "author", + "query": "Commission on the Status of Women", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "TvuAMRkXLh2xY7zFkUKzt", + "Y5DsjTHY-u7Mbf4L7B6JS", + "fR53JFa8xQfPTsWmaTzy5", + "Fhw90ohV0b4d6-_oJjixg", + "NFxoUtF2UXQrkCao7y1mg", + "Nc59-_Ck2zDzW7N7LoA8L", + "A-9hxql_cZ8vl_duOB8v8", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0" + ], + "relevantCount": 6 + }, + { + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tt-002", + "category": "topic", + "query": "rent control", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "-l-iQxshYz8DF4nrW0dy4", + "FeBsBGdvWUVRQIKV6pfsF", + "Y5DsjTHY-u7Mbf4L7B6JS" + ], + "relevantCount": 6 + }, + { + "id": "tt-003", + "category": "topic", + "query": "same day voter registration", + "metrics": { + "recallAt10": 0.8, + "mrr": 0.5, + "ndcgAt10": 0.6698623505528797 + }, + "top10": [ + "n214rkLkVmTQngHYGShYa", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "65VMrIqujcAENO9dZP5Gg", + "kWt58RStbQMUB7VAqXNCV", + "g_CMBoU8VJyXAB_0n_Hx5", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz", + "c4TkesyGcrZWBcn07LsTu" + ], + "relevantCount": 14 + }, + { + "id": "tt-004", + "category": "topic", + "query": "vote by mail", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "U7M6KOrOk3HxXp3HMtxga", + "wUCZhFjmgD-gQf1rwlZBO", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0", + "s7YU0Lr8siUL6DZsczrVe", + "rZOoRP_eVkKCsB1EQ64tR", + "ZFMH6QiRVy_ay0TfHLu_u", + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "0f-XBB35ZN3aY4qYD3aIq" + ], + "relevantCount": 3 + }, + { + "id": "tt-005", + "category": "topic", + "query": "solitary confinement", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + }, + { + "id": "tt-006", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tt-007", + "category": "topic", + "query": "zoning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tt-008", + "category": "topic", + "query": "fossil fuels", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "l1I8kGowVtz2WPrYovaPV", + "GDrFCpA4hAdsIw_Rm5X3X", + "-glcyOYtYYPv-IQCOLTu4", + "dUExko-64U-IHhp8lyPri" + ], + "relevantCount": 14 + }, + { + "id": "tt-009", + "category": "topic", + "query": "gender affirming care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Fhw90ohV0b4d6-_oJjixg", + "QddFVr819ct-AL2R1mpSt", + "mC9cec9DPZmJh-h0k59pL", + "YoMcbg3flFzl2ZuNDBsG4", + "qj9B-cAH0m6Zc1tenFcSN", + "7C300VIQg4-XGPdTM9dTq" + ], + "relevantCount": 6 + }, + { + "id": "tt-010", + "category": "topic", + "query": "language access", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "JwZRV5xz-g7VAI7AbHSFS", + "wJk41fu1FiVBJ1xC8GFBE", + "VwGDFqblM0NpNdygg5jj3", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "YZR3tRTH_AxEbmjFrQ1vc", + "gtnHu_C21R-d1YXD04drW", + "Bs8eththvqbwf63jWoIyz", + "WnfgfrLKQDAiUHcNswyT5" + ], + "relevantCount": 6 + }, + { + "id": "tt-011", + "category": "topic", + "query": "parole", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "G2hQLzmZ5GVbZw0IVaWmT", + "B0ZfwvCJxhgQPhi1qUWBE", + "HWYONHILNhIg1YogNtBis", + "tH5mbM__T0n2uCu5qLdPZ", + "xWyIDifmJf0c2Qqz08Wu5", + "SCSHutdvYlIgGvCAMLGop", + "evQA35IoJkYpR0X4mNbcq", + "1e-dFpB_PC6uL_lXhALKT", + "JWXz3tBMSthD1hbfJlC7a", + "VyQaykWaAosNzTi5ZMXVF" + ], + "relevantCount": 21 + }, + { + "id": "tt-012", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "-01jqkUE2o1lPjijGhV3V", + "jyGFy24gK9DJU4A5nb_34", + "tzuzCzguXwJXggnIRpjH8", + "dJZJPWdoBPU9DonfEUVta" + ], + "relevantCount": 25 + }, + { + "id": "ts-001", + "category": "synonym", + "query": "renters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-002", + "category": "synonym", + "query": "tenants", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-003", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "751FZTubP1RtXBZqhCbA9", + "L-3FT2B_7oofbiiOiRexN", + "3FtHbKJu9pAUcAttyGHvN", + "C8n-9BfBnF0oODvYkH7BW", + "Ri_EnpBWxNRBNfub-Jdn8", + "jbagjzjw6Wh1ciFuVSb_S", + "KWMM39oYqore5zm4A384g" + ], + "relevantCount": 26 + }, + { + "id": "ts-004", + "category": "synonym", + "query": "doctor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi", + "klgcdRRi8wbTeCV7VZRYE", + "-01jqkUE2o1lPjijGhV3V" + ], + "relevantCount": 42 + }, + { + "id": "ts-005", + "category": "synonym", + "query": "servicemember", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "L-3FT2B_7oofbiiOiRexN", + "4lMlyTqg1oh8lE4VDSHjs", + "9-awIt_tkxdiaPXPhQBB7", + "3FtHbKJu9pAUcAttyGHvN", + "Y1OMEyu2Tp1GkKMwLk3Se", + "dGmpznN-Ug5JADMuKFcC8", + "JFgq7bV_BWj0QQIGN2qKx", + "V2WhRLPdA4LHRYSki_wBL" + ], + "relevantCount": 16 + }, + { + "id": "ts-006", + "category": "synonym", + "query": "opiate", + "metrics": { + "recallAt10": 0.6666666666666666, + "mrr": 1, + "ndcgAt10": 0.7751482523375254 + }, + "top10": [ + "SBcIo8CsixaeEhBTYBR6W", + "stwHXrRVbe9vJYIDzI6CW", + "j8eAKtpVbb9Vd05l6TIiX", + "2aDdN2OCct1-sOQsBogkd" + ], + "relevantCount": 6 + }, + { + "id": "ts-007", + "category": "synonym", + "query": "clean energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "GDrFCpA4hAdsIw_Rm5X3X", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "of8VK0lewRoYWyDgfNykB" + ], + "relevantCount": 6 + }, + { + "id": "tm-001", + "category": "misspelling", + "query": "progresive massachusets", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "tm-002", + "category": "misspelling", + "query": "leage of women voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "tm-003", + "category": "misspelling", + "query": "rankd choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tm-004", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tm-005", + "category": "misspelling", + "query": "zonning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tm-006", + "category": "misspelling", + "query": "solitary confinment", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/testimony-token-separators.json b/tests/search-eval/baselines/testimony-token-separators.json new file mode 100644 index 000000000..b5a93ad6a --- /dev/null +++ b/tests/search-eval/baselines/testimony-token-separators.json @@ -0,0 +1,924 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "publishedTestimony", + "collection": "publishedTestimony_eval", + "serverVersion": "30.2", + "corpusMd5": "2f6c3a70929df36f4ffa373a44486cf7", + "gitSha": "c4a843c12cd9c72061ddae203bff02aa4ae2db18", + "timestamp": "2026-08-19T20:57:06.052Z", + "sortBy": "_text_match:desc,publishedAt:desc", + "court": 194, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.9974358974358974, + "mrr": 1, + "ndcgAt10": 0.9796148438144686, + "queryCount": 39 + }, + "categories": { + "author": { + "recallAt10": 0.9833333333333334, + "mrr": 1, + "ndcgAt10": 0.9816590194751601, + "queryCount": 6 + }, + "bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 8 + }, + "misspelling": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9429187326594431, + "queryCount": 6 + }, + "synonym": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 7 + }, + "topic": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.9714593663297215, + "queryCount": 12 + } + }, + "queries": [ + { + "id": "tb-001", + "category": "bill-id", + "query": "S531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-002", + "category": "bill-id", + "query": "S 531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1fSJjeB4F2hKx7xktRO00", + "O8Q64gYX42olvz7w9NczZ", + "0wpvjt7NwI_wHGE47nKbC", + "j95dKA76AwuBFdXhKz1Vu", + "bsp6UU1XycOMBjsL5B6oK", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw" + ], + "relevantCount": 108 + }, + { + "id": "tb-003", + "category": "bill-id", + "query": "S.531", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "dsFGMNRAMzfNW4bFEwafw", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z", + "PQefJ508_vvqTnoGzp2d0", + "VFY12LS1u2OKx1tI7qLk8", + "Ol_Hb4XciXTzUH1BRRNWY" + ], + "relevantCount": 108 + }, + { + "id": "tb-004", + "category": "bill-id", + "query": "H4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG", + "5V9bzQe2iZNmTALY2Quso", + "d2E2LxE3r03TgpgjMG5NX", + "9-awIt_tkxdiaPXPhQBB7" + ], + "relevantCount": 28 + }, + { + "id": "tb-005", + "category": "bill-id", + "query": "H 4351", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "9-awIt_tkxdiaPXPhQBB7", + "sSa8ZLDXwVOR5PRBLzOF0", + "IN1SeVROwhkoyerBYC4fo", + "8Xjcbln58m_qmfQP05tct", + "6zphapXlY5Lp1N13oJMRD", + "GXINovmMEERyTJUEbOKEE", + "hsbL7POvqXTw2e5DUuTki", + "4lMlyTqg1oh8lE4VDSHjs", + "6fPT4NSDkgXvTBahF56xt", + "3KFRIeToLSmcTd0GnQ9zG" + ], + "relevantCount": 28 + }, + { + "id": "tb-006", + "category": "bill-id", + "query": "S505", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "q7FABYQzOvfmjegGbYa0-", + "Kjme82E2J3xh_iavVpoyc", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "g_CMBoU8VJyXAB_0n_Hx5", + "65VMrIqujcAENO9dZP5Gg", + "n214rkLkVmTQngHYGShYa", + "c-FkwCtiKzdl1Z_QGZ2zS", + "nlIiKUB5qTgGcVEQc2xPz" + ], + "relevantCount": 7 + }, + { + "id": "tb-007", + "category": "bill-id", + "query": "H2057", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "p-Fj1pOglGAmu18SEZVis", + "qA3IwUwvx5MxvFbqGc7sd", + "uXdD5bXYgaRAla6i9UAxz", + "1iLkOnpKnIyMPZ0EJ6q-8", + "zQ0790mAoAXDi3wH5EysT", + "byQC1IHIabnkVGSLxeQ46" + ], + "relevantCount": 6 + }, + { + "id": "tb-008", + "category": "bill-id", + "query": "H.1405", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Dc6RugqP0HRxcujz5pLDF", + "Fhw90ohV0b4d6-_oJjixg", + "c7CaVvPRtM6WP-4puuIL9", + "nthww4idwGAivhUavH1KO", + "4UdhIwSdZQKYBsulWHTie", + "uiCjZ0jMjwTOnF8hRKFNH", + "L_gM_T02MsULa-W3BXldU", + "biQVm39ND9kHg9rGlmphQ", + "QfqfGXsYn0K4Iu_38SGF_" + ], + "relevantCount": 6 + }, + { + "id": "ta-001", + "category": "author", + "query": "Progressive Massachusetts", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "ta-002", + "category": "author", + "query": "League of Women Voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "ta-003", + "category": "author", + "query": "T Stephen Jones", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "9-ITBjvC_UUWw-8HhQa9k", + "-01jqkUE2o1lPjijGhV3V", + "0ANKQpAgWF-1sSLPY7aq_", + "7erqVBGptvNi_KUs-EOp2" + ], + "relevantCount": 51 + }, + { + "id": "ta-004", + "category": "author", + "query": "Elaine Almquist", + "metrics": { + "recallAt10": 0.9, + "mrr": 1, + "ndcgAt10": 0.8899541168509599 + }, + "top10": [ + "qkCngGUEqPpW56990vrXS", + "zCw3UiFfn7NpEADJF4pFE", + "7yrt6_JiCsG7N34SdUssW", + "DXt9s5U1LgPRcoT3iaAtb", + "Dc6RugqP0HRxcujz5pLDF", + "Ns08eB38sg4fXiUVyPIS-", + "izBjYVA4sX_hv2LqJNQ7G", + "-sYilJcb7FYgM-xPApD-E", + "JwZRV5xz-g7VAI7AbHSFS", + "ML-T7WWVi7L4Moz961IyO" + ], + "relevantCount": 23 + }, + { + "id": "ta-005", + "category": "author", + "query": "Partners in Democracy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "1lOiHvfe3gdqKkfVyUX5P", + "rxS5QAxVif3dC-cLVF7Bb", + "n0cd5-Ga0RBro4CUrGwtZ", + "LiqNtrg-sc03SCuN6Z0py", + "Pw4c_r2EUikJuKCxrUqah", + "NYPkatkLuretCPsXulkdI", + "HEQ9nYz14M8UWESrjHJmo", + "HhPnj15sNI5XTk_G2DIp4", + "hkrlPHtg0Iz-k5bii2aLr", + "IayRrND2zy-f2XSd9cjJS" + ], + "relevantCount": 20 + }, + { + "id": "ta-006", + "category": "author", + "query": "Commission on the Status of Women", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "TvuAMRkXLh2xY7zFkUKzt", + "Y5DsjTHY-u7Mbf4L7B6JS", + "fR53JFa8xQfPTsWmaTzy5", + "Fhw90ohV0b4d6-_oJjixg", + "NFxoUtF2UXQrkCao7y1mg", + "Nc59-_Ck2zDzW7N7LoA8L", + "A-9hxql_cZ8vl_duOB8v8", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0" + ], + "relevantCount": 6 + }, + { + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tt-002", + "category": "topic", + "query": "rent control", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "-l-iQxshYz8DF4nrW0dy4", + "FeBsBGdvWUVRQIKV6pfsF", + "Y5DsjTHY-u7Mbf4L7B6JS" + ], + "relevantCount": 6 + }, + { + "id": "tt-003", + "category": "topic", + "query": "same day voter registration", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "J4jkxIzvLivJtI5vDDKGF", + "n214rkLkVmTQngHYGShYa", + "Kjme82E2J3xh_iavVpoyc", + "AHXLF18qIoNN9O75snH7F", + "6K0bAeoxUTX6kGiaaCHLn", + "RPZx9woG9hVb3OMFtSQXd", + "65VMrIqujcAENO9dZP5Gg", + "kWt58RStbQMUB7VAqXNCV", + "g_CMBoU8VJyXAB_0n_Hx5", + "1lOiHvfe3gdqKkfVyUX5P" + ], + "relevantCount": 18 + }, + { + "id": "tt-004", + "category": "topic", + "query": "vote by mail", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "c-42kp84LZ75LcjxE6T1F", + "v-JlERq0w6jS9Qyw-v1fg", + "n0cd5-Ga0RBro4CUrGwtZ", + "U7M6KOrOk3HxXp3HMtxga", + "wUCZhFjmgD-gQf1rwlZBO", + "uZfPyMpEKytkwPxH7vCoe", + "_mY95B40agX1xkgaw2Vk0", + "s7YU0Lr8siUL6DZsczrVe", + "0f-XBB35ZN3aY4qYD3aIq", + "rZOoRP_eVkKCsB1EQ64tR" + ], + "relevantCount": 3 + }, + { + "id": "tt-005", + "category": "topic", + "query": "solitary confinement", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + }, + { + "id": "tt-006", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tt-007", + "category": "topic", + "query": "zoning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tt-008", + "category": "topic", + "query": "fossil fuels", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "l1I8kGowVtz2WPrYovaPV", + "GDrFCpA4hAdsIw_Rm5X3X", + "-glcyOYtYYPv-IQCOLTu4", + "dUExko-64U-IHhp8lyPri" + ], + "relevantCount": 14 + }, + { + "id": "tt-009", + "category": "topic", + "query": "gender affirming care", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "Fhw90ohV0b4d6-_oJjixg", + "QddFVr819ct-AL2R1mpSt", + "mC9cec9DPZmJh-h0k59pL", + "YoMcbg3flFzl2ZuNDBsG4", + "qj9B-cAH0m6Zc1tenFcSN", + "7C300VIQg4-XGPdTM9dTq" + ], + "relevantCount": 6 + }, + { + "id": "tt-010", + "category": "topic", + "query": "language access", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "JwZRV5xz-g7VAI7AbHSFS", + "wJk41fu1FiVBJ1xC8GFBE", + "VwGDFqblM0NpNdygg5jj3", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "YZR3tRTH_AxEbmjFrQ1vc", + "gtnHu_C21R-d1YXD04drW", + "Bs8eththvqbwf63jWoIyz", + "WnfgfrLKQDAiUHcNswyT5" + ], + "relevantCount": 6 + }, + { + "id": "tt-011", + "category": "topic", + "query": "parole", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "G2hQLzmZ5GVbZw0IVaWmT", + "B0ZfwvCJxhgQPhi1qUWBE", + "HWYONHILNhIg1YogNtBis", + "tH5mbM__T0n2uCu5qLdPZ", + "xWyIDifmJf0c2Qqz08Wu5", + "SCSHutdvYlIgGvCAMLGop", + "evQA35IoJkYpR0X4mNbcq", + "1e-dFpB_PC6uL_lXhALKT", + "JWXz3tBMSthD1hbfJlC7a", + "VyQaykWaAosNzTi5ZMXVF" + ], + "relevantCount": 21 + }, + { + "id": "tt-012", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "-01jqkUE2o1lPjijGhV3V", + "jyGFy24gK9DJU4A5nb_34", + "tzuzCzguXwJXggnIRpjH8", + "dJZJPWdoBPU9DonfEUVta" + ], + "relevantCount": 25 + }, + { + "id": "ts-001", + "category": "synonym", + "query": "renters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-002", + "category": "synonym", + "query": "tenants", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "X4C9g3rP9A5tKiBtkiNTF", + "C8n-9BfBnF0oODvYkH7BW", + "lsZSRKcHwgsW_scJjO480", + "as6Is9LPZ2J-zPMzAJUUu" + ], + "relevantCount": 35 + }, + { + "id": "ts-003", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "751FZTubP1RtXBZqhCbA9", + "L-3FT2B_7oofbiiOiRexN", + "3FtHbKJu9pAUcAttyGHvN", + "C8n-9BfBnF0oODvYkH7BW", + "Ri_EnpBWxNRBNfub-Jdn8", + "jbagjzjw6Wh1ciFuVSb_S", + "KWMM39oYqore5zm4A384g" + ], + "relevantCount": 26 + }, + { + "id": "ts-004", + "category": "synonym", + "query": "doctor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "x54aPTzKw4mzfCHDGzvjr", + "tKlwe6ivPE1zcoNFvPooU", + "7yrt6_JiCsG7N34SdUssW", + "uUmrMcsmvHJAJLGdK2_Y4", + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi", + "klgcdRRi8wbTeCV7VZRYE", + "-01jqkUE2o1lPjijGhV3V" + ], + "relevantCount": 42 + }, + { + "id": "ts-005", + "category": "synonym", + "query": "servicemember", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "L-3FT2B_7oofbiiOiRexN", + "4lMlyTqg1oh8lE4VDSHjs", + "9-awIt_tkxdiaPXPhQBB7", + "3FtHbKJu9pAUcAttyGHvN", + "Y1OMEyu2Tp1GkKMwLk3Se", + "dGmpznN-Ug5JADMuKFcC8", + "JFgq7bV_BWj0QQIGN2qKx", + "V2WhRLPdA4LHRYSki_wBL" + ], + "relevantCount": 16 + }, + { + "id": "ts-006", + "category": "synonym", + "query": "opiate", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "tH5mbM__T0n2uCu5qLdPZ", + "SBcIo8CsixaeEhBTYBR6W", + "stwHXrRVbe9vJYIDzI6CW", + "sC6kejehL-M2kpkSEnnfV", + "j8eAKtpVbb9Vd05l6TIiX", + "2aDdN2OCct1-sOQsBogkd" + ], + "relevantCount": 6 + }, + { + "id": "ts-007", + "category": "synonym", + "query": "clean energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "il85zcsUI33WROqVNRURb", + "q6OliiiesY1xzxmAVjuKY", + "0oN1j4dwm8UBbilDIliJ2", + "PqQDouJ4kaUN9R0VhSYS0", + "RFSu0M5xezxVa1bBt9qve", + "GDrFCpA4hAdsIw_Rm5X3X", + "EildpYQGTwzXhlnDEqqJd", + "syWoGzG3fuHNKS6tcW2e4", + "A2keKEFOnMVvmssvDcBs9", + "of8VK0lewRoYWyDgfNykB" + ], + "relevantCount": 6 + }, + { + "id": "tm-001", + "category": "misspelling", + "query": "progresive massachusets", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "evG1J59pHtD892srT_HSD", + "b0L9RQh_NZ2EnnljzCNRj", + "pi5rvUyjKlvmMNewnOf_D", + "eam7r9I54vLt29Sk6A-6D", + "seIDlGYzq_vEasViQdiN2", + "8cahVYFjE4R73eGH_2dEF", + "K5fegU_C0p1Qnyq4xexmz", + "RPlMJNham07rUs9FnZhtw", + "dPaqTI3uE2tKfqSjvnjBY", + "ECfI2TLNAozLWAJ9mi_nL" + ], + "relevantCount": 122 + }, + { + "id": "tm-002", + "category": "misspelling", + "query": "leage of women voters", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "QCwx_oImL54_8fifC7dss", + "AGusMrxuxEbkyPxIwGxld", + "BgMzB0KujpKISNB4_Nprf", + "FBWyrabLoxZuH3VMvFMuv", + "QATSQq06ilckdBDgUqGwE", + "phMMN9IYufTSmNzU97pzD", + "lkWCmC3HAy96sThW3Vueb", + "LEu7VKw_xaNQHqIFXgFwG", + "iL9cYJkGWX4Yq1i2biYf6", + "TPgjZsn9vXKQ2aJF0UUEi" + ], + "relevantCount": 121 + }, + { + "id": "tm-003", + "category": "misspelling", + "query": "rankd choice voting", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6575123959566591 + }, + "top10": [ + "B5xcpPPD3VBtvGdzE8hyM", + "pQ-NeP4xOzbuhH8fGq9ea", + "0pICmOpKXlbDf0APyn1kI", + "9OuaIMy4lyzKGM7odk_D1", + "CdHtztqVLe6x9PWBICz2F", + "jPFcvSFfzB6Z1kN8rZk2J", + "YPu6EPQGzTJZkPXfafmD3", + "ViYoho0abCQifXHNDbjin", + "KYRHh20qkFcJmVbEg1sTg", + "92Is4Ysah1wA6bSY62X_z" + ], + "relevantCount": 121 + }, + { + "id": "tm-004", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "C8n-9BfBnF0oODvYkH7BW", + "as6Is9LPZ2J-zPMzAJUUu", + "jycTZ6dvmsbDtAXhY_52F", + "gc8r_z72KmHh3ODDmFiVM", + "QSRoYGesdL0MwTdDEH-IB", + "lsZSRKcHwgsW_scJjO480" + ], + "relevantCount": 11 + }, + { + "id": "tm-005", + "category": "misspelling", + "query": "zonning", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eam7r9I54vLt29Sk6A-6D", + "HEq90V_3KxxP4YSsyIhpT", + "rl6IFaDSZ88PYX1IJjA0N", + "vVY5uJaW2d_j882fhlQ04", + "Z44XOXbB1FmyRaWYwDdCW", + "U8V-O5oCaJvThKemGgKIH", + "p9s_gokSsUZ_z5uyCaRTD", + "LN8U2qzdz-L-5K56kVKi3", + "-l-iQxshYz8DF4nrW0dy4", + "Bs8eththvqbwf63jWoIyz" + ], + "relevantCount": 12 + }, + { + "id": "tm-006", + "category": "misspelling", + "query": "solitary confinment", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "eyoxyKQ66yNQLahDj7gbR", + "SnR_-S6h2SmLvJkuXf4ie", + "p_F4FPB_l6RyZmbNU2Laz", + "RF6gia3v4OMWFyp59RwVI" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/typesense-30-number-variants.json b/tests/search-eval/baselines/typesense-30-number-variants.json new file mode 100644 index 000000000..f05cef3a9 --- /dev/null +++ b/tests/search-eval/baselines/typesense-30-number-variants.json @@ -0,0 +1,1703 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "6041d1fed1399eece7aa523428e97397", + "gitSha": "c4a843c12cd9c72061ddae203bff02aa4ae2db18", + "timestamp": "2026-08-19T20:58:35.665Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8532894736842105, + "mrr": 0.9594820384294069, + "ndcgAt10": 0.737651675424721, + "queryCount": 76 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1, + "queryCount": 15 + }, + "hyphenation": { + "recallAt10": 0.44166666666666665, + "mrr": 1, + "ndcgAt10": 0.5736344100720298, + "queryCount": 6 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8699999999999999, + "mrr": 1, + "ndcgAt10": 0.4985867229196039, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.96, + "mrr": 1, + "ndcgAt10": 0.9727329844310522, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.78, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.4972308270259364, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564", "192-S2565"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2475", "192-H2475"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.17571068025797132 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615", + "192-S1130" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2318", + "192-H2258" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H723", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S40", + "192-S2788", + "192-S2556", + "192-S2511", + "192-S2509", + "192-S2507", + "192-S2506", + "192-S2504", + "192-S2501", + "192-S2500" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H2884", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6104531109861133 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071", + "192-H2068" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S97", + "192-H560", + "192-S1830", + "192-H4481", + "192-H3526", + "192-H2939", + "192-H1158", + "192-S901", + "192-S883", + "192-S873" + ], + "relevantCount": 22 + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8701249883466593 + }, + "top10": [ + "192-S412", + "192-S133", + "192-S659", + "192-S400", + "192-H749", + "192-H736", + "192-H735", + "192-H3020", + "192-S807", + "192-S766" + ], + "relevantCount": 16 + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": [ + "192-S100", + "192-S2761", + "192-S2667", + "192-H947", + "192-H4102", + "192-H352", + "192-H1094", + "192-S1475", + "192-H297", + "192-H2372" + ], + "relevantCount": 13 + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.5637884576902256 + }, + "top10": [ + "192-S978", + "192-H272", + "192-H1779", + "192-H1497", + "192-S982", + "192-S650", + "192-S645", + "192-S480", + "192-S285", + "192-S2078" + ], + "relevantCount": 11 + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "metrics": { + "recallAt10": 0.25, + "mrr": 1, + "ndcgAt10": 0.41253177989255346 + }, + "top10": [ + "192-H4134", + "192-S2225", + "192-S2265", + "192-S2230", + "192-S2229", + "192-S2136", + "192-S1371", + "192-H3288", + "192-H2355", + "192-H1162" + ], + "relevantCount": 8 + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.647939623894138 + }, + "top10": [ + "192-S2403", + "192-H2112", + "192-H2091", + "192-H2456", + "192-S1653", + "192-H2822", + "192-S2580" + ], + "relevantCount": 8 + } + ] +} diff --git a/tests/search-eval/baselines/typesense-30-synonyms.json b/tests/search-eval/baselines/typesense-30-synonyms.json new file mode 100644 index 000000000..da7f5803c --- /dev/null +++ b/tests/search-eval/baselines/typesense-30-synonyms.json @@ -0,0 +1,1573 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "a0499d7bb8597f88dbd737893bcd8a4d", + "gitSha": "37e184296e74e34dcfb2de1dabdef41c96486e2c", + "timestamp": "2026-08-18T20:21:13.002Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8457142857142858, + "mrr": 0.9131519274376417, + "ndcgAt10": 0.7088531553120943, + "queryCount": 70 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 0.8, + "mrr": 0.8, + "ndcgAt10": 0.8, + "queryCount": 15 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8699999999999999, + "mrr": 1, + "ndcgAt10": 0.4985867229196039, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.96, + "mrr": 1, + "ndcgAt10": 0.9727329844310522, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.78, + "mrr": 0.9555555555555555, + "ndcgAt10": 0.4972308270259364, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-H1795", + "192-S610", + "192-S2720", + "192-S2136", + "192-S1179", + "192-H878", + "192-H3288", + "192-H1959", + "192-H136", + "192-S904" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4352", "192-H564", "192-H1961", "192-H1762"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2170", + "192-S2158", + "192-H3372", + "192-H3302", + "192-H2475", + "192-H2076", + "192-H1521", + "192-S2289", + "192-H4470", + "192-H4459" + ], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.17571068025797132 + }, + "top10": [ + "192-H43", + "192-H2635", + "192-H1529", + "192-S965", + "192-S942", + "192-S939", + "192-S547", + "192-S1618", + "192-S1615", + "192-S1130" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S942", + "192-S2762", + "192-S1614", + "192-S1602", + "192-S1568", + "192-S1563", + "192-S1562", + "192-S1561", + "192-S1537", + "192-H43" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S366", + "192-S2748", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-S1673", + "192-H682", + "192-H681" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S76", + "192-S74", + "192-S73" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S740", + "192-S330", + "192-S210", + "192-S1457", + "192-S1306", + "192-S1211", + "192-H2402", + "192-H2367", + "192-H2318", + "192-H2258" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H723", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S40", + "192-S2788", + "192-S2556", + "192-S2511", + "192-S2509", + "192-S2507", + "192-S2506", + "192-S2504", + "192-S2501", + "192-S2500" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H2884", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6104531109861133 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071", + "192-H2068" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/typesense-30-weights.json b/tests/search-eval/baselines/typesense-30-weights.json new file mode 100644 index 000000000..8504d548f --- /dev/null +++ b/tests/search-eval/baselines/typesense-30-weights.json @@ -0,0 +1,1573 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "a0499d7bb8597f88dbd737893bcd8a4d", + "gitSha": "0d747ef1430b5ca4afcd9a85d7b2ba3b3e2f815d", + "timestamp": "2026-08-18T19:41:51.019Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.8214285714285714, + "mrr": 0.8917233560090704, + "ndcgAt10": 0.6871465359811022, + "queryCount": 70 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 0.8, + "mrr": 0.8, + "ndcgAt10": 0.8, + "queryCount": 15 + }, + "member-committee": { + "recallAt10": 0.8866666666666667, + "mrr": 0.8539682539682539, + "ndcgAt10": 0.8641271505214204, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8699999999999999, + "mrr": 1, + "ndcgAt10": 0.4985867229196039, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.7899999999999999, + "mrr": 0.9, + "ndcgAt10": 0.8262463153675897, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.78, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.4945009938991955, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-H1795", + "192-S610", + "192-S2720", + "192-S2136", + "192-S1179", + "192-H878", + "192-H3288", + "192-H1959", + "192-H136", + "192-S904" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4352", "192-H564", "192-H1961", "192-H1762"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2170", + "192-S2158", + "192-H3372", + "192-H3302", + "192-H2475", + "192-H2076", + "192-H1521", + "192-S2289", + "192-H4470", + "192-H4459" + ], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1698", + "192-S99", + "192-S98", + "192-S882", + "192-S881", + "192-S880", + "192-S879", + "192-S878", + "192-S877", + "192-S876" + ], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S910", + "192-S909", + "192-S908", + "192-S907", + "192-S856", + "192-S817", + "192-S816", + "192-S815", + "192-S814", + "192-S730" + ], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S895", + "192-S845", + "192-S844", + "192-S796", + "192-S795", + "192-S794", + "192-S793", + "192-S792", + "192-S791", + "192-S790" + ], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H867", + "192-H866", + "192-H865", + "192-H768", + "192-H767", + "192-H732", + "192-H731", + "192-H62", + "192-H559", + "192-H4388" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2618", + "192-S992", + "192-S991", + "192-S990", + "192-S989", + "192-S988", + "192-S95", + "192-S94", + "192-S873", + "192-S759" + ], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S984", + "192-S983", + "192-S982", + "192-S981", + "192-S980", + "192-S979", + "192-S978", + "192-S977", + "192-S976", + "192-S975" + ], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H881", + "192-H774", + "192-H570", + "192-H569", + "192-H568", + "192-H567", + "192-H566", + "192-H565", + "192-H4412", + "192-H4401" + ], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S15", + "192-S852", + "192-S851", + "192-S850", + "192-S849", + "192-S807", + "192-S806", + "192-S805", + "192-S76", + "192-S712" + ], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S902", + "192-S854", + "192-S853", + "192-S78", + "192-S720", + "192-S719", + "192-S718", + "192-S717", + "192-S716", + "192-S715" + ], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H95", "192-H4430", "192-H3771", "192-H3720"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 0.16666666666666666, + "ndcgAt10": 0.3510684246681534 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H1968", + "192-H629", + "192-H3415", + "192-S2749", + "192-S2747", + "192-S2731", + "192-S2723", + "192-S2708" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2613", + "192-H4329", + "192-H4328", + "192-S820", + "192-S819", + "192-S817", + "192-S816", + "192-S813", + "192-S810", + "192-S806" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0.9, + "mrr": 0.5, + "ndcgAt10": 0.7799082337019199 + }, + "top10": [ + "192-S2743", + "192-S1812", + "192-H3085", + "192-H3081", + "192-H2973", + "192-H2928", + "192-H2895", + "192-H2826", + "192-S788", + "192-S1997" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5582604437617528 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S2749", + "192-S2747", + "192-S362", + "192-S358", + "192-S319" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.14285714285714285, + "ndcgAt10": 0.2726701556894781 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S604", + "192-S2515", + "192-S2473", + "192-S2237" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.12111401772315426 + }, + "top10": [ + "192-H2527", + "192-S942", + "192-H1734", + "192-S939", + "192-S514", + "192-S1130", + "192-S1009", + "192-H43", + "192-H308", + "192-H2635" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.47907112408275354 + }, + "top10": [ + "192-S2174", + "192-H4044", + "192-H3346", + "192-H3319", + "192-H3260", + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2168", + "192-S2145" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H3269", + "192-H2145", + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3923", + "192-H3708", + "192-H3335", + "192-H3330" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2762", + "192-S1568", + "192-S1563", + "192-S1561", + "192-H43", + "192-H3847", + "192-H3846", + "192-H2635", + "192-H2533", + "192-H2505" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8643145546088337 + }, + "top10": [ + "192-H2470", + "192-S1540", + "192-H2874", + "192-H2491", + "192-H2488", + "192-H2439", + "192-S942", + "192-S939", + "192-S1615", + "192-S1562" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8701249883466593 + }, + "top10": [ + "192-S366", + "192-S2748", + "192-H682", + "192-H642", + "192-H127", + "192-S285", + "192-H675", + "192-H1620", + "192-S384", + "192-S377" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H19", + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-H628", + "192-H594", + "192-H4443", + "192-H2759", + "192-H2688" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-S77", + "192-S69", + "192-S63", + "192-H2056" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2600", + "192-S2079", + "192-S1458", + "192-S1436", + "192-S1420", + "192-S1399", + "192-S1134", + "192-S1009", + "192-H4510", + "192-H437" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.800693766409882 + }, + "top10": [ + "192-H723", + "192-S1961", + "192-H734", + "192-H732", + "192-H3734", + "192-H2903", + "192-H2947", + "192-S766", + "192-S252", + "192-H500" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2215", + "192-H353", + "192-S2566", + "192-S2477", + "192-S246", + "192-S228", + "192-S227", + "192-S199", + "192-H473", + "192-H472" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H2884", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S636", + "192-S1318", + "192-S1310", + "192-S1300", + "192-S1297", + "192-S1284", + "192-S1283", + "192-S1267", + "192-S1257" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S868", + "192-S867", + "192-H1377", + "192-H1373", + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S2025", + "192-S1985" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6104531109861133 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071", + "192-H2068" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/baselines/typesense-30.json b/tests/search-eval/baselines/typesense-30.json new file mode 100644 index 000000000..a70da3c7b --- /dev/null +++ b/tests/search-eval/baselines/typesense-30.json @@ -0,0 +1,1474 @@ +{ + "meta": { + "env": "local", + "url": "", + "alias": "bills", + "collection": "bills_eval", + "serverVersion": "30.2", + "corpusMd5": "a0499d7bb8597f88dbd737893bcd8a4d", + "gitSha": "a1661656ddfb64d6c8f55aee9dc302a2929bf114", + "timestamp": "2026-08-18T19:08:05.186Z", + "sortBy": "_text_match:desc,testimonyCount:desc", + "court": 192, + "skippedQueries": [] + }, + "overall": { + "recallAt10": 0.6514285714285716, + "mrr": 0.7673015873015874, + "ndcgAt10": 0.5251301729907821, + "queryCount": 70 + }, + "categories": { + "exact-bill-id": { + "recallAt10": 0.8, + "mrr": 0.8, + "ndcgAt10": 0.8, + "queryCount": 15 + }, + "member-committee": { + "recallAt10": 0.09333333333333334, + "mrr": 0.2733333333333333, + "ndcgAt10": 0.11884984315846131, + "queryCount": 15 + }, + "misspelling": { + "recallAt10": 0.8699999999999999, + "mrr": 1, + "ndcgAt10": 0.4985867229196039, + "queryCount": 10 + }, + "synonym": { + "recallAt10": 0.7899999999999999, + "mrr": 0.9, + "ndcgAt10": 0.8262463153675897, + "queryCount": 10 + }, + "topic": { + "recallAt10": 0.78, + "mrr": 0.9305555555555556, + "ndcgAt10": 0.4864017039552942, + "queryCount": 20 + } + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H100", + "192-H1002", + "192-H1001", + "192-H1000", + "192-H1006", + "192-H1004", + "192-H1003", + "192-H1009", + "192-H1008", + "192-H1007" + ], + "relevantCount": 1 + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-H1795", + "192-S904", + "192-S9", + "192-S898", + "192-S873", + "192-S805", + "192-S804", + "192-S782", + "192-S771", + "192-S766" + ], + "relevantCount": 1 + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S9", + "192-S992", + "192-S99", + "192-S978", + "192-S973", + "192-S944", + "192-S94", + "192-S931", + "192-S909", + "192-S902" + ], + "relevantCount": 1 + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4352"], + "relevantCount": 1 + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S2564"], + "relevantCount": 1 + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H4374"], + "relevantCount": 1 + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H73", + "192-H739", + "192-H737", + "192-H736", + "192-H735", + "192-H734", + "192-H733", + "192-H732", + "192-H731", + "192-H730" + ], + "relevantCount": 1 + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H3999"], + "relevantCount": 1 + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1333"], + "relevantCount": 1 + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2170", "192-S2158", "192-H3372", "192-H3302"], + "relevantCount": 1 + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-H600"], + "relevantCount": 1 + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-H1", + "192-S736", + "192-S451", + "192-S2759", + "192-S1841", + "192-S1633", + "192-S147", + "192-S130", + "192-S119", + "192-H4154" + ], + "relevantCount": 1 + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": ["192-S1563"], + "relevantCount": 1 + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["192-S1698"], + "relevantCount": 444 + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4316"], + "relevantCount": 290 + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S1"], + "relevantCount": 527 + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S733", + "192-S571", + "192-S2737", + "192-S2523", + "192-S1796", + "192-S143", + "192-H877", + "192-H4433", + "192-H2908", + "192-H2833" + ], + "relevantCount": 198 + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S44", "192-H3765"], + "relevantCount": 306 + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H2756"], + "relevantCount": 106 + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [], + "relevantCount": 143 + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "metrics": { + "recallAt10": 0.1, + "mrr": 1, + "ndcgAt10": 0.22009176629808017 + }, + "top10": ["192-S15", "192-S2580"], + "relevantCount": 557 + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-H4341", "192-S7", "192-S2544", "192-H3476"], + "relevantCount": 663 + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": ["192-S2544", "192-H3476"], + "relevantCount": 4 + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S1534", + "192-S1220", + "192-H629", + "192-H3415", + "192-H1968", + "192-S82", + "192-S785", + "192-S771", + "192-S364", + "192-S2626" + ], + "relevantCount": 715 + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8415908474314976 + }, + "top10": [ + "192-S798", + "192-S791", + "192-S787", + "192-S785", + "192-S782", + "192-S779", + "192-S770", + "192-S758", + "192-S743", + "192-S675" + ], + "relevantCount": 264 + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2743", + "192-S2070", + "192-S1866", + "192-S1731", + "192-S1730", + "192-H4518", + "192-H4513", + "192-H4429", + "192-H4428", + "192-H3803" + ], + "relevantCount": 111 + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.43735247915031 + }, + "top10": [ + "192-S1220", + "192-H1968", + "192-S2769", + "192-S2756", + "192-S2755", + "192-S846", + "192-S387", + "192-S381", + "192-S378", + "192-S365" + ], + "relevantCount": 63 + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "metrics": { + "recallAt10": 0.1, + "mrr": 0.1, + "ndcgAt10": 0.06362078819895171 + }, + "top10": [ + "192-S2715", + "192-S2711", + "192-S2647", + "192-H4488", + "192-H4455", + "192-H4400", + "192-S568", + "192-S566", + "192-S2229", + "192-S2222" + ], + "relevantCount": 120 + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.5, + "ndcgAt10": 0.12111401772315426 + }, + "top10": [ + "192-H2527", + "192-S942", + "192-H1734", + "192-S939", + "192-S514", + "192-S1130", + "192-S1009", + "192-H43", + "192-H308", + "192-H2635" + ], + "relevantCount": 93 + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S703", + "192-S675", + "192-S636", + "192-S292", + "192-S2790", + "192-S2584", + "192-S2572", + "192-S2528", + "192-S2514" + ], + "relevantCount": 66 + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6186468626657726 + }, + "top10": [ + "192-S2227", + "192-S2154", + "192-H953", + "192-H4348", + "192-H3310", + "192-H3303", + "192-H3302", + "192-H3266", + "192-H2924", + "192-H4524" + ], + "relevantCount": 12 + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.6172182802301824 + }, + "top10": [ + "192-S371", + "192-S353", + "192-S343", + "192-H712", + "192-H696", + "192-S386", + "192-S372", + "192-S286", + "192-H718", + "192-H672" + ], + "relevantCount": 13 + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S868", + "192-S867", + "192-S2025", + "192-S1985", + "192-S1853", + "192-S1808" + ], + "relevantCount": 40 + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "metrics": { + "recallAt10": 0.4, + "mrr": 1, + "ndcgAt10": 0.7945202298211209 + }, + "top10": [ + "192-H2086", + "192-H2071", + "192-S2", + "192-S347", + "192-S2472", + "192-S2467", + "192-H2097", + "192-H2405", + "192-H3796", + "192-H2101" + ], + "relevantCount": 21 + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "metrics": { + "recallAt10": 0.3, + "mrr": 1, + "ndcgAt10": 0.48664114167177874 + }, + "top10": [ + "192-S1827", + "192-H2883", + "192-H1320", + "192-H3030", + "192-H1366", + "192-S2580" + ], + "relevantCount": 21 + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S939", + "192-S2678", + "192-S190", + "192-S176", + "192-S1488", + "192-S1134", + "192-S1087", + "192-S1073", + "192-H4330", + "192-H4013" + ], + "relevantCount": 26 + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S2208", + "192-S2203", + "192-S2180", + "192-S2174", + "192-S2168", + "192-S2145", + "192-S1896", + "192-S1326", + "192-H4044", + "192-H3346" + ], + "relevantCount": 34 + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6310385047070085 + }, + "top10": [ + "192-S678", + "192-H125", + "192-H1101", + "192-S770", + "192-S766", + "192-S682", + "192-S2774", + "192-S2656", + "192-S2", + "192-S1470" + ], + "relevantCount": 3 + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.40293031154000103 + }, + "top10": [ + "192-S28", + "192-H71", + "192-H73", + "192-S484", + "192-S451", + "192-H834", + "192-H805", + "192-H3950", + "192-S459", + "192-H4367" + ], + "relevantCount": 5 + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "metrics": { + "recallAt10": 0.4, + "mrr": 0.1111111111111111, + "ndcgAt10": 0.04633996212748669 + }, + "top10": [ + "192-S363", + "192-S349", + "192-S314", + "192-H714", + "192-H686", + "192-H580", + "192-H564", + "192-H547", + "192-S298", + "192-S2532" + ], + "relevantCount": 5 + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S590", + "192-S2738", + "192-S2153", + "192-S1940", + "192-H3330", + "192-H3269", + "192-H2159", + "192-H2145", + "192-S2157", + "192-H3964" + ], + "relevantCount": 21 + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2762", + "192-S1568", + "192-S1563", + "192-S1561", + "192-H43", + "192-H3847", + "192-H3846", + "192-H2635", + "192-H2533", + "192-H2505" + ], + "relevantCount": 93 + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8643145546088337 + }, + "top10": [ + "192-H2470", + "192-S1540", + "192-H2874", + "192-H2491", + "192-H2488", + "192-H2439", + "192-S942", + "192-S939", + "192-S1615", + "192-S1562" + ], + "relevantCount": 93 + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "metrics": { + "recallAt10": 0.8, + "mrr": 1, + "ndcgAt10": 0.8701249883466593 + }, + "top10": [ + "192-S366", + "192-S2748", + "192-H682", + "192-H642", + "192-H127", + "192-S285", + "192-H675", + "192-H1620", + "192-S384", + "192-S377" + ], + "relevantCount": 39 + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S1953", + "192-S1791", + "192-S1778", + "192-S1721", + "192-H628", + "192-H594", + "192-H2759", + "192-H2688", + "192-H2655", + "192-H2620" + ], + "relevantCount": 39 + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S77", + "192-S69", + "192-S63", + "192-S2664", + "192-S2662", + "192-S2661", + "192-S2660", + "192-H4507", + "192-H4440", + "192-H2056" + ], + "relevantCount": 57 + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "metrics": { + "recallAt10": 0, + "mrr": 0, + "ndcgAt10": 0 + }, + "top10": [ + "192-S2600", + "192-S2079", + "192-S1458", + "192-S1436", + "192-S1420", + "192-S1399", + "192-S1134", + "192-S1009", + "192-H4510", + "192-H437" + ], + "relevantCount": 15 + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S878", + "192-S1948", + "192-H760", + "192-H730", + "192-H2911", + "192-H2846", + "192-H2820", + "192-H2773", + "192-S900", + "192-S399" + ], + "relevantCount": 27 + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "metrics": { + "recallAt10": 0.7, + "mrr": 1, + "ndcgAt10": 0.800693766409882 + }, + "top10": [ + "192-H723", + "192-S1961", + "192-H734", + "192-H732", + "192-H3734", + "192-H2903", + "192-H2947", + "192-S766", + "192-S252", + "192-H500" + ], + "relevantCount": 41 + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 1 + }, + "top10": [ + "192-S2215", + "192-H353", + "192-S2566", + "192-S2477", + "192-S246", + "192-S228", + "192-S227", + "192-S199", + "192-H473", + "192-H472" + ], + "relevantCount": 82 + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "metrics": { + "recallAt10": 0.6, + "mrr": 1, + "ndcgAt10": 0.727329844310522 + }, + "top10": [ + "192-S339", + "192-H649", + "192-H2884", + "192-H200", + "192-H1590", + "192-S1912", + "192-S852", + "192-S822", + "192-S358", + "192-S349" + ], + "relevantCount": 15 + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S820", + "192-S703", + "192-S675", + "192-S636", + "192-S292", + "192-S2790", + "192-S2584", + "192-S2572", + "192-S2528", + "192-S2514" + ], + "relevantCount": 66 + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.4491790547278348 + }, + "top10": [ + "192-S571", + "192-S2737", + "192-S1796", + "192-H877", + "192-H2908", + "192-H2833", + "192-H258", + "192-H2159", + "192-S1853", + "192-H2890" + ], + "relevantCount": 27 + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S76", + "192-S74", + "192-S73", + "192-S64", + "192-S2663", + "192-S2529", + "192-S1828", + "192-S1583", + "192-H4026", + "192-H3079" + ], + "relevantCount": 57 + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S903", + "192-S895", + "192-S888", + "192-S882", + "192-S868", + "192-S867", + "192-S2025", + "192-S1985", + "192-S1853", + "192-S1808" + ], + "relevantCount": 40 + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.43812463509980126 + }, + "top10": [ + "192-S874", + "192-S82", + "192-S142", + "192-S110", + "192-H3959", + "192-H3838", + "192-H3778", + "192-H264", + "192-H229", + "192-H1436" + ], + "relevantCount": 26 + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.6104531109861133 + }, + "top10": [ + "192-S347", + "192-S1304", + "192-S1302", + "192-H2496", + "192-H2405", + "192-H2110", + "192-H2096", + "192-H2086", + "192-H2071", + "192-H2068" + ], + "relevantCount": 21 + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.3170853252047272 + }, + "top10": [ + "192-S951", + "192-S702", + "192-S2678", + "192-S2417", + "192-S190", + "192-S176", + "192-S1123", + "192-S1076", + "192-S1023", + "192-S1015" + ], + "relevantCount": 26 + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "metrics": { + "recallAt10": 0.5, + "mrr": 1, + "ndcgAt10": 0.5933275909929555 + }, + "top10": [ + "192-S921", + "192-H4505", + "192-H1808", + "192-H1911", + "192-S904", + "192-S891", + "192-S886", + "192-S883", + "192-S874", + "192-S865" + ], + "relevantCount": 24 + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "metrics": { + "recallAt10": 0.2, + "mrr": 1, + "ndcgAt10": 0.7237527203934739 + }, + "top10": [ + "192-H2411", + "192-H2394", + "192-S2626", + "192-S2622", + "192-S251", + "192-S2472", + "192-S2467", + "192-S2079", + "192-S2", + "192-S1755" + ], + "relevantCount": 12 + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "metrics": { + "recallAt10": 1, + "mrr": 1, + "ndcgAt10": 0.902688816176951 + }, + "top10": [ + "192-S1176", + "192-H1994", + "192-S1159", + "192-H2043", + "192-S1213", + "192-S1179", + "192-H1971", + "192-H1964", + "192-H1959", + "192-S805" + ], + "relevantCount": 4 + } + ] +} diff --git a/tests/search-eval/corpus/bills/docs.jsonl.gz b/tests/search-eval/corpus/bills/docs.jsonl.gz new file mode 100644 index 000000000..315f6bfbd Binary files /dev/null and b/tests/search-eval/corpus/bills/docs.jsonl.gz differ diff --git a/tests/search-eval/corpus/bills/meta.json b/tests/search-eval/corpus/bills/meta.json new file mode 100644 index 000000000..7cd1b7cdc --- /dev/null +++ b/tests/search-eval/corpus/bills/meta.json @@ -0,0 +1,17 @@ +{ + "alias": "bills", + "source": "tests/integration/exportedTestData", + "count": 7337, + "courts": { + "192": 7334, + "193": 3 + }, + "jsonlMd5": "daf353d4bb28eec428b433d37ccf6d8a", + "enriched": [ + { + "field": "summary", + "source": "prod", + "count": 7034 + } + ] +} diff --git a/tests/search-eval/corpus/bills/schema.json b/tests/search-eval/corpus/bills/schema.json new file mode 100644 index 000000000..2caa150e6 --- /dev/null +++ b/tests/search-eval/corpus/bills/schema.json @@ -0,0 +1,115 @@ +{ + "token_separators": ["-"], + "fields": [ + { + "name": "number", + "type": "string", + "facet": false + }, + { + "name": "numberVariants", + "type": "string[]", + "facet": false + }, + { + "name": "court", + "type": "int32", + "facet": true + }, + { + "name": "title", + "type": "string", + "facet": false + }, + { + "name": "pinslip", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "summary", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "legislationType", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "body", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "city", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "currentCommittee", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "testimonyCount", + "type": "int32" + }, + { + "name": "primarySponsor", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "cosponsors", + "type": "string[]", + "facet": true + }, + { + "name": "cosponsorCount", + "type": "int32" + }, + { + "name": "nextHearingAt", + "type": "int64", + "optional": true + }, + { + "name": "latestTestimonyAt", + "type": "int64", + "optional": true + }, + { + "name": "endorseCount", + "type": "int32" + }, + { + "name": "opposeCount", + "type": "int32" + }, + { + "name": "neutralCount", + "type": "int32" + }, + { + "name": "topics.lvl0", + "type": "string[]", + "facet": true, + "optional": true + }, + { + "name": "topics.lvl1", + "type": "string[]", + "facet": true, + "optional": true + } + ], + "default_sorting_field": "testimonyCount" +} diff --git a/tests/search-eval/corpus/hearings/docs.jsonl.gz b/tests/search-eval/corpus/hearings/docs.jsonl.gz new file mode 100644 index 000000000..5cd7d0dbe Binary files /dev/null and b/tests/search-eval/corpus/hearings/docs.jsonl.gz differ diff --git a/tests/search-eval/corpus/hearings/meta.json b/tests/search-eval/corpus/hearings/meta.json new file mode 100644 index 000000000..18258c528 --- /dev/null +++ b/tests/search-eval/corpus/hearings/meta.json @@ -0,0 +1,11 @@ +{ + "alias": "hearings", + "source": "prod", + "count": 962, + "courts": { + "192": 107, + "193": 389, + "194": 466 + }, + "jsonlMd5": "fa6e9fe42e3da564da0bd72eef1e55aa" +} diff --git a/tests/search-eval/corpus/hearings/schema.json b/tests/search-eval/corpus/hearings/schema.json new file mode 100644 index 000000000..028b76b3e --- /dev/null +++ b/tests/search-eval/corpus/hearings/schema.json @@ -0,0 +1,91 @@ +{ + "token_separators": ["-"], + "fields": [ + { + "name": "eventId", + "type": "int32", + "facet": false + }, + { + "name": "title", + "type": "string", + "facet": false + }, + { + "name": "description", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "startsAt", + "type": "int64", + "facet": false + }, + { + "name": "month", + "type": "string", + "facet": true + }, + { + "name": "year", + "type": "int32", + "facet": true + }, + { + "name": "committeeCode", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "committeeName", + "type": "string", + "facet": true, + "optional": true + }, + { + "name": "locationName", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "locationCity", + "type": "string", + "facet": false, + "optional": true + }, + { + "name": "chairNames", + "type": "string[]", + "facet": true + }, + { + "name": "agendaTopics", + "type": "string[]", + "facet": false + }, + { + "name": "billNumbers", + "type": "string[]", + "facet": false + }, + { + "name": "billSlugs", + "type": "string[]", + "facet": false + }, + { + "name": "court", + "type": "int32", + "facet": true + }, + { + "name": "hasVideo", + "type": "bool", + "facet": true + } + ], + "default_sorting_field": "startsAt" +} diff --git a/tests/search-eval/corpus/publishedTestimony/docs.jsonl.gz b/tests/search-eval/corpus/publishedTestimony/docs.jsonl.gz new file mode 100644 index 000000000..742c03931 Binary files /dev/null and b/tests/search-eval/corpus/publishedTestimony/docs.jsonl.gz differ diff --git a/tests/search-eval/corpus/publishedTestimony/meta.json b/tests/search-eval/corpus/publishedTestimony/meta.json new file mode 100644 index 000000000..b9918f8e5 --- /dev/null +++ b/tests/search-eval/corpus/publishedTestimony/meta.json @@ -0,0 +1,13 @@ +{ + "alias": "publishedTestimony", + "source": "prod", + "count": 1056, + "courts": { + "192": 2, + "193": 487, + "194": 567 + }, + "jsonlMd5": "2f6c3a70929df36f4ffa373a44486cf7", + "orderBy": "publishedAt:desc", + "redacted": ["authorUid", "fullName"] +} diff --git a/tests/search-eval/corpus/publishedTestimony/schema.json b/tests/search-eval/corpus/publishedTestimony/schema.json new file mode 100644 index 000000000..7673c5d25 --- /dev/null +++ b/tests/search-eval/corpus/publishedTestimony/schema.json @@ -0,0 +1,71 @@ +{ + "token_separators": ["-"], + "fields": [ + { + "name": "billId", + "type": "string", + "facet": true + }, + { + "name": "billIdVariants", + "type": "string[]", + "facet": false + }, + { + "name": "court", + "type": "int32", + "facet": true + }, + { + "name": "position", + "type": "string", + "facet": true + }, + { + "name": "content", + "type": "string", + "facet": false + }, + { + "name": "authorUid", + "type": "string", + "facet": false + }, + { + "name": "authorRole", + "type": "string", + "facet": true + }, + { + "name": "authorDisplayName", + "type": "string", + "facet": true + }, + { + "name": "fullName", + "type": "string", + "facet": false + }, + { + "name": "version", + "type": "int32", + "facet": false + }, + { + "name": "public", + "type": "bool", + "facet": false + }, + { + "name": "publishedAt", + "type": "int64", + "facet": false + }, + { + "name": "updatedAt", + "type": "int64", + "facet": false + } + ], + "default_sorting_field": "publishedAt" +} diff --git a/tests/search-eval/goldens/bills.json b/tests/search-eval/goldens/bills.json new file mode 100644 index 000000000..6a3df72ac --- /dev/null +++ b/tests/search-eval/goldens/bills.json @@ -0,0 +1,1446 @@ +{ + "collection": "bills", + "defaults": { + "sort_by": "_eval(legislationType:!=`Order` && legislationType:!=`Extension Order`):desc,_text_match:desc,testimonyCount:desc", + "courtFilter": true + }, + "queries": [ + { + "id": "exact-001", + "category": "exact-bill-id", + "query": "H.100", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H100" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-002", + "category": "exact-bill-id", + "query": "h100", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H100" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-003", + "category": "exact-bill-id", + "query": "H 100", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H100" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-004", + "category": "exact-bill-id", + "query": "S.9", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S9" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-005", + "category": "exact-bill-id", + "query": "s9", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S9" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-006", + "category": "exact-bill-id", + "query": "S 2564", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S2564" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-007", + "category": "exact-bill-id", + "query": "S2564", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S2564" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-008", + "category": "exact-bill-id", + "query": "H.4374", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H4374" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-009", + "category": "exact-bill-id", + "query": "h.73", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H73" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-010", + "category": "exact-bill-id", + "query": "H3999", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H3999" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-011", + "category": "exact-bill-id", + "query": "S.1333", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S1333" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-012", + "category": "exact-bill-id", + "query": "s 2475", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S2475" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-013", + "category": "exact-bill-id", + "query": "H600", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H600" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-014", + "category": "exact-bill-id", + "query": "H.1", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "H1" + }, + "grade": 3 + } + ] + }, + { + "id": "exact-015", + "category": "exact-bill-id", + "query": "S.1563", + "relevant": [ + { + "rule": { + "field": "number", + "equals": "S1563" + }, + "grade": 3 + } + ] + }, + { + "id": "mc-001", + "category": "member-committee", + "query": "Diana DiZoglio", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Diana DiZoglio" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-002", + "category": "member-committee", + "query": "Bruce Tarr", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Bruce E. Tarr" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-003", + "category": "member-committee", + "query": "Jason Lewis", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Jason M. Lewis" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-004", + "category": "member-committee", + "query": "Tackey Chan", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Tackey Chan" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-005", + "category": "member-committee", + "query": "Julian Cyr", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Julian Cyr" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-006", + "category": "member-committee", + "query": "Cynthia Creem", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Cynthia Stone Creem" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-007", + "category": "member-committee", + "query": "Marjorie Decker", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Marjorie C. Decker" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-008", + "category": "member-committee", + "query": "Michael Moore", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Michael O. Moore" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-009", + "category": "member-committee", + "query": "Patrick O'Connor", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Patrick M. O'Connor" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-010", + "category": "member-committee", + "query": "Aaron Michlewitz", + "relevant": [ + { + "rule": { + "anyField": ["primarySponsor", "cosponsors"], + "equals": "Aaron Michlewitz" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-011", + "category": "member-committee", + "query": "Ways and Means Committee", + "relevant": [ + { + "rule": { + "field": "currentCommittee", + "contains": "Ways and Means" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-012", + "category": "member-committee", + "query": "Health Care Financing", + "relevant": [ + { + "rule": { + "field": "currentCommittee", + "contains": "Health Care Financing" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-013", + "category": "member-committee", + "query": "Committee on Revenue", + "relevant": [ + { + "rule": { + "field": "currentCommittee", + "contains": "Committee on Revenue" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-014", + "category": "member-committee", + "query": "Education Committee", + "relevant": [ + { + "rule": { + "field": "currentCommittee", + "contains": "Education" + }, + "grade": 1 + } + ] + }, + { + "id": "mc-015", + "category": "member-committee", + "query": "Telecommunications Utilities and Energy", + "relevant": [ + { + "rule": { + "field": "currentCommittee", + "contains": "Telecommunications" + }, + "grade": 1 + } + ] + }, + { + "id": "topic-001", + "category": "topic", + "query": "gun control", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "firearm" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "gun" + }, + "grade": 1 + }, + { + "docId": "192-H4038", + "grade": 3 + }, + { + "docId": "192-S1563", + "grade": 3 + } + ] + }, + { + "id": "topic-002", + "category": "topic", + "query": "mental health", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "mental health" + }, + "grade": 1 + }, + { + "docId": "192-H1090", + "grade": 3 + }, + { + "docId": "192-H1039", + "grade": 3 + } + ] + }, + { + "id": "topic-003", + "category": "topic", + "query": "climate change", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "climate" + }, + "grade": 1 + }, + { + "docId": "192-S9", + "grade": 3 + }, + { + "docId": "192-H2159", + "grade": 3 + } + ] + }, + { + "id": "topic-004", + "category": "topic", + "query": "offshore wind", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "offshore wind" + }, + "grade": 1 + }, + { + "docId": "192-H3302", + "grade": 3 + }, + { + "docId": "192-H3310", + "grade": 3 + } + ] + }, + { + "id": "topic-005", + "category": "topic", + "query": "marijuana", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "marijuana" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "cannabis" + }, + "grade": 1 + }, + { + "docId": "192-H154", + "grade": 3 + }, + { + "docId": "192-H145", + "grade": 3 + } + ] + }, + { + "id": "topic-006", + "category": "topic", + "query": "minimum wage", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "minimum wage" + }, + "grade": 1 + }, + { + "docId": "192-S1176", + "grade": 3 + }, + { + "docId": "192-H2043", + "grade": 3 + }, + { + "docId": "192-H2013", + "grade": 1 + } + ] + }, + { + "id": "topic-007", + "category": "topic", + "query": "charter schools", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "charter school" + }, + "grade": 1 + }, + { + "docId": "192-H712", + "grade": 3 + }, + { + "docId": "192-H696", + "grade": 3 + }, + { + "docId": "192-H676", + "grade": 1 + }, + { + "docId": "192-H598", + "grade": 1 + } + ] + }, + { + "id": "topic-008", + "category": "topic", + "query": "affordable housing", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "affordable housing" + }, + "grade": 1 + }, + { + "docId": "192-H1384", + "grade": 3 + }, + { + "docId": "192-H1386", + "grade": 3 + } + ] + }, + { + "id": "topic-009", + "category": "topic", + "query": "homelessness", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "homeless" + }, + "grade": 1 + }, + { + "docId": "192-H1436", + "grade": 3 + }, + { + "docId": "192-H1385", + "grade": 3 + } + ] + }, + { + "id": "topic-010", + "category": "topic", + "query": "opioid addiction", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "opioid" + }, + "grade": 1 + }, + { + "docId": "192-H2086", + "grade": 3 + } + ] + }, + { + "id": "topic-011", + "category": "topic", + "query": "student loan forgiveness", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "student loan" + }, + "grade": 1 + }, + { + "docId": "192-H1320", + "grade": 3 + } + ] + }, + { + "id": "topic-012", + "category": "topic", + "query": "domestic violence", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "domestic violence" + }, + "grade": 1 + }, + { + "docId": "192-H1643", + "grade": 3 + }, + { + "docId": "192-H1510", + "grade": 3 + } + ] + }, + { + "id": "topic-013", + "category": "topic", + "query": "sexual assault", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "sexual assault" + }, + "grade": 1 + }, + { + "docId": "192-H1644", + "grade": 3 + }, + { + "docId": "192-H1701", + "grade": 3 + } + ] + }, + { + "id": "topic-014", + "category": "topic", + "query": "solar energy", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "solar" + }, + "grade": 1 + }, + { + "docId": "192-H3260", + "grade": 3 + }, + { + "docId": "192-H2164", + "grade": 3 + } + ] + }, + { + "id": "topic-015", + "category": "topic", + "query": "eviction", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "evict" + }, + "grade": 1 + }, + { + "docId": "192-H1808", + "grade": 3 + }, + { + "docId": "192-H1911", + "grade": 3 + } + ] + }, + { + "id": "topic-016", + "category": "topic", + "query": "vaccines", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "vaccin" + }, + "grade": 1 + }, + { + "docId": "192-H2411", + "grade": 3 + } + ] + }, + { + "id": "topic-017", + "category": "topic", + "query": "telehealth", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "telehealth" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "telemedicine" + }, + "grade": 1 + }, + { + "docId": "192-H1101", + "grade": 3 + } + ] + }, + { + "id": "topic-018", + "category": "topic", + "query": "voting by mail", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "voting by mail" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "mail in voting" + }, + "grade": 1 + }, + { + "docId": "192-H73", + "grade": 3 + }, + { + "docId": "192-H787", + "grade": 3 + } + ] + }, + { + "id": "topic-019", + "category": "topic", + "query": "school lunch", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "student nutrition" + }, + "grade": 1 + }, + { + "docId": "192-H3999", + "grade": 3 + }, + { + "docId": "192-H715", + "grade": 3 + } + ] + }, + { + "id": "topic-020", + "category": "topic", + "query": "renewable energy", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "renewable" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-001", + "category": "synonym", + "query": "firearms", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "firearm" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "gun" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-002", + "category": "synonym", + "query": "guns", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "firearm" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "gun" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-003", + "category": "synonym", + "query": "educator", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "educator" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "teacher" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-004", + "category": "synonym", + "query": "teachers", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "educator" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "teacher" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-005", + "category": "synonym", + "query": "cannabis", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "cannabis" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "marijuana" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-006", + "category": "synonym", + "query": "doctors", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "physician" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-007", + "category": "synonym", + "query": "elderly", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "elderly" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "older adult" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-008", + "category": "synonym", + "query": "senior citizens", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "senior citizen" + }, + "grade": 1 + }, + { + "rule": { + "field": "title", + "contains": "elderly" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-009", + "category": "synonym", + "query": "liquor", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "liquor" + }, + "grade": 3 + }, + { + "rule": { + "field": "title", + "contains": "alcoholic beverage" + }, + "grade": 3 + }, + { + "rule": { + "field": "title", + "contains": "alcohol" + }, + "grade": 1 + } + ] + }, + { + "id": "syn-010", + "category": "synonym", + "query": "childcare", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "childcare" + }, + "grade": 1 + } + ] + }, + { + "id": "mis-001", + "category": "misspelling", + "query": "mental helth", + "sameAs": "topic-002" + }, + { + "id": "mis-002", + "category": "misspelling", + "query": "clmate change", + "sameAs": "topic-003" + }, + { + "id": "mis-003", + "category": "misspelling", + "query": "marijuanna", + "sameAs": "topic-005" + }, + { + "id": "mis-004", + "category": "misspelling", + "query": "afordable housing", + "sameAs": "topic-008" + }, + { + "id": "mis-005", + "category": "misspelling", + "query": "homelesness", + "sameAs": "topic-009" + }, + { + "id": "mis-006", + "category": "misspelling", + "query": "opiod", + "sameAs": "topic-010" + }, + { + "id": "mis-007", + "category": "misspelling", + "query": "domestic violance", + "sameAs": "topic-012" + }, + { + "id": "mis-008", + "category": "misspelling", + "query": "evicton", + "sameAs": "topic-015" + }, + { + "id": "mis-009", + "category": "misspelling", + "query": "vacines", + "sameAs": "topic-016" + }, + { + "id": "mis-010", + "category": "misspelling", + "query": "minimun wage", + "sameAs": "topic-006" + }, + { + "id": "hyp-001", + "category": "hyphenation", + "query": "low income", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "low income" + }, + "grade": 3 + } + ] + }, + { + "id": "hyp-002", + "category": "hyphenation", + "query": "long term care", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "long term" + }, + "grade": 3 + } + ] + }, + { + "id": "hyp-003", + "category": "hyphenation", + "query": "well being", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "well being" + }, + "grade": 3 + } + ] + }, + { + "id": "hyp-004", + "category": "hyphenation", + "query": "decision making", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "decision making" + }, + "grade": 3 + } + ] + }, + { + "id": "hyp-005", + "category": "hyphenation", + "query": "zero emission vehicles", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "zero emission" + }, + "grade": 3 + } + ] + }, + { + "id": "hyp-006", + "category": "hyphenation", + "query": "post traumatic stress", + "relevant": [ + { + "rule": { + "field": "title", + "contains": "post traumatic" + }, + "grade": 3 + } + ] + }, + { + "id": "pl-001", + "category": "plain-language", + "query": "tenants facing eviction", + "relevant": [ + { + "docId": "192-H1434", + "grade": 3 + }, + { + "docId": "192-S891", + "grade": 3 + }, + { + "docId": "192-H1436", + "grade": 3 + }, + { + "docId": "192-S874", + "grade": 3 + }, + { + "docId": "192-H1440", + "grade": 3 + }, + { + "docId": "192-S889", + "grade": 3 + }, + { + "docId": "192-H1911", + "grade": 3 + }, + { + "docId": "192-H1808", + "grade": 1 + }, + { + "docId": "192-H4505", + "grade": 1 + }, + { + "docId": "192-S921", + "grade": 1 + }, + { + "docId": "192-H1404", + "grade": 1 + }, + { + "docId": "192-H4354", + "grade": 1 + }, + { + "docId": "192-S944", + "grade": 1 + }, + { + "docId": "192-H4148", + "grade": 1 + }, + { + "docId": "192-S2467", + "grade": 1 + }, + { + "docId": "192-H1378", + "grade": 1 + }, + { + "docId": "192-S886", + "grade": 1 + } + ] + }, + { + "id": "pl-002", + "category": "plain-language", + "query": "cap insulin costs", + "relevant": [ + { + "docId": "192-H4034", + "grade": 3 + }, + { + "docId": "192-H1055", + "grade": 3 + }, + { + "docId": "192-S2651", + "grade": 3 + }, + { + "docId": "192-H1132", + "grade": 1 + }, + { + "docId": "192-S691", + "grade": 1 + }, + { + "docId": "192-S2537", + "grade": 1 + }, + { + "docId": "192-H729", + "grade": 1 + }, + { + "docId": "192-S2695", + "grade": 1 + }, + { + "docId": "192-S771", + "grade": 1 + }, + { + "docId": "192-S1991", + "grade": 1 + } + ] + }, + { + "id": "pl-003", + "category": "plain-language", + "query": "lead in school drinking water", + "relevant": [ + { + "docId": "192-H906", + "grade": 3 + }, + { + "docId": "192-S580", + "grade": 3 + }, + { + "docId": "192-H1725", + "grade": 1 + }, + { + "docId": "192-S1056", + "grade": 1 + }, + { + "docId": "192-H939", + "grade": 1 + } + ] + }, + { + "id": "pl-004", + "category": "plain-language", + "query": "property tax break for solar", + "relevant": [ + { + "docId": "192-H2827", + "grade": 3 + }, + { + "docId": "192-H3000", + "grade": 3 + } + ] + }, + { + "id": "pl-005", + "category": "plain-language", + "query": "background checks for gun sales", + "relevant": [ + { + "docId": "192-H3729", + "grade": 3 + }, + { + "docId": "192-H3846", + "grade": 1 + } + ] + }, + { + "id": "pl-006", + "category": "plain-language", + "query": "limit rent increases", + "relevant": [ + { + "docId": "192-H1440", + "grade": 3 + }, + { + "docId": "192-S889", + "grade": 3 + }, + { + "docId": "192-H1404", + "grade": 3 + }, + { + "docId": "192-H4354", + "grade": 3 + }, + { + "docId": "192-H1378", + "grade": 1 + }, + { + "docId": "192-S886", + "grade": 1 + }, + { + "docId": "192-H3721", + "grade": 1 + } + ] + } + ] +} diff --git a/tests/search-eval/goldens/hearings.json b/tests/search-eval/goldens/hearings.json new file mode 100644 index 000000000..074dc6cb2 --- /dev/null +++ b/tests/search-eval/goldens/hearings.json @@ -0,0 +1,608 @@ +{ + "collection": "hearings", + "defaults": { + "sort_by": "_text_match:desc,startsAt:desc", + "courtFilter": true + }, + "queries": [ + { + "id": "hc-001", + "category": "committee", + "query": "Ways and Means", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Ways and Means" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-002", + "category": "committee", + "query": "Judiciary", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Judiciary" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-003", + "category": "committee", + "query": "Financial Services", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Financial Services" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-004", + "category": "committee", + "query": "Public Health", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Public Health" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-005", + "category": "committee", + "query": "Transportation", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Transportation" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-006", + "category": "committee", + "query": "Housing", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Housing" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-007", + "category": "committee", + "query": "Election Laws", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Election Laws" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-008", + "category": "committee", + "query": "Revenue", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Revenue" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-009", + "category": "committee", + "query": "Consumer Protection", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Consumer Protection" + }, + "grade": 3 + } + ] + }, + { + "id": "hc-010", + "category": "committee", + "query": "Municipalities and Regional Government", + "relevant": [ + { + "rule": { + "field": "committeeName", + "contains": "Municipalities" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-001", + "category": "chair", + "query": "Rebecca Rausch", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Rebecca L. Rausch" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-002", + "category": "chair", + "query": "Cindy Friedman", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Cindy F. Friedman" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-003", + "category": "chair", + "query": "Alice Peisch", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Alice Hanlon Peisch" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-004", + "category": "chair", + "query": "Tackey Chan", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Tackey Chan" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-005", + "category": "chair", + "query": "Brendan Crighton", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Brendan P. Crighton" + }, + "grade": 3 + } + ] + }, + { + "id": "hch-006", + "category": "chair", + "query": "Daniel Ryan", + "relevant": [ + { + "rule": { + "field": "chairNames", + "equals": "Daniel J. Ryan" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-001", + "category": "bill-on-agenda", + "query": "H2391", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "H2391" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-002", + "category": "bill-on-agenda", + "query": "H 2391", + "sameAs": "hb-001" + }, + { + "id": "hb-003", + "category": "bill-on-agenda", + "query": "H.2391", + "sameAs": "hb-001" + }, + { + "id": "hb-004", + "category": "bill-on-agenda", + "query": "S1489", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "S1489" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-005", + "category": "bill-on-agenda", + "query": "S 1489", + "sameAs": "hb-004" + }, + { + "id": "hb-006", + "category": "bill-on-agenda", + "query": "H63", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "H63" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-007", + "category": "bill-on-agenda", + "query": "H 63", + "sameAs": "hb-006" + }, + { + "id": "hb-008", + "category": "bill-on-agenda", + "query": "S6", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "S6" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-009", + "category": "bill-on-agenda", + "query": "H56", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "H56" + }, + "grade": 3 + } + ] + }, + { + "id": "hb-010", + "category": "bill-on-agenda", + "query": "H.2424", + "relevant": [ + { + "rule": { + "field": "billNumbers", + "equals": "H2424" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-001", + "category": "agenda-topic", + "query": "home rule petitions", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Home Rule" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-002", + "category": "agenda-topic", + "query": "budget hearing", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Budget Hearing" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-003", + "category": "agenda-topic", + "query": "miscellaneous matters", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Miscellaneous" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-004", + "category": "agenda-topic", + "query": "disabilities", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "disabilit" + }, + "grade": 1 + }, + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "disabled" + }, + "grade": 1 + } + ] + }, + { + "id": "ht-005", + "category": "agenda-topic", + "query": "public safety", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Public Safety" + }, + "grade": 1 + } + ] + }, + { + "id": "ht-006", + "category": "agenda-topic", + "query": "late files", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Late File" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-007", + "category": "agenda-topic", + "query": "health care cost growth benchmark", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "Cost Growth" + }, + "grade": 3 + } + ] + }, + { + "id": "ht-008", + "category": "agenda-topic", + "query": "climate", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "climate" + }, + "grade": 1 + } + ] + }, + { + "id": "hl-001", + "category": "location", + "query": "Gardner Auditorium", + "relevant": [ + { + "rule": { + "field": "locationName", + "contains": "Gardner" + }, + "grade": 3 + } + ] + }, + { + "id": "hl-002", + "category": "location", + "query": "Virtual Hearing", + "relevant": [ + { + "rule": { + "field": "locationName", + "contains": "Virtual" + }, + "grade": 3 + } + ] + }, + { + "id": "hl-003", + "category": "location", + "query": "Written Testimony Only", + "relevant": [ + { + "rule": { + "field": "locationName", + "contains": "Written Testimony" + }, + "grade": 3 + } + ] + }, + { + "id": "hl-004", + "category": "location", + "query": "UMass Amherst", + "relevant": [ + { + "rule": { + "field": "locationName", + "contains": "UMass" + }, + "grade": 3 + } + ] + }, + { + "id": "hl-005", + "category": "location", + "query": "Northampton", + "relevant": [ + { + "rule": { + "anyField": ["locationName", "locationCity"], + "contains": "Northampton" + }, + "grade": 3 + } + ] + }, + { + "id": "hs-001", + "category": "synonym", + "query": "cannabis", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "marijuana" + }, + "grade": 3 + } + ] + }, + { + "id": "hs-002", + "category": "synonym", + "query": "guns", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "firearm" + }, + "grade": 3 + } + ] + }, + { + "id": "hs-003", + "category": "synonym", + "query": "liquor", + "relevant": [ + { + "rule": { + "anyField": ["agendaTopics", "description"], + "contains": "alcohol" + }, + "grade": 3 + } + ] + }, + { + "id": "hm-001", + "category": "misspelling", + "query": "judicary", + "sameAs": "hc-002" + }, + { + "id": "hm-002", + "category": "misspelling", + "query": "transportaton", + "sameAs": "hc-005" + }, + { + "id": "hm-003", + "category": "misspelling", + "query": "gardner auditorum", + "sameAs": "hl-001" + }, + { + "id": "hm-004", + "category": "misspelling", + "query": "municipalites", + "sameAs": "hc-010" + }, + { + "id": "hm-005", + "category": "misspelling", + "query": "consumer protecton", + "sameAs": "hc-009" + }, + { + "id": "hm-006", + "category": "misspelling", + "query": "cindy freidman", + "sameAs": "hch-002" + } + ] +} diff --git a/tests/search-eval/goldens/publishedTestimony.json b/tests/search-eval/goldens/publishedTestimony.json new file mode 100644 index 000000000..bb0f2c3e3 --- /dev/null +++ b/tests/search-eval/goldens/publishedTestimony.json @@ -0,0 +1,532 @@ +{ + "collection": "publishedTestimony", + "defaults": { + "sort_by": "_text_match:desc,publishedAt:desc", + "courtFilter": true + }, + "queries": [ + { + "id": "tb-001", + "category": "bill-id", + "query": "S531", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "S531" + }, + "grade": 3 + } + ] + }, + { + "id": "tb-002", + "category": "bill-id", + "query": "S 531", + "sameAs": "tb-001" + }, + { + "id": "tb-003", + "category": "bill-id", + "query": "S.531", + "sameAs": "tb-001" + }, + { + "id": "tb-004", + "category": "bill-id", + "query": "H4351", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "H4351" + }, + "grade": 3 + } + ] + }, + { + "id": "tb-005", + "category": "bill-id", + "query": "H 4351", + "sameAs": "tb-004" + }, + { + "id": "tb-006", + "category": "bill-id", + "query": "S505", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "S505" + }, + "grade": 3 + } + ] + }, + { + "id": "tb-007", + "category": "bill-id", + "query": "H2057", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "H2057" + }, + "grade": 3 + } + ] + }, + { + "id": "tb-008", + "category": "bill-id", + "query": "H.1405", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "H1405" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-001", + "category": "author", + "query": "Progressive Massachusetts", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "equals": "Progressive Massachusetts" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-002", + "category": "author", + "query": "League of Women Voters", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "equals": "League of Women Voters of Massachusetts" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-003", + "category": "author", + "query": "T Stephen Jones", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "equals": "T Stephen Jones" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-004", + "category": "author", + "query": "Elaine Almquist", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "equals": "Elaine F. Almquist" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-005", + "category": "author", + "query": "Partners in Democracy", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "equals": "Partners in Democracy" + }, + "grade": 3 + } + ] + }, + { + "id": "ta-006", + "category": "author", + "query": "Commission on the Status of Women", + "relevant": [ + { + "rule": { + "field": "authorDisplayName", + "contains": "Commission on the Status of Women" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-001", + "category": "topic", + "query": "ranked choice voting", + "relevant": [ + { + "rule": { + "field": "billId", + "equals": "S531" + }, + "grade": 3 + }, + { + "rule": { + "field": "content", + "contains": "ranked choice" + }, + "grade": 1 + } + ] + }, + { + "id": "tt-002", + "category": "topic", + "query": "rent control", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "rent control" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-003", + "category": "topic", + "query": "same day voter registration", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "same day registration" + }, + "grade": 3 + }, + { + "rule": { + "field": "content", + "contains": "same day voter registration" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-004", + "category": "topic", + "query": "vote by mail", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "vote by mail" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-005", + "category": "topic", + "query": "solitary confinement", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "solitary confinement" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-006", + "category": "topic", + "query": "eviction", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "eviction" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-007", + "category": "topic", + "query": "zoning", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "zoning" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-008", + "category": "topic", + "query": "fossil fuels", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "fossil fuel" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-009", + "category": "topic", + "query": "gender affirming care", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "gender affirming" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-010", + "category": "topic", + "query": "language access", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "language access" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-011", + "category": "topic", + "query": "parole", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "parole" + }, + "grade": 3 + } + ] + }, + { + "id": "tt-012", + "category": "topic", + "query": "homelessness", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "homeless" + }, + "grade": 1 + } + ] + }, + { + "id": "ts-001", + "category": "synonym", + "query": "renters", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "tenant" + }, + "grade": 1 + }, + { + "rule": { + "field": "content", + "contains": "renter" + }, + "grade": 1 + } + ] + }, + { + "id": "ts-002", + "category": "synonym", + "query": "tenants", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "tenant" + }, + "grade": 1 + }, + { + "rule": { + "field": "content", + "contains": "renter" + }, + "grade": 1 + } + ] + }, + { + "id": "ts-003", + "category": "synonym", + "query": "teachers", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "educator" + }, + "grade": 1 + }, + { + "rule": { + "field": "content", + "contains": "teacher" + }, + "grade": 1 + } + ] + }, + { + "id": "ts-004", + "category": "synonym", + "query": "doctor", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "physician" + }, + "grade": 1 + }, + { + "rule": { + "field": "content", + "contains": "doctor" + }, + "grade": 1 + } + ] + }, + { + "id": "ts-005", + "category": "synonym", + "query": "servicemember", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "veteran" + }, + "grade": 3 + } + ] + }, + { + "id": "ts-006", + "category": "synonym", + "query": "opiate", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "opioid" + }, + "grade": 3 + } + ] + }, + { + "id": "ts-007", + "category": "synonym", + "query": "clean energy", + "relevant": [ + { + "rule": { + "field": "content", + "contains": "renewable energy" + }, + "grade": 1 + }, + { + "rule": { + "field": "content", + "contains": "clean energy" + }, + "grade": 1 + } + ] + }, + { + "id": "tm-001", + "category": "misspelling", + "query": "progresive massachusets", + "sameAs": "ta-001" + }, + { + "id": "tm-002", + "category": "misspelling", + "query": "leage of women voters", + "sameAs": "ta-002" + }, + { + "id": "tm-003", + "category": "misspelling", + "query": "rankd choice voting", + "sameAs": "tt-001" + }, + { + "id": "tm-004", + "category": "misspelling", + "query": "evicton", + "sameAs": "tt-006" + }, + { + "id": "tm-005", + "category": "misspelling", + "query": "zonning", + "sameAs": "tt-007" + }, + { + "id": "tm-006", + "category": "misspelling", + "query": "solitary confinment", + "sameAs": "tt-005" + } + ] +} diff --git a/yarn.lock b/yarn.lock index 03bd7ee1c..9c3e3815a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6168,13 +6168,6 @@ axe-core@^4.10.0: resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.10.0.tgz" integrity sha512-Mr2ZakwQ7XUAjp7pAwQWRhhK8mQQ6JAaNWSjmjxil0R8BPioMtQsTLOolGYkji1rcL++3dCqZA3zWqpT+9Ew6g== -axios@^0.26.0: - version "0.26.1" - resolved "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz" - integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== - dependencies: - follow-redirects "^1.14.8" - axios@^0.27.2: version "0.27.2" resolved "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz" @@ -6183,6 +6176,16 @@ axios@^0.27.2: follow-redirects "^1.14.9" form-data "^4.0.0" +axios@^1.15.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.19.0.tgz#ddf864d4c8233c0e6873746ab59361537d05ad39" + integrity sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw== + dependencies: + follow-redirects "^1.16.0" + form-data "^4.0.6" + https-proxy-agent "^5.0.1" + proxy-from-env "^2.1.0" + axobject-query@^3.2.1: version "3.2.1" resolved "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz" @@ -9769,11 +9772,16 @@ fn.name@1.x.x: resolved "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.8, follow-redirects@^1.14.9: +follow-redirects@^1.14.9: version "1.15.3" resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz" integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== +follow-redirects@^1.16.0: + version "1.16.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" + integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== + for-each@^0.3.3: version "0.3.3" resolved "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz" @@ -9838,6 +9846,17 @@ form-data@^4.0.1: hasown "^2.0.2" mime-types "^2.1.12" +form-data@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.6.tgz#28e864e1b786dbebb68db1f452f9635278665827" + integrity sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.4" + mime-types "^2.1.35" + forwarded@0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" @@ -10521,6 +10540,13 @@ hasown@^2.0.1, hasown@^2.0.2: dependencies: function-bind "^1.1.2" +hasown@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.4.tgz#8c62d8cb90beb2aad5d0a5b67581ad9854c3f003" + integrity sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A== + dependencies: + function-bind "^1.1.2" + hast-util-whitespace@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.1.tgz" @@ -12679,10 +12705,10 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" -loglevel@^1.8.0: - version "1.8.1" - resolved "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz" - integrity sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg== +loglevel@^1.9.2: + version "1.9.2" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" + integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== logrocket@^3.0.1: version "3.0.1" @@ -14982,6 +15008,11 @@ proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== +proxy-from-env@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" + integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== + psl@^1.1.33: version "1.9.0" resolved "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz" @@ -17573,7 +17604,7 @@ tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.4 resolved "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^2.8.0: +tslib@^2.8.0, tslib@^2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== @@ -17759,20 +17790,21 @@ typescript@5.3.3: resolved "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz" integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== -typesense-instantsearch-adapter@^2.4.0: - version "2.7.1" - resolved "https://registry.npmjs.org/typesense-instantsearch-adapter/-/typesense-instantsearch-adapter-2.7.1.tgz" - integrity sha512-GSEwkNzXoVcL4iSV/LDiZvjx/knctqfGLLKUNRe5twgzFprnoU07UUh2mQ5Z0/aP1moY/UpBF1bMKhTU6ECoAQ== +typesense-instantsearch-adapter@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/typesense-instantsearch-adapter/-/typesense-instantsearch-adapter-3.0.2.tgz#5a2fafd88eaeb5b9acab7d83a2879e0dc5969d7e" + integrity sha512-xVIXnzLAU4YsdTwJeRZqzPYeGOi1DHwFKBdTmsvHipZuOUfjSl9Ma8YUCBhjBeAkBkGuYEhEEPiiocr7g0WrfA== dependencies: - typesense "^1.7.0" + typesense "^3.0.3" -typesense@^1.2.2, typesense@^1.7.0: - version "1.7.2" - resolved "https://registry.npmjs.org/typesense/-/typesense-1.7.2.tgz" - integrity sha512-hgQESOiyNJq+w2mpRJa/a1UMhWtJ/+sb0p7NoeCDSkikm9sasisJdnc7uhQchM6vTWKw2sMLWUBNbAhItR6zUQ== +typesense@^3.0.3, typesense@^3.0.6: + version "3.0.6" + resolved "https://registry.yarnpkg.com/typesense/-/typesense-3.0.6.tgz#41a4094d26b214bb8c1be52e7824eccf728764e8" + integrity sha512-d3LL1qOLS8FCRxgAOqH+uDuK+VVA+/HYI1frU9fjYVwOg68mg/dX6XTtZ4yKKAkDCasB/se5uh/ZpMdOz/uJNg== dependencies: - axios "^0.26.0" - loglevel "^1.8.0" + axios "^1.15.0" + loglevel "^1.9.2" + tslib "^2.8.1" ufo@^1.3.2: version "1.4.0"