Some search improvements

.github/workflows/repo-checks.yml

@@ -49,7 +49,8 @@ jobs:
 
     services:
       typesense:
-        image: typesense/typesense:0.24.0
+        # Keep in sync with infra/Dockerfile.search
+        image: typesense/typesense:30.2
         ports:
           - 8108:8108
         env:

.gitignore

@@ -102,3 +102,9 @@ CLAUDE.md
 #gcloud
 .gcloudignore
 
+# search-eval: every corpus is committed, because none of them can be rebuilt
+# byte-for-byte from what the repo holds — bills now joins `summary` in from a
+# live project on top of the emulator fixture, and hearings/publishedTestimony
+# come from prod outright. See tests/search-eval/README.md. Labeling sheets are
+# authoring scratch.
+tests/search-eval/labeling-sheet-*.md

components/search/SortBy.tsx

@@ -42,7 +42,11 @@ export type SortByWithConfigurationItem = SortByItem & {
 
 export const SortBy = ({ items }: { items: SortByWithConfigurationItem[] }) => {
   const sortBy = useSortBy({ items }),
-    selected = items.find(i => i.value === sortBy.currentRefinement)!
+    // A routed URL can restore a sort value that no longer exists — an option
+    // renamed since the link was shared. connectSortBy passes it through with
+    // only a dev-mode warning, so fall back to the default option instead of
+    // crashing the page on `.configure` of undefined.
+    selected = items.find(i => i.value === sortBy.currentRefinement) ?? items[0]
   useConfigure(selected.configure ?? {})
   return (
     <StyledSelect

components/search/bills/BillHit.tsx

@@ -1,4 +1,4 @@
-import { Highlight } from "react-instantsearch"
+import { Highlight, Snippet } from "react-instantsearch"
 import {
   faCheckCircle,
   faMinusCircle,
@@ -19,6 +19,11 @@ import { useTranslation } from "next-i18next"
 type BillRecord = {
   number: string
   title: string
+  /** The clerk's petition blurb — see functions/src/bills/search.ts. */
+  pinslip?: string
+  /** The LLM plain-language description — see functions/src/bills/search.ts.
+   * Absent on procedural orders, and on bills the enrichment has not reached. */
+  summary?: string
   city?: string
   court: number
   currentCommittee?: string
@@ -133,9 +138,33 @@ export const DisplayUpcomingHearing = ({
   return null
 }
 
+/** Typesense omits a field from `highlight` when the query did not match it,
+ * and the adapter then falls back to the raw value — so an unguarded
+ * `<Snippet>` would print the petition boilerplate ("By Mr. X of Y, a petition
+ * (accompanied by bill, House, No. N)…") under every result. Show a snippet
+ * only where it explains why this bill is in the list.
+ */
+const matched = (hit: Hit<BillRecord>, attribute: "summary" | "pinslip") => {
+  // A plain string attribute snippets to a single result, never an array.
+  const snippet = hit._snippetResult?.[attribute]
+  return !!snippet && !Array.isArray(snippet) && snippet.matchLevel !== "none"
+}
+
+/** Which blurb to show under the title, or nothing — in priority order, so the
+ * summary wins where both matched: it says what the bill does in plain
+ * language, where the pinslip is the clerk's procedural note. The fallback is
+ * not a rare path — roughly one bill in four on the current court has no
+ * summary yet, because the trigger in llm/ needs DocumentText the scrape has
+ * not always captured (#8), and procedural orders are excluded by the converter
+ * on purpose.
+ */
+const snippetAttribute = (hit: Hit<BillRecord>) =>
+  (["summary", "pinslip"] as const).find(attribute => matched(hit, attribute))
+
 export const BillHit = ({ hit }: { hit: Hit<BillRecord> }) => {
   const url = maple.bill({ id: hit.number, court: hit.court })
   const hearingDate = hit.nextHearingAt && hit.nextHearingAt / 1000 // convert to seconds
+  const snippet = snippetAttribute(hit)
   const { t } = useTranslation("common")
 
   return (
@@ -159,6 +188,11 @@ export const BillHit = ({ hit }: { hit: Hit<BillRecord> }) => {
                   {formatBillId(hit.number)} -{" "}
                   <Highlight attribute="title" hit={hit} />
                 </Card.Title>
+                {snippet && (
+                  <div className="mt-1 text-muted">
+                    <Snippet attribute={snippet} hit={hit} />
+                  </div>
+                )}
                 <div className="d-flex justify-content-between flex-column">
                   <span className="blurb">
                     {(() => {

components/search/bills/BillSearch.tsx

@@ -23,16 +23,14 @@ import { BillHit } from "./BillHit"
 import { useBillRefinements } from "./useBillRefinements"
 import { SortBy, SortByWithConfigurationItem } from "../SortBy"
 import { getServerConfig, VirtualFilters } from "../common"
+import { billsSearchParams } from "../searchParams"
 import { useBillSort } from "./useBillSort"
 import { FC, useState } from "react"
 import { pathToSearchState, searchStateToUrl } from "../routingHelpers"
 
 const searchClient = new TypesenseInstantSearchAdapter({
   server: getServerConfig(),
-  additionalSearchParameters: {
-    query_by: "number,title,body",
-    exclude_fields: "body"
-  }
+  additionalSearchParameters: billsSearchParams
 }).searchClient
 
 const extractLastSegmentOfRefinements = (items: any[]) => {

components/search/bills/useBillRefinements.tsx

@@ -37,6 +37,11 @@ export const useBillRefinements = () => {
             attribute: "court"
           },
           { attribute: "currentCommittee" },
+          /** Bills outrank procedural documents by default (see
+           * billsRelevanceSort in ../searchParams.ts, and #95). This is how a
+           * searcher who actually wants an Extension Order gets back to it —
+           * refining to the type lifts the whole demoted class. */
+          { attribute: "legislationType" },
           { attribute: "city" },
           { attribute: "primarySponsor" },
           { attribute: "cosponsors" }

components/search/bills/useBillSort.tsx

@@ -1,6 +1,7 @@
 import { useMemo, useRef } from "react"
 import { SortByWithConfigurationItem } from "../SortBy"
 import { useTranslation } from "next-i18next"
+import { billsRelevanceSort } from "../searchParams"
 
 export const useBillSort = () => {
   const now = useRef(new Date().getTime())
@@ -16,7 +17,7 @@ export const useBillSort = () => {
       },
       {
         label: t("sort_by.relevance"),
-        value: "bills/sort/_text_match:desc,testimonyCount:desc"
+        value: `bills/sort/${billsRelevanceSort}`
       },
       {
         label: t("sort_by.testimony_count"),

components/search/common.tsx

@@ -41,6 +41,7 @@ export function VirtualFilters({ type }: { type: "bill" | "testimony" }) {
       : [
           "court",
           "currentCommittee",
+          "legislationType",
           "city",
           "primarySponsor",
           "cosponsors",

components/search/hearings/HearingSearch.tsx

@@ -1,40 +1,27 @@
 import { Hit } from "instantsearch.js"
 import { useInstantSearch } from "react-instantsearch"
-import { SearchPage } from "../shared"
+import { SearchPage, SortOptionInput } from "../shared"
 import { HearingHit } from "./HearingHit"
 import {
   CURRENT_COURT_NUMBER,
   formatCourtFilterLabel,
   formatCourtSubtitle
 } from "../courtSessions"
 import { useMemo, useRef } from "react"
-
-/* carbon copy of type in functions/src/hearings/search.ts */
-type HearingSearchRecord = {
-  id: string
-  eventId: number
-  title: string
-  description?: string
-  startsAt: number
-  month: string
-  year: number
-  committeeCode?: string
-  committeeName?: string
-  locationName?: string
-  locationCity?: string
-  chairNames: string[]
-  agendaTopics: string[]
-  billNumbers: string[]
-  billSlugs: string[]
-  court: number
-  hasVideo: boolean
-}
+import type { HearingSearchRecord } from "functions/src/hearings/types"
+import { hearingsRelevanceSort, hearingsSearchParams } from "../searchParams"
 
 export type HearingHitData = Hit<HearingSearchRecord>
 
-const useHearingSort = () => {
+/** Relevance searches every hearing, past and upcoming. Clearing the window
+ * explicitly is load-bearing: SortBy calls useConfigure(selected.configure ?? {}),
+ * so the previously selected option's startsAt bound has to be overwritten.
+ */
+const noTimeWindow: SortOptionInput["configure"] = { numericRefinements: {} }
+
+const useHearingSort = (): SortOptionInput[] => {
   const now = useRef(new Date().getTime())
-  return useMemo(
+  return useMemo<SortOptionInput[]>(
     () => [
       {
         labelKey: "sort_by.past_newest",
@@ -59,15 +46,24 @@ const useHearingSort = () => {
         }
       },
       {
+        // "upcoming" already owns startsAt:asc, and react-instantsearch needs a
+        // unique index name per sort option — eventId is the tiebreak that
+        // makes this one distinct without changing the ordering users see.
         labelKey: "sort_by.past_oldest",
-        value: "hearings/sort/startsAt:asc,startsAt:asc",
+        value: "hearings/sort/startsAt:asc,eventId:asc",
         configure: {
           numericRefinements: {
             startsAt: {
               "<=": [now.current]
             }
           }
         }
+      },
+      {
+        // The only sort under which text ranking is observable.
+        labelKey: "sort_by.relevance",
+        value: `hearings/sort/${hearingsRelevanceSort}`,
+        configure: noTimeWindow
       }
     ],
     []
@@ -89,11 +85,7 @@ export const HearingSearch = () => {
           }
         }
       }}
-      searchParameters={{
-        query_by:
-          "title,description,agendaTopics,billNumbers,chairNames,locationName,locationCity",
-        sort_by: "startsAt:asc"
-      }}
+      searchParameters={hearingsSearchParams}
       hitComponent={HearingHit}
       filterPanelConfig={{
         filters: [