diff --git a/composer.json b/composer.json index a1adda516d3..4779da4bd58 100644 --- a/composer.json +++ b/composer.json @@ -55,13 +55,16 @@ "pestphp/pest": "^4.7", "phpstan/phpstan": "^2.2", "phpunit/phpunit": "^12.5.23", - "spatie/laravel-ray": "^1.43.6" + "spatie/laravel-ray": "^1.43.6", + "typesense/typesense-php": "^6.0" }, "conflict": { - "algolia/algoliasearch-client-php": "<4.32" + "algolia/algoliasearch-client-php": "<4.32", + "typesense/typesense-php": "<6.0" }, "suggest": { - "algolia/algoliasearch-client-php": "Required to use Algolia search driver (^4.32)" + "algolia/algoliasearch-client-php": "Required to use Algolia search driver (^4.32)", + "typesense/typesense-php": "Required to use Typesense search driver (^6.0)" }, "config": { "optimize-autoloader": true, @@ -69,7 +72,8 @@ "sort-packages": true, "allow-plugins": { "composer/package-versions-deprecated": true, - "pestphp/pest-plugin": true + "pestphp/pest-plugin": true, + "php-http/discovery": true } }, "extra": { diff --git a/config/search.php b/config/search.php index 4292708fa75..c4e5c8a57d3 100644 --- a/config/search.php +++ b/config/search.php @@ -63,6 +63,24 @@ ], ], + 'typesense' => [ + 'credentials' => [ + 'api_key' => env('TYPESENSE_API_KEY', ''), + 'search_only_api_key' => env('TYPESENSE_SEARCH_ONLY_API_KEY'), + ], + 'nodes' => [ + [ + 'host' => env('TYPESENSE_HOST', 'localhost'), + 'port' => env('TYPESENSE_PORT', '8108'), + 'path' => env('TYPESENSE_PATH', ''), + 'protocol' => env('TYPESENSE_PROTOCOL', 'http'), + ], + ], + 'client' => [ + 'connection_timeout_seconds' => env('TYPESENSE_CONNECTION_TIMEOUT_SECONDS', 2), + ], + ], + ], /* diff --git a/lang/en/search.php b/lang/en/search.php new file mode 100644 index 00000000000..0104e913074 --- /dev/null +++ b/lang/en/search.php @@ -0,0 +1,17 @@ + 'Build the search index now?', + 'configured' => 'Search index [:index] configured to use [:driver].', + 'connection_failed' => 'Could not connect to the search server.', + 'driver_prompt' => 'Which search driver would you like to use?', + 'embedding_fields_prompt' => 'Fields to use for semantic search', + 'existing_nl_model' => 'Use an existing Typesense model', + 'installing_client' => 'Installing :package...', + 'invalid_algolia_id' => 'Enter a valid Algolia application ID.', + 'invalid_driver' => 'Choose local, algolia, or typesense.', + 'invalid_handle' => 'Use letters, numbers, dashes, and underscores.', + 'invalid_url' => 'Enter an HTTP or HTTPS URL without credentials, a query, or a fragment.', + 'keep_key' => 'Leave blank to keep the configured key', + 'new_nl_model' => 'Configure an OpenAI model', +]; diff --git a/src/Console/Commands/InstallSearch.php b/src/Console/Commands/InstallSearch.php new file mode 100644 index 00000000000..24ae00a0a20 --- /dev/null +++ b/src/Console/Commands/InstallSearch.php @@ -0,0 +1,319 @@ +argument('driver') ?? select( + label: __('statamic::search.driver_prompt'), + options: ['local' => __('Local'), 'algolia' => 'Algolia', 'typesense' => 'Typesense'], + ); + + if (! in_array($driver, ['local', 'algolia', 'typesense'], true)) { + $this->components->error(__('statamic::search.invalid_driver')); + + return self::FAILURE; + } + + $this->index = $this->option('index') ?? text( + label: __('Search index handle'), + default: config('statamic.search.default', 'default'), + required: true, + validate: fn ($value) => preg_match('/^[a-zA-Z0-9_-]+$/', $value) ? null : __('statamic::search.invalid_handle'), + ); + + if (! preg_match('/^[a-zA-Z0-9_-]+$/', $this->index)) { + $this->components->error(__('statamic::search.invalid_handle')); + + return self::FAILURE; + } + + $this->variables = $this->changes = $this->removals = []; + $this->existing = config("statamic.search.indexes.{$this->index}", []); + $this->prefix = strtoupper($driver.($this->index === 'default' ? '' : '_'.$this->index)); + $this->set('driver', $driver); + + if ($this->existing === []) { + $this->set('searchables', $this->index === 'cp' ? ['content', 'users', 'addons'] : 'content'); + $this->set('fields', ['title']); + } + + if (($previous = $this->existing['driver'] ?? null) && $previous !== $driver) { + foreach (['settings', 'credentials', 'client', 'nodes'] as $key) { + $this->removals[] = ['indexes', $this->index, $key]; + } + } + + $configPath = config_path('statamic/search.php'); + $source = File::get(File::exists($configPath) ? $configPath : __DIR__.'/../../../config/search.php'); + + try { + if ($driver === 'algolia') { + $this->configureAlgolia(); + } elseif ($driver === 'typesense') { + $this->configureTypesense(); + } + + $contents = app(ConfigUpdater::class)->render($source, $this->changes, $this->removals); + } catch (\RuntimeException|ConnectionException $e) { + $this->components->error($e instanceof ConnectionException ? __('statamic::search.connection_failed') : $e->getMessage()); + + return self::FAILURE; + } + + $this->installClient($driver); + $this->writeEnvironment(); + File::ensureDirectoryExists(dirname($configPath)); + File::put($configPath, $contents); + $this->callSilent('config:clear'); + $this->components->info(__('statamic::search.configured', ['index' => $this->index, 'driver' => $driver])); + + if ($this->option('build') || confirm(__('statamic::search.build_prompt'), false)) { + // A fresh process loads both the new config and any newly installed SDK. + $result = Process::path(base_path()) + ->env($this->variables) + ->forever() + ->run([ + (new PhpExecutableFinder)->find(false) ?: 'php', + defined('ARTISAN_BINARY') ? ARTISAN_BINARY : 'artisan', + 'statamic:search:update', + $this->index, + ], fn ($type, $output) => $this->output->write($output)); + + return $result->successful() ? self::SUCCESS : self::FAILURE; + } + + return self::SUCCESS; + } + + private function configureAlgolia(): void + { + $config = array_replace_recursive(config('statamic.search.drivers.algolia', []), $this->existing); + $id = text( + label: __('Algolia application ID'), + default: Arr::get($config, 'credentials.id', ''), + required: true, + validate: fn ($value) => preg_match('/^[a-zA-Z0-9]+$/', $value) ? null : __('statamic::search.invalid_algolia_id'), + ); + $key = $this->apiKey(__('Algolia admin API key'), Arr::get($config, 'credentials.secret', '')); + + if (! Http::timeout(10)->withHeaders([ + 'X-Algolia-Application-Id' => $id, + 'X-Algolia-API-Key' => $key, + ])->get("https://{$id}.algolia.net/1/indexes", ['length' => 1])->successful()) { + throw new \RuntimeException('Could not authenticate with Algolia. Check the application ID and admin API key.'); + } + + $this->set('credentials.id', $this->environment('APP_ID', $id)); + $this->set('credentials.secret', $this->environment('SECRET', $key)); + } + + private function configureTypesense(): void + { + $config = array_replace_recursive(config('statamic.search.drivers.typesense', []), $this->existing); + $node = Arr::get($config, 'client.nodes.0', Arr::get($config, 'nodes.0', [])); + $url = text( + label: __('Typesense URL'), + default: sprintf('%s://%s:%s%s', $node['protocol'] ?? 'http', $node['host'] ?? 'localhost', $node['port'] ?? '8108', $node['path'] ?? ''), + required: true, + validate: function ($value) { + $url = parse_url($value); + + $valid = $url && in_array($url['scheme'] ?? null, ['http', 'https'], true) + && isset($url['host']) && ! isset($url['user']) && ! isset($url['pass']) + && ! isset($url['query']) && ! isset($url['fragment']); + + return $valid ? null : __('statamic::search.invalid_url'); + }, + ); + $url = rtrim($url, '/'); + $key = $this->apiKey(__('Typesense admin API key'), Arr::get($config, 'client.api_key', Arr::get($config, 'credentials.api_key', ''))); + $request = Http::timeout(10)->withHeaders(['X-TYPESENSE-API-KEY' => $key]); + + if (! $request->get($url.'/collections')->successful()) { + throw new \RuntimeException('Could not authenticate with Typesense. Check the URL and admin API key.'); + } + + $urlParts = parse_url($url); + // Legacy client credentials would otherwise override the new index credentials. + $this->set('client', Arr::except($config['client'] ?? [], ['api_key', 'nodes', 'nearest_node', 'search_only_api_key'])); + $this->set('credentials.api_key', $this->environment('API_KEY', $key)); + $this->set('nodes', [[ + 'host' => $this->environment('HOST', $urlParts['host']), + 'port' => $this->environment('PORT', (string) ($urlParts['port'] ?? ($urlParts['scheme'] === 'https' ? 443 : 80))), + 'protocol' => $this->environment('PROTOCOL', $urlParts['scheme']), + 'path' => $this->environment('PATH', $urlParts['path'] ?? ''), + ]]); + + $searchKey = $this->apiKey( + __('Typesense search-only API key (optional)'), + Arr::get($config, 'credentials.search_only_api_key') ?? Arr::get($config, 'client.search_only_api_key', ''), + false, + ); + + if ($searchKey !== '') { + $this->set('credentials.search_only_api_key', $this->environment('SEARCH_ONLY_API_KEY', $searchKey)); + } + + if (confirm(__('Configure semantic search?'), false)) { + $available = array_values(array_diff($this->existing['fields'] ?? ['title'], ['date'])); + + if ($available === []) { + throw new \RuntimeException('Configure at least one text field on the index before enabling semantic search.'); + } + + $fields = multiselect( + label: __('statamic::search.embedding_fields_prompt'), + options: $available, + default: array_values(array_intersect(['title', 'content'], $available)), + required: true, + ); + $this->set('settings.embedding', [ + 'from' => $fields, + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + ]); + } + + if (confirm(__('Configure natural language search?'), false)) { + $this->configureNaturalLanguage($request, $url, $config); + } + } + + private function configureNaturalLanguage(PendingRequest $request, string $url, array $config): void + { + $setup = select( + label: __('Natural language model setup'), + options: [ + 'existing' => __('statamic::search.existing_nl_model'), + 'openai' => __('statamic::search.new_nl_model'), + ], + ); + $model = text( + label: __('Typesense natural language model ID'), + default: Arr::get($config, 'settings.natural_language.model', $setup === 'openai' ? 'statamic_'.$this->index : ''), + required: true, + ); + + if ($setup === 'existing') { + if (! $request->get($url.'/nl_search_models/'.rawurlencode($model))->successful()) { + throw new \RuntimeException('Could not find that natural language model. Configure an LLM in Typesense first.'); + } + + $this->set('settings.natural_language', ['model' => $model]); + + return; + } + + $name = text(__('OpenAI model name'), default: 'openai/gpt-4.1-mini', required: true); + $key = $this->apiKey(__('OpenAI API key'), Arr::get($config, 'settings.natural_language.model_config.api_key', '')); + $this->set('settings.natural_language', [ + 'model' => $model, + 'model_config' => [ + 'model_name' => str_starts_with($name, 'openai/') ? $name : 'openai/'.$name, + 'api_key' => $this->environment('NL_API_KEY', $key), + ], + ]); + } + + private function apiKey(string $label, ?string $existing, bool $required = true): string + { + $existing ??= ''; + + return password( + label: $label, + placeholder: $existing !== '' ? __('statamic::search.keep_key') : '', + required: $required && $existing === '', + ) ?: $existing; + } + + private function set(string $path, mixed $value): void + { + $this->changes[] = [['indexes', $this->index, ...explode('.', $path)], $value]; + } + + private function environment(string $suffix, string $value): FuncCall + { + $key = $this->prefix.'_'.$suffix; + $this->variables[$key] = $value; + + return new FuncCall(new Name('env'), [new Arg(new String_($key))]); + } + + private function installClient(string $driver): void + { + $package = match ($driver) { + 'algolia' => 'algolia/algoliasearch-client-php', + 'typesense' => 'typesense/typesense-php', + default => null, + }; + + if ($package && ! Composer::isInstalled($package)) { + spin( + callback: fn () => Composer::withoutQueue()->throwOnFailure()->require($package, $driver === 'typesense' ? '^6.0' : '^4.32'), + message: __('statamic::search.installing_client', ['package' => $package]), + ); + } + } + + private function writeEnvironment(): void + { + if ($this->variables === []) { + return; + } + + $path = $this->laravel->environmentFilePath(); + $contents = File::exists($path) ? File::get($path) : ''; + + foreach ($this->variables as $key => $value) { + $escaped = str_replace(['\\', '"', '$', "\r", "\n"], ['\\\\', '\\"', '\\$', '\\r', '\\n'], $value); + $line = $key.'="'.$escaped.'"'; + $pattern = '/^'.preg_quote($key, '/').'\h*=.*$/m'; + $contents = preg_match($pattern, $contents) + ? preg_replace_callback($pattern, fn () => $line, $contents) + : rtrim($contents).PHP_EOL.$line.PHP_EOL; + } + + File::put($path, $contents); + } +} diff --git a/src/Providers/ConsoleServiceProvider.php b/src/Providers/ConsoleServiceProvider.php index 8d058c405ed..d9848664fa8 100644 --- a/src/Providers/ConsoleServiceProvider.php +++ b/src/Providers/ConsoleServiceProvider.php @@ -19,6 +19,7 @@ class ConsoleServiceProvider extends ServiceProvider Commands\Install::class, Commands\InstallCollaboration::class, Commands\InstallEloquentDriver::class, + Commands\InstallSearch::class, Commands\InstallSsg::class, Commands\FlatCamp::class, Commands\LicenseSet::class, diff --git a/src/Search/ConfigUpdater.php b/src/Search/ConfigUpdater.php new file mode 100644 index 00000000000..9b54f64d2ae --- /dev/null +++ b/src/Search/ConfigUpdater.php @@ -0,0 +1,97 @@ +traverse($file->ast()); + $config = null; + + foreach ($nodes as $node) { + if ($node instanceof Return_ && $node->expr instanceof Array_) { + $config = $node->expr; + break; + } + } + + if (! $config) { + throw new \RuntimeException('The search config must return an array to configure it automatically.'); + } + + foreach ($removals as $path) { + $this->remove($config, $path); + } + + foreach ($changes as [$path, $value]) { + $this->set($config, $path, $value); + } + + return $file->ast($nodes)->render(); + } + + private function set(Array_ $array, array $path, mixed $value): void + { + $key = array_shift($path); + + foreach ($array->items as $item) { + if (! $item || ! $item->key instanceof String_ || $item->key->value !== $key) { + continue; + } + + if ($path === []) { + $item->value = BuilderHelpers::normalizeValue($value); + + return; + } + + if (! $item->value instanceof Array_) { + throw new \RuntimeException("Search configuration [{$key}] must be an array to configure it automatically."); + } + + $this->set($item->value, $path, $value); + + return; + } + + $expression = $path === [] ? BuilderHelpers::normalizeValue($value) : new Array_; + + if ($path !== []) { + $this->set($expression, $path, $value); + } + + $array->items[] = new ArrayItem($expression, new String_($key)); + } + + private function remove(Array_ $array, array $path): void + { + $key = array_shift($path); + + foreach ($array->items as $position => $item) { + if (! $item || ! $item->key instanceof String_ || $item->key->value !== $key) { + continue; + } + + if ($path === []) { + array_splice($array->items, $position, 1); + } elseif ($item->value instanceof Array_) { + $this->remove($item->value, $path); + } else { + throw new \RuntimeException("Search configuration [{$key}] must be an array to configure it automatically."); + } + + return; + } + } +} diff --git a/src/Search/IndexManager.php b/src/Search/IndexManager.php index 9e7567bb8dd..1a757ccef92 100644 --- a/src/Search/IndexManager.php +++ b/src/Search/IndexManager.php @@ -8,8 +8,10 @@ use Statamic\Search\Algolia\Index as AlgoliaIndex; use Statamic\Search\Comb\Index as CombIndex; use Statamic\Search\Null\NullIndex; +use Statamic\Search\Typesense\Index as TypesenseIndex; use Statamic\Support\Manager; use Statamic\Support\Str; +use Typesense\Client as TypesenseClient; class IndexManager extends Manager { @@ -122,6 +124,20 @@ public function createAlgoliaDriver(array $config, $name, $locale) return new AlgoliaIndex($client, $name, $config, $locale); } + public function createTypesenseDriver(array $config, $name, $locale) + { + $clientConfig = $config['client'] ?? []; + $clientConfig['api_key'] ??= $config['credentials']['api_key'] ?? ''; + $clientConfig['nodes'] ??= $config['nodes'] ?? []; + + return new TypesenseIndex( + new TypesenseClient($clientConfig), + $name, + $config, + $locale, + ); + } + protected function callLocalizedCustomCreator(array $config, string $name, ?string $locale) { return $this->customCreators[$config['driver']]($this->app, $config, $name, $locale); diff --git a/src/Search/QueryBuilder.php b/src/Search/QueryBuilder.php index 5173bad6282..6fa7e42dfdf 100644 --- a/src/Search/QueryBuilder.php +++ b/src/Search/QueryBuilder.php @@ -16,6 +16,7 @@ abstract class QueryBuilder extends BaseQueryBuilder protected $query; protected $index; protected $withData = true; + protected bool $naturalLanguage = false; public function __construct(Index $index) { @@ -43,6 +44,18 @@ public function withoutData() return $this; } + public function naturalLanguage(bool $enabled = true) + { + $this->naturalLanguage = $enabled; + + return $this; + } + + public function naturalLanguageParams(): array + { + return []; + } + public function get($columns = ['*']) { return $this->withFakeQueryLogging(fn () => parent::get($columns)); diff --git a/src/Search/Tags.php b/src/Search/Tags.php index 909ba146418..b08b548663d 100644 --- a/src/Search/Tags.php +++ b/src/Search/Tags.php @@ -4,6 +4,7 @@ use Statamic\Facades\Search; use Statamic\Facades\Site; +use Statamic\Support\Arr; use Statamic\Tags\Concerns; use Statamic\Tags\Tags as BaseTags; @@ -30,7 +31,8 @@ public function results() $builder = Search::index($this->params->get('index')) ->ensureExists() ->search($query) - ->withData($supplementData); + ->withData($supplementData) + ->naturalLanguage($this->params->bool('natural_language')); $this->querySite($builder); $this->queryStatus($builder); @@ -39,8 +41,47 @@ public function results() $this->queryOrderBys($builder); $results = $this->getQueryResults($builder); + $naturalLanguage = $builder->naturalLanguageParams(); - return $this->output($results); + if ($naturalLanguage) { + $collection = $results instanceof \Illuminate\Contracts\Pagination\Paginator + ? $results->getCollection() + : $results; + + if (method_exists($collection, 'supplement')) { + $collection->supplement('natural_language', $naturalLanguage); + } + } + + $output = $this->output($results); + + if ($naturalLanguage && is_array($output)) { + $output['natural_language'] = $naturalLanguage; + } + + return $output; + } + + public function scopedKey() + { + $index = Search::index($this->params->get('index')); + + if (! method_exists($index, 'scopedSearchKey')) { + throw new \LogicException( + "Search index [{$index->name()}] does not support scoped search keys." + ); + } + + $parameters = Arr::except($this->params->all(), [ + 'index', + 'expires', + 'expires_at', + ]); + + return $index->scopedSearchKey( + $parameters, + $this->params->int('expires', 3600), + ); } protected function queryStatus($query) diff --git a/src/Search/Typesense/FilterTranslator.php b/src/Search/Typesense/FilterTranslator.php new file mode 100644 index 00000000000..01d59ffa2e8 --- /dev/null +++ b/src/Search/Typesense/FilterTranslator.php @@ -0,0 +1,267 @@ +where($where); + + if (is_null($filter)) { + return null; + } + + $boolean = $where['boolean'] ?? 'and'; + + if (($where['type'] ?? null) === 'NotIn') { + $boolean = 'and'; + } + $filters[] = [ + 'boolean' => $boolean, + 'filter' => $filter, + ]; + } + + return collect($filters)->reduce(function (?string $expression, array $filter) { + if (is_null($expression)) { + return $filter['filter']; + } + + $operator = $filter['boolean'] === 'or' ? '||' : '&&'; + + return "({$expression} {$operator} {$filter['filter']})"; + }); + } + + public function sorts(array $orders): ?string + { + if ($orders === []) { + return ''; + } + + if (count($orders) > 3) { + return null; + } + + $sorts = collect($orders)->map(function (OrderBy $order) { + $field = $this->field($order->sort); + + if (! $field || ! $this->schema->sortable($field)) { + return null; + } + + return "{$field}:{$order->direction}"; + }); + + return $sorts->contains(null) ? null : $sorts->implode(','); + } + + public function escape(mixed $value, ?string $type = null): ?string + { + if ($value instanceof DateTimeInterface) { + $value = $value->getTimestamp(); + } + + $type = str_replace('[]', '', (string) $type); + + if (in_array($type, ['int32', 'int64'], true)) { + if (! is_int($value) && ! is_numeric($value)) { + return null; + } + + if ((float) $value !== (float) (int) $value) { + return null; + } + + return (string) ((int) $value); + } + + if ($type === 'float') { + if (! is_float($value) && ! is_int($value) && ! is_numeric($value)) { + return null; + } + + return (string) ((float) $value); + } + + if ($type === 'bool' || is_bool($value)) { + $value = is_bool($value) + ? $value + : filter_var($value, FILTER_VALIDATE_BOOL, FILTER_NULL_ON_FAILURE); + + return is_null($value) ? null : ($value ? 'true' : 'false'); + } + + if (in_array($type, ['string', 'auto'], true) && is_scalar($value)) { + $value = (string) $value; + } + + if (is_int($value) || is_float($value)) { + return (string) $value; + } + + if (! is_string($value)) { + return null; + } + + $value = str_replace(['\\', '`'], ['\\\\', '\\`'], $value); + + return "`{$value}`"; + } + + private function where(array $where): ?string + { + if (($where['type'] ?? null) === 'Nested') { + $query = $where['query'] ?? null; + + if (! $query || ! method_exists($query, 'rawWheres')) { + return null; + } + + $nested = $this->filters($query->rawWheres()); + + return $nested === '' ? null : $nested; + } + + $field = $this->field($where['column'] ?? ''); + + if (! $field || ! $this->schema->filterable($field)) { + return null; + } + + $type = $this->schema->field($field)['type'] ?? null; + $array = str_ends_with((string) $type, '[]'); + $json = str_starts_with($where['type'] ?? '', 'Json'); + $scalarType = str_replace('[]', '', (string) $type); + $supportedTypes = ['string', 'int32', 'int64', 'float', 'bool']; + + // Array membership and scalar equality are different operations in Statamic. + if ($array !== $json || ! in_array($scalarType, $supportedTypes, true)) { + return null; + } + + return match ($where['type'] ?? null) { + 'Basic' => $this->basic($field, $type, $where), + 'In' => $this->in($field, $type, $where['values'] ?? [], false), + 'NotIn' => $this->in($field, $type, $where['values'] ?? [], true), + 'Between' => $this->between($field, $type, $where['values'] ?? [], false), + 'NotBetween' => $this->between($field, $type, $where['values'] ?? [], true), + 'JsonContains' => $this->contains($field, $type, $where['values'] ?? [], false), + 'JsonDoesntContain' => $this->contains($field, $type, $where['values'] ?? [], true), + 'JsonOverlaps' => $this->in($field, $type, $where['values'] ?? [], false), + 'JsonDoesntOverlap' => $this->in($field, $type, $where['values'] ?? [], true), + default => null, + }; + } + + private function basic(string $field, ?string $type, array $where): ?string + { + $comparison = $where['operator'] ?? ''; + $operators = [ + '=' => ':=', + '<>' => ':!=', + '!=' => ':!=', + '>' => ':>', + '>=' => ':>=', + '<' => ':<', + '<=' => ':<=', + ]; + + if (! $operator = $operators[$comparison] ?? null) { + return null; + } + + if (in_array($comparison, ['>', '>=', '<', '<='], true) && ! $this->supportsRange($type)) { + return null; + } + + if (is_null($value = $this->escape($where['value'] ?? null, $type))) { + return null; + } + + return "{$field}{$operator}{$value}"; + } + + private function in(string $field, ?string $type, array $values, bool $negated): ?string + { + if ($values === []) { + return null; + } + + $values = collect($values)->map(fn ($value) => $this->escape($value, $type)); + + if ($values->contains(null)) { + return null; + } + + $operator = $negated ? ':!=' : ':='; + + return sprintf('%s%s[%s]', $field, $operator, $values->implode(',')); + } + + private function between(string $field, ?string $type, array $values, bool $negated): ?string + { + if (count($values) !== 2 || ! $this->supportsRange($type)) { + return null; + } + + [$from, $to] = array_map(fn ($value) => $this->escape($value, $type), $values); + + if (is_null($from) || is_null($to)) { + return null; + } + + return $negated + ? "({$field}:<{$from} || {$field}:>{$to})" + : "({$field}:>={$from} && {$field}:<={$to})"; + } + + private function contains(string $field, ?string $type, array $values, bool $negated): ?string + { + if ($values === []) { + return null; + } + + $values = collect($values)->map(fn ($value) => $this->escape($value, $type)); + + if ($values->contains(null)) { + return null; + } + + $operator = $negated ? ':!=' : ':='; + $boolean = $negated ? ' || ' : ' && '; + + $filter = $values + ->map(fn (string $value) => "{$field}{$operator}{$value}") + ->join($boolean); + + return $values->count() > 1 ? "({$filter})" : $filter; + } + + private function field(string $field): ?string + { + $field = str_replace('->', '.', $field); + + return preg_match('/^[\w*.-]+$/', $field) ? $field : null; + } + + private function supportsRange(?string $type): bool + { + return in_array($type, ['int32', 'int64', 'float'], true); + } +} diff --git a/src/Search/Typesense/Index.php b/src/Search/Typesense/Index.php new file mode 100644 index 00000000000..2d16f0b28b0 --- /dev/null +++ b/src/Search/Typesense/Index.php @@ -0,0 +1,721 @@ +normalizer = new Normalizer; + $this->schema = new Schema($client, $this->name, $config); + } + + public function search($query) + { + return (new Query($this))->query($query); + } + + public function client(): Client + { + return $this->client; + } + + public function schema(): Schema + { + return $this->schema; + } + + public function fields(Searchable $searchable) + { + return $this->normalizer->document(array_merge( + parent::fields($searchable), + $this->metadata($searchable), + )); + } + + public function insertDocuments(Documents $documents) + { + $this->importDocuments($this->name, $documents); + } + + public function delete($document) + { + try { + $this->collection()->documents[$this->documentId($document->getSearchReference())]->delete(); + } catch (ObjectNotFound) { + // The document or collection has already gone. + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + + $this->forgetCachedState(); + } + + public function exists() + { + return Blink::once($this->existsBlinkKey(), fn () => $this->activeCollectionName() !== null); + } + + public function update() + { + $this->ensureNaturalLanguageModel(); + + if (Arr::get($this->config, 'settings.reindex', 'alias') === 'in_place') { + return $this->updateInPlace(); + } + + return $this->updateUsingAlias(); + } + + public function searchUsingApi($query, array $parameters = []): array + { + $defaults = Arr::get( + $this->config, + 'settings.search_parameters', + Arr::get($this->config, 'settings.search_options', []), + ); + $defaultFilter = $defaults['filter_by'] ?? null; + $requestedFilter = $parameters['filter_by'] ?? null; + $parameters = array_merge($defaults, $parameters); + + if ($defaultFilter && $requestedFilter) { + $parameters['filter_by'] = "({$defaultFilter}) && ({$requestedFilter})"; + } + + $parameters['q'] = $query ?? ''; + + if (is_array($parameters['query_by'] ?? null)) { + $parameters['query_by'] = implode(',', $parameters['query_by']); + } + + if (is_string($parameters['query_by'] ?? null)) { + $parameters['query_by'] = str_replace('->', '.', $parameters['query_by']); + } + + if (is_array($parameters['exclude_fields'] ?? null)) { + $parameters['exclude_fields'] = implode(',', $parameters['exclude_fields']); + } + + if (is_string($parameters['exclude_fields'] ?? null)) { + $parameters['exclude_fields'] = str_replace('->', '.', $parameters['exclude_fields']); + $parameters['exclude_fields'] = $this->removeCommaSeparated( + $parameters['exclude_fields'], + 'reference', + ); + } + + if (blank($parameters['exclude_fields'] ?? null)) { + unset($parameters['exclude_fields']); + } + + if (is_array($parameters['include_fields'] ?? null)) { + $parameters['include_fields'] = implode(',', $parameters['include_fields']); + } + + if (is_string($parameters['include_fields'] ?? null)) { + $parameters['include_fields'] = str_replace('->', '.', $parameters['include_fields']); + } + + if (filled($parameters['include_fields'] ?? null)) { + $parameters['include_fields'] = $this->appendCommaSeparated( + $parameters['include_fields'], + 'reference', + ); + } + + if (blank($parameters['query_by'] ?? null)) { + unset($parameters['query_by']); + } + + if (! isset($parameters['query_by']) && ! isset($parameters['preset'])) { + $parameters['query_by'] = $this->schema->queryBy(); + } + + if ($embedding = Arr::get($this->config, 'settings.embedding')) { + $field = $this->embeddingFieldName(); + + if (isset($parameters['query_by'])) { + $parameters['query_by'] = $this->appendCommaSeparated( + $parameters['query_by'], + $field, + ); + } + $parameters['exclude_fields'] = $this->appendCommaSeparated( + $parameters['exclude_fields'] ?? null, + $field, + ); + + if (isset($embedding['alpha']) && ! isset($parameters['vector_query'])) { + $parameters['vector_query'] = sprintf( + '%s:([], alpha: %s)', + $field, + $embedding['alpha'], + ); + } + } + + $request = [ + 'searches' => [ + array_merge($parameters, ['collection' => $this->name]), + ], + ]; + + try { + $response = $this->client->multiSearch->perform($request, []); + } catch (TypesenseClientError $e) { + $this->handleTypesenseException($e); + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + + $result = $response['results'][0] ?? []; + + if ($error = $result['error'] ?? null) { + $this->handleTypesenseException(new TypesenseClientError( + $error, + (int) ($result['code'] ?? 0), + )); + } + + if ($naturalLanguageError = Arr::get($result, 'parsed_nl_query.error')) { + Log::warning('Typesense Natural Language Search fell back to the original query.', [ + 'index' => $this->name, + 'error' => $naturalLanguageError, + ]); + } + + $hits = collect($result['hits'] ?? []); + + if ($hits->isEmpty() && isset($result['grouped_hits'])) { + $hits = collect($result['grouped_hits']) + ->flatMap(fn (array $group) => $group['hits'] ?? []); + } + + $total = $hits->count(); + + return [ + 'found' => (int) ($result['found'] ?? $total), + 'hits' => $hits->values()->map(function (array $hit, int $position) use ($total) { + $document = $hit['document'] ?? []; + $document['reference'] ??= $document['id'] ?? null; + $document['search_score'] = $total - $position; + $document['search_text_match'] = $hit['text_match'] ?? null; + $document['search_text_match_info'] = $hit['text_match_info'] ?? null; + $document['search_highlights'] = ($hit['highlights'] ?? []) + ?: ($hit['highlight'] ?? []); + + return $document; + }), + 'raw' => $result, + ]; + } + + public function ensureNaturalLanguageModel(): ?string + { + $settings = Arr::get($this->config, 'settings.natural_language'); + + if (! $settings) { + return null; + } + + $id = $settings['model'] ?? 'statamic_'.$this->name; + $model = $settings['model_config'] ?? null; + + if (! $model) { + return $id; + } + + $cacheKey = 'statamic.search.typesense.nl-model.'.hash('sha256', serialize([ + $this->cacheFingerprint(), + $id, + $model, + ])); + + try { + Cache::remember($cacheKey, now()->addDay(), function () use ($id, $model) { + try { + $this->client->nlSearchModels[$id]->retrieve(); + $this->client->nlSearchModels[$id]->update($model); + } catch (ObjectNotFound) { + $this->client->nlSearchModels->create(array_merge($model, ['id' => $id])); + } + + return true; + }); + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + + return $id; + } + + public function scopedSearchKey(array $parameters = [], int $expires = 3600): string + { + $key = Arr::get($this->config, 'credentials.search_only_api_key') + ?? Arr::get($this->config, 'client.search_only_api_key'); + + if (! $key) { + throw new \LogicException( + "No Typesense search-only API key has been configured for index [{$this->name}]." + ); + } + + $defaults = Arr::get( + $this->config, + 'settings.search_parameters', + Arr::get($this->config, 'settings.search_options', []), + ); + $defaultFilter = $defaults['filter_by'] ?? null; + $site = $this->locale; + + if (! $site && Site::hasMultiple()) { + $site = Site::current()->handle(); + } + + $filters = array_filter([ + 'status:=published', + $site ? 'site:='.$this->escapeFilterValue($site) : null, + $defaultFilter ? "({$defaultFilter})" : null, + ]); + + if ($filter = $parameters['filter_by'] ?? null) { + $filters[] = "({$filter})"; + } + + $parameters['filter_by'] = implode(' && ', $filters); + $parameters['expires_at'] ??= now()->addSeconds($expires)->timestamp; + + if (Arr::get($this->config, 'settings.embedding')) { + $parameters['exclude_fields'] = $this->appendCommaSeparated( + $parameters['exclude_fields'] ?? null, + $this->embeddingFieldName(), + ); + } + + return $this->client->keys->generateScopedSearchKey($key, $parameters); + } + + public function documentId(string $reference): string + { + return hash('sha256', $reference); + } + + public function cacheFingerprint(): string + { + return hash('sha256', serialize([ + Arr::get($this->config, 'client.nodes', $this->config['nodes'] ?? []), + Arr::get($this->config, 'client.api_key', Arr::get($this->config, 'credentials.api_key')), + ])); + } + + public function searchCacheFingerprint(): string + { + return hash('sha256', serialize([ + $this->cacheFingerprint(), + $this->config['fields'] ?? [], + $this->config['settings'] ?? [], + Blink::get($this->searchVersionKey(), 0), + ])); + } + + public function extraAugmentedResultData(Result $result) + { + $raw = $result->getRawResult(); + $highlights = collect($raw['search_highlights'] ?? []) + ->mapWithKeys(function (array $highlight, int|string $key) { + $field = $highlight['field'] + ?? $highlight['field_name'] + ?? (is_string($key) ? $key : null); + + return $field ? [$field => $highlight] : []; + }) + ->all(); + + return [ + 'search_highlights' => $highlights, + 'search_snippets' => collect($highlights) + ->map(fn (array $highlight) => $highlight['snippet'] ?? $highlight['value'] ?? null) + ->filter() + ->all(), + 'search_text_match' => $raw['search_text_match'] ?? null, + ]; + } + + protected function deleteIndex() + { + $active = $this->activeCollectionName(); + + if (! $active) { + return; + } + + if ($active !== $this->name) { + try { + $this->client->aliases[$this->name]->delete(); + } catch (ObjectNotFound) { + // + } + } + + try { + $this->client->collections[$active]->delete(); + } catch (ObjectNotFound) { + // + } + + $this->forgetCachedState(); + } + + private function updateUsingAlias(): self + { + $previous = $this->activeCollectionName(); + $candidate = $this->newCollectionName(); + $legacyCollectionRemoved = false; + $candidateCreated = false; + + try { + $schema = $this->schema->definition($candidate); + + if ($previous) { + $existing = $this->client->collections[$previous]->retrieve(); + + foreach (['synonym_sets', 'curation_sets'] as $setting) { + if (! array_key_exists($setting, $schema) && array_key_exists($setting, $existing)) { + $schema[$setting] = $existing[$setting]; + } + } + } + + $this->client->collections->create($schema); + $candidateCreated = true; + $this->importAllSearchablesInto($candidate); + + if ($previous === $this->name) { + $this->client->collections[$previous]->delete(); + $legacyCollectionRemoved = true; + } + + $this->client->aliases->upsert($this->name, [ + 'collection_name' => $candidate, + ]); + } catch (Throwable $e) { + $this->forgetCachedState(); + + if ( + $candidateCreated + && ! $legacyCollectionRemoved + && $this->aliasPointsTo($candidate) === false + ) { + try { + $this->client->collections[$candidate]->delete(); + } catch (Throwable) { + // + } + } + + if ($e instanceof HttpClientException || $e instanceof PsrClientException) { + throw $this->connectionException($e); + } + + throw $e; + } + + if ($previous && $previous !== $this->name && $previous !== $candidate) { + try { + $this->client->collections[$previous]->delete(); + } catch (Throwable $e) { + Log::warning('The old Typesense collection could not be deleted after its alias was swapped.', [ + 'index' => $this->name, + 'collection' => $previous, + 'error' => $e->getMessage(), + ]); + } + } + + $this->forgetCachedState(); + + return $this; + } + + private function aliasPointsTo(string $collection): ?bool + { + try { + $active = $this->client->aliases[$this->name]->retrieve()['collection_name'] ?? null; + + return $active === $collection; + } catch (ObjectNotFound) { + return false; + } catch (Throwable) { + return null; + } + } + + private function updateInPlace(): self + { + $this->deleteIndex(); + $this->client->collections->create($this->schema->definition($this->name)); + $this->forgetCachedState(); + + $this->searchables()->lazy()->each(fn ($searchables) => $this->insertMultiple($searchables)); + + return $this; + } + + private function importAllSearchablesInto(string $collection): void + { + // A queued rebuild could swap the alias before its indexing jobs finish. + // Full alias rebuilds are synchronous; incremental writes still use the configured queue. + $this->searchables()->lazy()->each(function ($references) use ($collection) { + $references + ->chunk(config('statamic.search.chunk_size')) + ->each(function ($references) use ($collection) { + $this->importDocuments($collection, $this->documentsForReferences($references)); + }); + }); + } + + private function documentsForReferences($references): Documents + { + $providers = app(Providers::class); + + $documents = collect($references) + ->groupBy(fn ($reference) => explode('::', $reference)[0]) + ->flatMap(function ($references, $prefix) use ($providers) { + return $providers + ->getByPrefix($prefix) + ->find($references->map(fn ($reference) => Str::after($reference, '::'))->all()) + ->all(); + }) + ->mapWithKeys(function (Searchable $item) { + return [$item->getSearchReference() => $this->fields($item)]; + }); + + return new Documents($documents); + } + + private function importDocuments(string $collection, Documents $documents): void + { + if ($documents->isEmpty()) { + return; + } + + $sortField = Arr::get($this->config, 'settings.schema.default_sorting_field'); + $sortField = is_string($sortField) ? str_replace('->', '.', $sortField) : null; + + $documents = $documents + ->map(function (array $document, string $reference) use ($sortField) { + $document = array_merge($document, [ + 'id' => $this->documentId($reference), + 'reference' => $reference, + ]); + + if ($sortField && ! array_key_exists($sortField, $document)) { + $document[$sortField] = 0; + } + + return $document; + }) + ->values() + ->all(); + + $options = array_merge( + ['action' => 'upsert', 'dirty_values' => 'coerce_or_drop'], + Arr::get($this->config, 'settings.import_parameters', []), + ); + + try { + $responses = $this->client->collections[$collection]->documents->import($documents, $options); + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + + // Imports can partially succeed even when another document is rejected. + $this->forgetCachedState(); + + $failures = collect($responses) + ->filter(fn (array $response) => ($response['success'] ?? false) !== true); + + if ($failures->isNotEmpty()) { + $message = $failures + ->map(fn (array $failure) => $failure['error'] ?? 'Unknown Typesense import error.') + ->unique() + ->implode(' '); + + throw new \RuntimeException("Unable to import documents into Typesense index [{$this->name}]. {$message}"); + } + + } + + private function metadata(Searchable $searchable): array + { + $reference = $searchable->getSearchReference(); + + return [ + 'reference' => $reference, + 'type' => Str::before($reference, '::'), + 'status' => method_exists($searchable, 'status') ? $searchable->status() : 'published', + 'site' => $this->firstMethodValue($searchable, ['locale', 'site']), + 'collection' => $this->firstMethodValue($searchable, ['collectionHandle', 'collection']), + 'taxonomy' => $this->firstMethodValue($searchable, ['taxonomyHandle', 'taxonomy']), + 'container' => $this->firstMethodValue($searchable, ['containerHandle', 'container']), + 'date' => $this->firstMethodValue($searchable, ['date']), + ]; + } + + private function firstMethodValue(Searchable $searchable, array $methods): mixed + { + foreach ($methods as $method) { + if (method_exists($searchable, $method)) { + return $searchable->{$method}(); + } + } + + return null; + } + + private function activeCollectionName(): ?string + { + try { + return $this->client->aliases[$this->name]->retrieve()['collection_name'] ?? null; + } catch (ObjectNotFound) { + try { + $this->client->collections[$this->name]->retrieve(); + + return $this->name; + } catch (ObjectNotFound) { + return null; + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + } catch (HttpClientException|PsrClientException $e) { + throw $this->connectionException($e); + } + } + + private function collection() + { + return $this->client->collections[$this->name]; + } + + private function newCollectionName(): string + { + return sprintf( + '%s_%s_%s', + $this->name, + now()->format('YmdHis'), + Str::lower(Str::random(6)), + ); + } + + private function forgetCachedState(): void + { + Blink::forget($this->existsBlinkKey()); + $this->schema->forget(); + Blink::put($this->searchVersionKey(), Blink::get($this->searchVersionKey(), 0) + 1); + } + + private function searchVersionKey(): string + { + return 'search-typesense-version-'.$this->cacheFingerprint().'-'.$this->name; + } + + private function existsBlinkKey(): string + { + return 'search-typesense-exists-'.$this->cacheFingerprint().'-'.$this->name; + } + + private function handleTypesenseException(TypesenseClientError $e): never + { + $message = $e->getMessage(); + + if ( + $e instanceof ObjectNotFound + || Str::contains($message, ['Could not find a collection', 'Collection not found']) + ) { + throw new IndexNotFoundException("Index [{$this->name}] does not exist.", previous: $e); + } + + if (preg_match("~field named [`'\"]?([^`'\"\\s]+)~i", $message, $matches)) { + throw new \Exception( + "Field [{$matches[1]}] does not exist in this Typesense index.", + previous: $e, + ); + } + + throw $e; + } + + private function connectionException(Throwable $e): \RuntimeException + { + return new \RuntimeException( + 'Error connecting to Typesense. Check your API credentials and node configuration.', + previous: $e, + ); + } + + private function appendCommaSeparated(string|array|null $existing, string $value): string + { + $fields = is_array($existing) ? $existing : explode(',', (string) $existing); + + return collect($fields) + ->push($value) + ->map(fn (string $field) => str_replace('->', '.', trim($field))) + ->filter() + ->unique() + ->implode(','); + } + + private function removeCommaSeparated(string|array|null $existing, string $value): string + { + $fields = is_array($existing) ? $existing : explode(',', (string) $existing); + + return collect($fields) + ->map(fn (string $field) => str_replace('->', '.', trim($field))) + ->reject(fn (string $field) => $field === $value) + ->filter() + ->unique() + ->implode(','); + } + + private function embeddingFieldName(): string + { + return str_replace( + '->', + '.', + (string) Arr::get($this->config, 'settings.embedding.name', 'embedding'), + ); + } + + private function escapeFilterValue(string $value): string + { + return '`'.str_replace(['\\', '`'], ['\\\\', '\\`'], $value).'`'; + } +} diff --git a/src/Search/Typesense/Normalizer.php b/src/Search/Typesense/Normalizer.php new file mode 100644 index 00000000000..b05f7528c20 --- /dev/null +++ b/src/Search/Typesense/Normalizer.php @@ -0,0 +1,77 @@ +mapWithKeys(function ($value, $field) { + $field = is_string($field) ? str_replace('->', '.', $field) : $field; + + return [$field => $this->value($value)]; + }) + ->reject(fn ($value) => is_null($value)) + ->all(); + } + + public function value(mixed $value): mixed + { + if ($value instanceof DateTimeInterface) { + return $value->getTimestamp(); + } + + if ($value instanceof Site) { + return $value->handle(); + } + + if ($value instanceof BackedEnum) { + return $value->value; + } + + if ($value instanceof Collection) { + $isList = array_is_list($value->all()); + $value = $value + ->map(fn ($item) => $this->value($item)) + ->reject(fn ($item) => is_null($item)); + + return $isList ? $value->values()->all() : $value->all(); + } + + if (is_object($value) && method_exists($value, 'handle')) { + return $this->value($value->handle()); + } + + if ($value instanceof Arrayable) { + return $this->value($value->toArray()); + } + + if ($value instanceof JsonSerializable) { + return $this->value($value->jsonSerialize()); + } + + if (is_array($value)) { + $isList = array_is_list($value); + $value = collect($value) + ->map(fn ($item) => $this->value($item)) + ->reject(fn ($item) => is_null($item)); + + return $isList ? $value->values()->all() : $value->all(); + } + + if ($value instanceof Stringable) { + return (string) $value; + } + + return is_scalar($value) || is_null($value) ? $value : null; + } +} diff --git a/src/Search/Typesense/Query.php b/src/Search/Typesense/Query.php new file mode 100644 index 00000000000..7f12098c40c --- /dev/null +++ b/src/Search/Typesense/Query.php @@ -0,0 +1,290 @@ +canPushDown()) { + return parent::get($columns); + } + + return $this->withFakeQueryLogging(function () { + $response = $this->execute(array_merge( + $this->translatedParameters(), + [ + 'offset' => $this->offset, + 'limit' => $this->requestedLimit(), + ], + )); + + return $this->transformResults($response['hits']); + }); + } + + public function count() + { + if (! $this->canPushDown()) { + return parent::count(); + } + + $response = $this->execute(array_merge( + $this->translatedParameters(), + ['offset' => 0, 'limit' => 1], + )); + + $count = max(0, $response['found'] - $this->offset); + + return is_null($this->limit) ? $count : min($count, $this->limit); + } + + public function paginate($perPage = null, $columns = ['*'], $pageName = 'page', $page = null) + { + if (! $this->canPushDown()) { + return parent::paginate($perPage, $columns, $pageName, $page); + } + + $page = $page ?: Paginator::resolveCurrentPage($pageName); + $perPage = max( + 1, + min( + (int) ($perPage ?: $this->defaultPerPageSize()), + $this->maxResults(), + ), + ); + + $response = $this->withFakeQueryLogging(fn () => $this->execute(array_merge( + $this->translatedParameters(), + [ + 'offset' => max(0, ($page - 1) * $perPage), + 'limit' => $perPage, + ], + ))); + + return $this->paginator( + $this->transformResults($response['hits']), + $response['found'], + $perPage, + $page, + [ + 'path' => Paginator::resolveCurrentPath(), + 'pageName' => $pageName, + ], + ); + } + + public function getSearchResults($query) + { + return $this->execute([ + 'offset' => 0, + 'limit' => $this->maxResults(), + ])['hits']; + } + + public function naturalLanguageParams(): array + { + return $this->generatedNaturalLanguageParams; + } + + public function rawWheres(): array + { + return $this->wheres; + } + + protected function getCountForPagination() + { + if (! $this->canPushDown()) { + return parent::getCountForPagination(); + } + + return $this->execute(array_merge( + $this->translatedParameters(), + ['offset' => 0, 'limit' => 1], + ))['found']; + } + + private function execute(array $parameters): array + { + [$query, $parameters] = $this->naturalLanguageParameters($parameters); + $cacheKey = 'search-typesense-'.$this->index->searchCacheFingerprint().'-'.$this->index->name().'-'.hash('sha256', serialize([ + $query, + $parameters, + ])); + + $response = Blink::once( + $cacheKey, + fn () => $this->index->searchUsingApi($query, $parameters), + ); + + $this->rememberNaturalLanguageParams($response['raw']); + + return $response; + } + + private function canPushDown(): bool + { + if ($this->randomize) { + return false; + } + + $translator = $this->translator(); + + return ! is_null($translator->filters($this->wheres)) + && ! is_null($translator->sorts($this->orderBys)); + } + + private function translatedParameters(): array + { + $translator = $this->translator(); + $parameters = []; + + if ($filter = $translator->filters($this->wheres)) { + $parameters['filter_by'] = $filter; + } + + if ($sort = $translator->sorts($this->orderBys)) { + $parameters['sort_by'] = $sort; + } + + return $parameters; + } + + private function translator(): FilterTranslator + { + return new FilterTranslator($this->index->schema()); + } + + private function requestedLimit(): int + { + return min((int) ($this->limit ?? $this->maxResults()), $this->maxResults()); + } + + private function maxResults(): int + { + return max(1, min( + 250, + (int) Arr::get($this->index->config(), 'settings.max_results', 250), + )); + } + + private function naturalLanguageParameters(array $parameters): array + { + if (! $this->naturalLanguage) { + return [$this->query, $parameters]; + } + + $model = $this->index->ensureNaturalLanguageModel(); + + if (! $model) { + throw new \LogicException( + "Natural Language Search has not been configured for Typesense index [{$this->index->name()}]." + ); + } + + $cacheKey = $this->naturalLanguageCacheKey($model); + $generated = Blink::once($cacheKey, fn () => Cache::get($cacheKey)); + + if ($generated) { + $this->generatedNaturalLanguageParams = $generated; + + return $this->applyGeneratedParameters($generated, $parameters); + } + + $parameters['nl_query'] = true; + $parameters['nl_model_id'] = $model; + + return [$this->query, $parameters]; + } + + private function applyGeneratedParameters(array $generated, array $parameters): array + { + $defaults = Arr::get( + $this->index->config(), + 'settings.search_parameters', + Arr::get($this->index->config(), 'settings.search_options', []), + ); + + if (array_key_exists('sort_by', $defaults) && ! array_key_exists('sort_by', $parameters)) { + $parameters['sort_by'] = $defaults['sort_by']; + } + + $query = Arr::pull($generated, 'q', $this->query); + $generatedFilter = Arr::pull($generated, 'filter_by'); + $explicitFilter = $parameters['filter_by'] ?? null; + $parameters = array_merge($generated, $parameters); + + if ($generatedFilter && $explicitFilter) { + $parameters['filter_by'] = "({$explicitFilter}) && ({$generatedFilter})"; + } elseif ($generatedFilter) { + $parameters['filter_by'] = $generatedFilter; + } + + return [$query, $parameters]; + } + + private function rememberNaturalLanguageParams(array $response): void + { + if (! $this->naturalLanguage) { + return; + } + + $generated = Arr::get($response, 'parsed_nl_query.generated_params'); + + if (! is_array($generated)) { + return; + } + + $augmentedQuery = Arr::get($response, 'parsed_nl_query.augmented_params.q'); + + if (! is_null($augmentedQuery)) { + $generated['q'] = $augmentedQuery; + } + + $augmentedSort = Arr::get($response, 'parsed_nl_query.augmented_params.sort_by'); + + if (! is_null($augmentedSort)) { + $generated['sort_by'] = $augmentedSort; + } + + if ($generated === []) { + return; + } + + $this->generatedNaturalLanguageParams = $generated; + $model = Arr::get($this->index->config(), 'settings.natural_language.model') + ?? 'statamic_'.$this->index->name(); + + $cacheKey = $this->naturalLanguageCacheKey($model); + Blink::put($cacheKey, $generated); + Cache::put($cacheKey, $generated, now()->addMinutes(10)); + } + + private function naturalLanguageCacheKey(string $model): string + { + return 'statamic.search.typesense.nl-params.'.hash('sha256', serialize([ + $this->index->cacheFingerprint(), + $this->index->name(), + $model, + $this->query, + Arr::get($this->index->config(), 'fields'), + Arr::get($this->index->config(), 'settings.schema'), + Arr::get($this->index->config(), 'settings.search_parameters'), + Arr::get($this->index->config(), 'settings.search_options'), + Arr::get($this->index->config(), 'settings.natural_language'), + ])); + } +} diff --git a/src/Search/Typesense/Schema.php b/src/Search/Typesense/Schema.php new file mode 100644 index 00000000000..4180b02362a --- /dev/null +++ b/src/Search/Typesense/Schema.php @@ -0,0 +1,237 @@ +config, 'settings.schema', []); + $overrides = collect(Arr::pull($schema, 'fields', [])) + ->map(function (array $field) { + $field['name'] = $this->normalizeFieldName($field['name']); + + return $field; + }) + ->keyBy('name'); + + $fields = $this->defaultFields() + ->map(function (array $field) use ($overrides) { + return array_merge($field, $overrides->get($field['name'], [])); + }) + ->keyBy('name'); + + $overrides->each(function (array $field, string $name) use ($fields) { + if (! $fields->has($name)) { + $fields->put($name, $field); + } + }); + + $definition = array_merge( + [ + 'name' => $name, + 'enable_nested_fields' => true, + ], + $schema, + [ + 'name' => $name, + 'fields' => $fields->values()->all(), + ], + ); + + if ($sortField = $definition['default_sorting_field'] ?? null) { + $sortField = $this->normalizeFieldName($sortField); + $definition['default_sorting_field'] = $sortField; + $definition['fields'] = collect($definition['fields']) + ->map(function (array $field) use ($sortField) { + if ($field['name'] !== $sortField) { + return $field; + } + + $field['optional'] = false; + $field['sort'] = true; + + return $field; + }) + ->all(); + } + + return $definition; + } + + public function queryBy(): string + { + $queryBy = Arr::get($this->config, 'settings.search_parameters.query_by') + ?? Arr::get($this->config, 'settings.search_options.query_by'); + + if ($queryBy) { + $queryBy = is_array($queryBy) ? $queryBy : explode(',', $queryBy); + + if ($embedding = $this->embeddingField()) { + $queryBy[] = $embedding['name']; + } + + return collect($queryBy) + ->map(fn (string $field) => $this->normalizeFieldName(trim($field))) + ->filter() + ->unique() + ->implode(','); + } + + $searchableTypes = ['string', 'string[]']; + $fields = $this->fields(); + + $queryBy = collect($this->config['fields'] ?? []) + ->filter(fn ($field) => is_string($field)) + ->map(fn (string $field) => $this->normalizeFieldName($field)) + ->filter(function (string $field) use ($fields, $searchableTypes) { + $definition = $fields->get($field); + + return $definition + && ($definition['index'] ?? true) !== false + && in_array($definition['type'] ?? null, $searchableTypes, true); + }); + + if ($embedding = $this->embeddingField()) { + $queryBy->push($embedding['name']); + } + + return $queryBy->unique()->implode(',') ?: 'reference'; + } + + public function fields(): Collection + { + return Blink::once($this->blinkKey(), function () { + try { + $schema = $this->client->collections[$this->collection]->retrieve(); + } catch (ObjectNotFound $e) { + throw new IndexNotFoundException( + "Index [{$this->collection}] does not exist.", + previous: $e, + ); + } catch (HttpClientException|PsrClientException $e) { + throw new \RuntimeException( + 'Error connecting to Typesense. Check your API credentials and node configuration.', + previous: $e, + ); + } + + return collect($schema['fields'] ?? [])->keyBy('name'); + }); + } + + public function field(string $name): ?array + { + $fields = $this->fields(); + $name = $this->normalizeFieldName($name); + + return $fields->get($name); + } + + public function filterable(string $name): bool + { + $field = $this->field($name); + + return $field && ($field['index'] ?? true) !== false; + } + + public function sortable(string $name): bool + { + return ($this->field($name)['sort'] ?? false) === true; + } + + public function forget(): void + { + Blink::forget($this->blinkKey()); + } + + private function defaultFields(): Collection + { + $embeddingSources = $this->embeddingField()['embed']['from'] ?? []; + + $fields = collect($this->config['fields'] ?? []) + ->filter(fn ($field) => is_string($field) && $field !== 'id') + ->map(fn (string $name) => [ + 'name' => $this->normalizeFieldName($name), + 'type' => in_array($this->normalizeFieldName($name), $embeddingSources, true) ? 'string' : 'auto', + 'optional' => true, + ]); + + $fields->push( + ['name' => 'reference', 'type' => 'string'], + ['name' => 'type', 'type' => 'string', 'facet' => true], + ['name' => 'status', 'type' => 'string', 'facet' => true, 'optional' => true], + ['name' => 'site', 'type' => 'string', 'facet' => true, 'optional' => true], + ['name' => 'collection', 'type' => 'string', 'facet' => true, 'optional' => true], + ['name' => 'taxonomy', 'type' => 'string', 'facet' => true, 'optional' => true], + ['name' => 'container', 'type' => 'string', 'facet' => true, 'optional' => true], + ['name' => 'date', 'type' => 'int64', 'sort' => true, 'optional' => true], + ); + + if ($embedding = $this->embeddingField()) { + $fields->push($embedding); + } + + $fields->push([ + 'name' => '.*', + 'type' => 'auto', + 'optional' => true, + ]); + + return $fields->keyBy('name')->values(); + } + + private function embeddingField(): ?array + { + if (! $embedding = Arr::get($this->config, 'settings.embedding')) { + return null; + } + + $name = $this->normalizeFieldName(Arr::pull($embedding, 'name', 'embedding')); + Arr::forget($embedding, 'alpha'); + + if (is_array($embedding['from'] ?? null)) { + $embedding['from'] = array_map( + fn (string $field) => $this->normalizeFieldName($field), + $embedding['from'], + ); + } + + return [ + 'name' => $name, + 'type' => 'float[]', + 'embed' => $embedding, + ]; + } + + private function normalizeFieldName(string $name): string + { + return str_replace('->', '.', $name); + } + + private function blinkKey(): string + { + $connection = [ + Arr::get($this->config, 'client.nodes', $this->config['nodes'] ?? []), + Arr::get($this->config, 'client.api_key', Arr::get($this->config, 'credentials.api_key')), + ]; + + return 'search-typesense-schema-'.hash('sha256', serialize($connection)).'-'.$this->collection; + } +} diff --git a/tests/Console/Commands/InstallSearchTest.php b/tests/Console/Commands/InstallSearchTest.php new file mode 100644 index 00000000000..2aa6fdec9d6 --- /dev/null +++ b/tests/Console/Commands/InstallSearchTest.php @@ -0,0 +1,364 @@ +directory = sys_get_temp_dir().'/statamic-install-search-'.Str::random(12); + $this->app->useEnvironmentPath($this->directory); + $this->app->useConfigPath($this->directory.'/config'); + $this->configPath = config_path('statamic/search.php'); + File::ensureDirectoryExists(dirname($this->configPath)); + File::put($this->configPath, File::get(__DIR__.'/../../../config/search.php')); + File::put($this->directory.'/.env', "APP_NAME=Existing\n"); + Http::preventStrayRequests(); + } + + public function tearDown(): void + { + File::deleteDirectory($this->directory); + + parent::tearDown(); + } + + #[Test] + public function local_search_needs_no_packages_or_credentials() + { + Composer::shouldReceive('isInstalled')->never(); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'default']) + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $this->assertSame('local', (require $this->configPath)['indexes']['default']['driver']); + $this->assertSame("APP_NAME=Existing\n", File::get($this->directory.'/.env')); + Http::assertNothingSent(); + } + + #[Test] + public function it_prompts_for_the_driver_and_index_when_not_supplied() + { + $this->artisan(InstallSearch::class) + ->expectsChoice('Which search driver would you like to use?', 'local', ['local' => 'Local', 'algolia' => 'Algolia', 'typesense' => 'Typesense']) + ->expectsQuestion('Search index handle', 'blog') + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $this->assertSame('local', (require $this->configPath)['indexes']['blog']['driver']); + $this->assertSame(['title'], (require $this->configPath)['indexes']['blog']['fields']); + } + + #[Test] + public function algolia_installs_its_sdk_and_keeps_secrets_out_of_php_config() + { + Composer::shouldReceive('isInstalled')->with('algolia/algoliasearch-client-php')->once()->andReturnFalse(); + Composer::shouldReceive('withoutQueue')->once()->andReturnSelf(); + Composer::shouldReceive('throwOnFailure')->once()->andReturnSelf(); + Composer::shouldReceive('require')->with('algolia/algoliasearch-client-php', '^4.32')->once(); + Http::fake(['*.algolia.net/*' => Http::response(['items' => []])]); + + $this->artisan(InstallSearch::class, ['driver' => 'algolia', '--index' => 'default']) + ->expectsQuestion('Algolia application ID', 'APP123') + ->expectsQuestion('Algolia admin API key', 'admin-secret') + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $contents = File::get($this->configPath); + $this->assertSame('algolia', (require $this->configPath)['indexes']['default']['driver']); + $this->assertStringNotContainsString('admin-secret', $contents); + $this->assertStringContainsString("env('ALGOLIA_SECRET')", $contents); + $this->assertSame('admin-secret', $this->environment()['ALGOLIA_SECRET']); + Http::assertSent(fn ($request) => $request->hasHeader('X-Algolia-API-Key', 'admin-secret')); + } + + #[Test] + public function typesense_accepts_a_cloud_url_and_leaves_ai_features_optional() + { + Composer::shouldReceive('isInstalled')->with('typesense/typesense-php')->once()->andReturnTrue(); + Http::fake(['https://cluster.typesense.net/*' => Http::response([])]); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'default']) + ->expectsQuestion('Typesense URL', 'https://cluster.typesense.net/proxy/') + ->expectsQuestion('Typesense admin API key', 'admin-secret') + ->expectsQuestion('Typesense search-only API key (optional)', '') + ->expectsConfirmation('Configure semantic search?', false) + ->expectsConfirmation('Configure natural language search?', false) + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $env = $this->environment(); + $this->assertSame('cluster.typesense.net', $env['TYPESENSE_HOST']); + $this->assertSame('443', $env['TYPESENSE_PORT']); + $this->assertSame('/proxy', $env['TYPESENSE_PATH']); + $this->assertSame('typesense', (require $this->configPath)['indexes']['default']['driver']); + $this->assertArrayNotHasKey('settings', (require $this->configPath)['indexes']['default']); + $this->assertStringNotContainsString('admin-secret', File::get($this->configPath)); + Http::assertSent(fn ($request) => $request->url() === 'https://cluster.typesense.net/proxy/collections'); + } + + #[Test] + public function typesense_can_configure_semantic_search_and_an_existing_nl_model() + { + config(['statamic.search.indexes.default.fields' => ['title', 'content']]); + File::put($this->configPath, str_replace("'fields' => ['title']", "'fields' => ['title', 'content']", File::get($this->configPath))); + Composer::shouldReceive('isInstalled')->andReturnTrue(); + Http::fake(['https://cluster.typesense.net/*' => Http::response([])]); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'default']) + ->expectsQuestion('Typesense URL', 'https://cluster.typesense.net') + ->expectsQuestion('Typesense admin API key', 'admin-secret') + ->expectsQuestion('Typesense search-only API key (optional)', 'search-secret') + ->expectsConfirmation('Configure semantic search?', 'yes') + ->expectsChoice('Fields to use for semantic search', ['title'], ['title', 'content']) + ->expectsConfirmation('Configure natural language search?', 'yes') + ->expectsChoice('Natural language model setup', 'existing', [ + 'existing' => 'Use an existing Typesense model', + 'openai' => 'Configure an OpenAI model', + ]) + ->expectsQuestion('Typesense natural language model ID', 'articles') + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $settings = (require $this->configPath)['indexes']['default']['settings']; + $this->assertSame(['title'], $settings['embedding']['from']); + $this->assertSame('ts/all-MiniLM-L12-v2', $settings['embedding']['model_config']['model_name']); + $this->assertSame('articles', $settings['natural_language']['model']); + $this->assertSame('search-secret', $this->environment()['TYPESENSE_SEARCH_ONLY_API_KEY']); + Http::assertSent(fn ($request) => str_ends_with($request->url(), '/nl_search_models/articles')); + } + + #[Test] + public function it_can_configure_a_new_openai_model_without_writing_its_key_into_php() + { + Composer::shouldReceive('isInstalled')->andReturnTrue(); + Http::fake(['https://cluster.typesense.net/*' => Http::response([])]); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'blog']) + ->expectsQuestion('Typesense URL', 'https://cluster.typesense.net') + ->expectsQuestion('Typesense admin API key', 'admin-secret') + ->expectsQuestion('Typesense search-only API key (optional)', '') + ->expectsConfirmation('Configure semantic search?', 'no') + ->expectsConfirmation('Configure natural language search?', 'yes') + ->expectsChoice('Natural language model setup', 'openai', [ + 'existing' => 'Use an existing Typesense model', + 'openai' => 'Configure an OpenAI model', + ]) + ->expectsQuestion('Typesense natural language model ID', 'blog_model') + ->expectsQuestion('OpenAI model name', 'gpt-4.1-mini') + ->expectsQuestion('OpenAI API key', 'llm-secret') + ->expectsConfirmation('Build the search index now?', 'no') + ->assertSuccessful(); + + $settings = (require $this->configPath)['indexes']['blog']['settings']; + $this->assertSame('blog_model', $settings['natural_language']['model']); + $this->assertSame('openai/gpt-4.1-mini', $settings['natural_language']['model_config']['model_name']); + $this->assertSame('llm-secret', $this->environment()['TYPESENSE_BLOG_NL_API_KEY']); + $this->assertStringNotContainsString('llm-secret', File::get($this->configPath)); + } + + #[Test] + public function updating_env_keys_preserves_unrelated_values_and_escapes_special_characters() + { + File::put($this->directory.'/.env', "APP_NAME=Existing\nALGOLIA_SECRET=old\n"); + $key = 'with"quotes\\slashes${VARIABLE}#hash'; + Composer::shouldReceive('isInstalled')->andReturnTrue(); + Http::fake(['*.algolia.net/*' => Http::response(['items' => []])]); + + $this->artisan(InstallSearch::class, ['driver' => 'algolia', '--index' => 'default']) + ->expectsQuestion('Algolia application ID', 'APP123') + ->expectsQuestion('Algolia admin API key', $key) + ->expectsConfirmation('Build the search index now?', 'no') + ->assertSuccessful(); + + $this->assertSame($key, $this->environment()['ALGOLIA_SECRET']); + $this->assertSame('Existing', $this->environment()['APP_NAME']); + $this->assertSame(1, substr_count(File::get($this->directory.'/.env'), 'ALGOLIA_SECRET=')); + } + + #[Test] + public function invalid_credentials_leave_existing_config_and_environment_untouched() + { + $original = File::get($this->configPath); + Http::fake(['*' => Http::response(['message' => 'Forbidden'], 403)]); + Composer::shouldReceive('isInstalled')->never(); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'default']) + ->expectsQuestion('Typesense URL', 'https://cluster.typesense.net') + ->expectsQuestion('Typesense admin API key', 'bad-key') + ->assertFailed(); + + $this->assertSame($original, File::get($this->configPath)); + $this->assertSame("APP_NAME=Existing\n", File::get($this->directory.'/.env')); + } + + #[Test] + public function it_preserves_custom_index_code_and_other_indexes_when_switching_drivers() + { + $source = <<<'PHP' + [ + 'blog' => [ + 'driver' => 'algolia', + 'fields' => ['title', 'content'], + 'searchables' => 'collection:blog', + 'filter' => fn ($entry) => $entry->published(), + 'transformers' => ['title' => fn ($value) => strtoupper($value)], + 'settings' => ['searchableAttributes' => ['title']], + ], + 'other' => ['driver' => 'local', 'fields' => ['title']], + ], +]; +PHP; + File::put($this->configPath, $source); + config(['statamic.search.indexes.blog' => (require $this->configPath)['indexes']['blog']]); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'blog']) + ->expectsConfirmation('Build the search index now?', false) + ->assertSuccessful(); + + $config = require $this->configPath; + $this->assertSame('local', $config['indexes']['blog']['driver']); + $this->assertSame(['title', 'content'], $config['indexes']['blog']['fields']); + $this->assertSame('collection:blog', $config['indexes']['blog']['searchables']); + $this->assertSame('TITLE', $config['indexes']['blog']['transformers']['title']('title')); + $this->assertArrayNotHasKey('settings', $config['indexes']['blog']); + $this->assertSame(['driver' => 'local', 'fields' => ['title']], $config['indexes']['other']); + $this->assertStringContainsString('// Keep this comment and the transformer.', File::get($this->configPath)); + } + + #[Test] + public function building_uses_a_fresh_process_and_passes_the_selected_index() + { + Process::fake(); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'blog', '--build' => true]) + ->assertSuccessful(); + + Process::assertRan(fn ($process) => array_slice($process->command, -2) === ['statamic:search:update', 'blog']); + } + + #[Test] + public function it_publishes_the_search_config_when_it_is_missing() + { + File::delete($this->configPath); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'blog']) + ->expectsConfirmation('Build the search index now?', 'no') + ->assertSuccessful(); + + $this->assertSame('local', (require $this->configPath)['indexes']['blog']['driver']); + $this->assertArrayHasKey('algolia', (require $this->configPath)['drivers']); + } + + #[Test] + public function legacy_typesense_client_credentials_do_not_override_the_new_connection() + { + config(['statamic.search.drivers.typesense.client' => [ + 'api_key' => 'legacy-secret', + 'nodes' => [['host' => 'old.test', 'port' => '8108', 'protocol' => 'http']], + 'nearest_node' => ['host' => 'old.test', 'port' => '8108', 'protocol' => 'http'], + 'num_retries' => 3, + ]]); + Composer::shouldReceive('isInstalled')->andReturnTrue(); + Http::fake(['https://new.test/*' => Http::response([])]); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'blog']) + ->expectsQuestion('Typesense URL', 'https://new.test') + ->expectsQuestion('Typesense admin API key', 'new-secret') + ->expectsQuestion('Typesense search-only API key (optional)', '') + ->expectsConfirmation('Configure semantic search?', 'no') + ->expectsConfirmation('Configure natural language search?', 'no') + ->expectsConfirmation('Build the search index now?', 'no') + ->assertSuccessful(); + + $index = (require $this->configPath)['indexes']['blog']; + $this->assertSame(['num_retries' => 3], $index['client']); + $this->assertSame('new.test', $this->environment()['TYPESENSE_BLOG_HOST']); + $this->assertSame('new-secret', $this->environment()['TYPESENSE_BLOG_API_KEY']); + $this->assertArrayNotHasKey('TYPESENSE_API_KEY', $this->environment()); + } + + #[Test] + public function a_missing_natural_language_model_leaves_files_untouched() + { + $original = File::get($this->configPath); + Http::fake([ + 'https://cluster.typesense.net/collections' => Http::response([]), + 'https://cluster.typesense.net/nl_search_models/missing' => Http::response([], 404), + ]); + + $this->artisan(InstallSearch::class, ['driver' => 'typesense', '--index' => 'default']) + ->expectsQuestion('Typesense URL', 'https://cluster.typesense.net') + ->expectsQuestion('Typesense admin API key', 'admin-secret') + ->expectsQuestion('Typesense search-only API key (optional)', '') + ->expectsConfirmation('Configure semantic search?', 'no') + ->expectsConfirmation('Configure natural language search?', 'yes') + ->expectsChoice('Natural language model setup', 'existing', [ + 'existing' => 'Use an existing Typesense model', + 'openai' => 'Configure an OpenAI model', + ]) + ->expectsQuestion('Typesense natural language model ID', 'missing') + ->assertFailed(); + + $this->assertSame($original, File::get($this->configPath)); + $this->assertSame("APP_NAME=Existing\n", File::get($this->directory.'/.env')); + } + + #[Test] + public function a_failed_build_returns_a_failure_without_discarding_the_saved_configuration() + { + Process::fake([Process::result(exitCode: 1)]); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'blog', '--build' => true]) + ->assertFailed(); + + $this->assertSame('local', (require $this->configPath)['indexes']['blog']['driver']); + } + + #[Test] + public function unsupported_dynamic_config_is_not_rewritten_or_evaluated_again() + { + $source = " []]);\n"; + File::put($this->configPath, $source); + + $this->artisan(InstallSearch::class, ['driver' => 'local', '--index' => 'blog']) + ->assertFailed(); + + $this->assertSame($source, File::get($this->configPath)); + } + + #[Test] + public function invalid_drivers_are_rejected_without_changes() + { + $original = File::get($this->configPath); + + $this->artisan(InstallSearch::class, ['driver' => 'unknown'])->assertFailed(); + + $this->assertSame($original, File::get($this->configPath)); + } + + private function environment(): array + { + return Dotenv::parse(File::get($this->directory.'/.env')); + } +} diff --git a/tests/Search/IndexManagerTest.php b/tests/Search/IndexManagerTest.php index 0de50496a57..e6f2212bc85 100644 --- a/tests/Search/IndexManagerTest.php +++ b/tests/Search/IndexManagerTest.php @@ -7,6 +7,8 @@ use Statamic\Search\Comb\Index as CombIndex; use Statamic\Search\IndexManager; use Statamic\Search\Null\NullIndex; +use Statamic\Search\Typesense\Index as TypesenseIndex; +use Tests\Search\Typesense\FakeHttpClient; use Tests\TestCase; class IndexManagerTest extends TestCase @@ -126,4 +128,69 @@ public function it_uses_the_cp_index_from_the_config_if_it_exists() 'searchables' => ['collections:pages', 'collections:blog'], ], $manager->index('cp')->config()); } + + #[Test] + public function it_builds_a_typesense_driver() + { + config([ + 'statamic.search.indexes.typesense' => [ + 'driver' => 'typesense', + 'searchables' => [], + 'fields' => ['title'], + ], + 'statamic.search.drivers.typesense' => [ + 'credentials' => [ + 'api_key' => 'admin-key', + 'search_only_api_key' => 'search-key', + ], + 'nodes' => [ + [ + 'host' => 'typesense.test', + 'port' => '8108', + 'protocol' => 'http', + ], + ], + 'client' => [ + 'client' => new FakeHttpClient, + 'num_retries' => 0, + 'randomize_nodes' => false, + ], + ], + ]); + + $manager = new IndexManager($this->app); + + $this->assertInstanceOf(TypesenseIndex::class, $manager->index('typesense')); + } + + #[Test] + public function it_accepts_the_rad_pack_typesense_client_configuration() + { + config([ + 'statamic.search.indexes.typesense' => [ + 'driver' => 'typesense', + 'searchables' => [], + 'fields' => ['title'], + ], + 'statamic.search.drivers.typesense' => [ + 'client' => [ + 'api_key' => 'admin-key', + 'nodes' => [ + [ + 'host' => 'typesense.test', + 'port' => '8108', + 'protocol' => 'http', + ], + ], + 'client' => new FakeHttpClient, + 'num_retries' => 0, + 'randomize_nodes' => false, + ], + ], + ]); + + $manager = new IndexManager($this->app); + + $this->assertInstanceOf(TypesenseIndex::class, $manager->index('typesense')); + } } diff --git a/tests/Search/Typesense/FakeHttpClient.php b/tests/Search/Typesense/FakeHttpClient.php new file mode 100644 index 00000000000..ba075f29686 --- /dev/null +++ b/tests/Search/Typesense/FakeHttpClient.php @@ -0,0 +1,51 @@ +responses[] = new Response( + $status, + ['Content-Type' => 'application/json'], + is_array($body) ? json_encode($body, JSON_THROW_ON_ERROR) : $body, + ); + + return $this; + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = [ + 'method' => $request->getMethod(), + 'uri' => (string) $request->getUri(), + 'body' => (string) $request->getBody(), + 'headers' => $request->getHeaders(), + ]; + + if ($this->responses === []) { + throw new \RuntimeException("No fake response queued for {$request->getMethod()} {$request->getUri()}."); + } + + return array_shift($this->responses); + } + + public function request(int $index): array + { + return $this->requests[$index]; + } + + public function json(int $index): array + { + return json_decode($this->request($index)['body'], true, flags: JSON_THROW_ON_ERROR); + } +} diff --git a/tests/Search/Typesense/MakesTypesenseIndexes.php b/tests/Search/Typesense/MakesTypesenseIndexes.php new file mode 100644 index 00000000000..085c417ad09 --- /dev/null +++ b/tests/Search/Typesense/MakesTypesenseIndexes.php @@ -0,0 +1,50 @@ + 'admin-key', + 'nodes' => [ + [ + 'host' => 'typesense.test', + 'port' => '8108', + 'protocol' => 'http', + ], + ], + 'num_retries' => 0, + 'randomize_nodes' => false, + 'client' => $http, + ]); + + $config = array_replace_recursive([ + 'fields' => ['title'], + 'searchables' => [], + 'settings' => [ + 'search_parameters' => [ + 'query_by' => 'title', + ], + ], + 'credentials' => [ + 'api_key' => 'admin-key', + 'search_only_api_key' => 'search-key', + ], + ], $config); + + return [ + new Index($client, $name, $config, $locale), + $http, + $client, + ]; + } +} diff --git a/tests/Search/TypesenseFilterTranslatorTest.php b/tests/Search/TypesenseFilterTranslatorTest.php new file mode 100644 index 00000000000..b374b4163f8 --- /dev/null +++ b/tests/Search/TypesenseFilterTranslatorTest.php @@ -0,0 +1,210 @@ +schema([ + 'title' => ['type' => 'string'], + 'price' => ['type' => 'float'], + 'published' => ['type' => 'bool'], + ])); + + $this->assertSame( + "((title:=`Jack's \\`book\\`` && price:>=19.95) && published:=true)", + $translator->filters([ + $this->basic('title', '=', "Jack's `book`"), + $this->basic('price', '>=', '19.95'), + $this->basic('published', '=', 1), + ]), + ); + } + + #[Test] + public function it_translates_lists_ranges_and_array_semantics() + { + $translator = new FilterTranslator($this->schema([ + 'status' => ['type' => 'string'], + 'price' => ['type' => 'int32'], + 'tags' => ['type' => 'string[]'], + ])); + + $this->assertSame( + '(((status:=[`published`,`scheduled`] && (price:>=10 && price:<=20)) && (tags:=`php` && tags:=`cms`)) && tags:!=[`draft`,`private`])', + $translator->filters([ + ['type' => 'In', 'column' => 'status', 'values' => ['published', 'scheduled'], 'boolean' => 'and'], + ['type' => 'Between', 'column' => 'price', 'values' => [10, 20], 'boolean' => 'and'], + ['type' => 'JsonContains', 'column' => 'tags', 'values' => ['php', 'cms'], 'boolean' => 'and'], + ['type' => 'JsonDoesntOverlap', 'column' => 'tags', 'values' => ['draft', 'private'], 'boolean' => 'and'], + ]), + ); + } + + #[Test] + public function it_translates_nested_boolean_groups() + { + $translator = new FilterTranslator($this->schema([ + 'status' => ['type' => 'string'], + 'site' => ['type' => 'string'], + ])); + + $nested = new class + { + public function rawWheres(): array + { + return [ + [ + 'type' => 'Basic', + 'column' => 'site', + 'operator' => '=', + 'value' => 'en', + 'boolean' => 'and', + ], + [ + 'type' => 'Basic', + 'column' => 'site', + 'operator' => '=', + 'value' => 'fr', + 'boolean' => 'or', + ], + ]; + } + }; + + $this->assertSame( + '(status:=`published` && (site:=`en` || site:=`fr`))', + $translator->filters([ + $this->basic('status', '=', 'published'), + ['type' => 'Nested', 'query' => $nested, 'boolean' => 'and'], + ]), + ); + + $this->assertSame( + '((site:=`en` || site:=`fr`) && status:=`published`)', + $translator->filters([ + $this->basic('site', '=', 'en'), + array_merge($this->basic('site', '=', 'fr'), ['boolean' => 'or']), + $this->basic('status', '=', 'published'), + ]), + ); + + $this->assertSame( + '(status:=`published` && site:!=[`fr`])', + $translator->filters([ + $this->basic('status', '=', 'published'), + ['type' => 'NotIn', 'column' => 'site', 'values' => ['fr'], 'boolean' => 'or'], + ]), + ); + } + + #[Test] + public function unsupported_filters_and_sorts_return_null_instead_of_being_dropped() + { + $translator = new FilterTranslator($this->schema([ + 'title' => ['type' => 'string', 'sort' => false], + 'date' => ['type' => 'int64', 'sort' => true], + ])); + + $this->assertNull($translator->filters([ + $this->basic('title', 'like', '%foo%'), + ])); + $this->assertNull($translator->filters([ + $this->basic('title', '>', 'foo'), + ])); + $this->assertNull($translator->filters([ + ['type' => 'Between', 'column' => 'title', 'values' => ['a', 'z'], 'boolean' => 'and'], + ])); + $this->assertNull($translator->filters([ + ['type' => 'Null', 'column' => 'title', 'boolean' => 'and'], + ])); + $this->assertNull($translator->filters([ + [ + 'type' => 'Nested', + 'query' => new class + { + public function rawWheres(): array + { + return []; + } + }, + 'boolean' => 'and', + ], + ])); + $this->assertNull($translator->filters([ + $this->basic('unknown', '=', 'foo'), + ])); + $this->assertNull($translator->sorts([new OrderBy('title', 'asc')])); + $this->assertNull($translator->sorts([ + new OrderBy('date', 'asc'), + new OrderBy('date', 'desc'), + new OrderBy('date', 'asc'), + new OrderBy('date', 'desc'), + ])); + $this->assertSame('date:desc', $translator->sorts([new OrderBy('date', 'desc')])); + } + + #[Test] + public function fractional_integer_operands_fall_back_without_truncation() + { + $translator = new FilterTranslator($this->schema(['price' => ['type' => 'int32']])); + + $this->assertNull($translator->filters([$this->basic('price', '<', 1.5)])); + $this->assertNull($translator->filters([ + ['type' => 'Between', 'column' => 'price', 'values' => [1.5, 3]], + ])); + $this->assertSame('price:<2', $translator->filters([$this->basic('price', '<', 2.0)])); + } + + #[Test] + public function array_membership_is_only_pushed_down_for_json_clauses() + { + $translator = new FilterTranslator($this->schema([ + 'tags' => ['type' => 'string[]'], + 'title' => ['type' => 'string'], + ])); + + foreach (['In', 'NotIn'] as $type) { + $this->assertNull($translator->filters([ + ['type' => $type, 'column' => 'tags', 'values' => ['red']], + ])); + } + + $this->assertNull($translator->filters([$this->basic('tags', '=', 'red')])); + $this->assertNull($translator->filters([ + ['type' => 'JsonContains', 'column' => 'title', 'values' => ['red']], + ])); + $this->assertSame('tags:=[`red`]', $translator->filters([ + ['type' => 'JsonOverlaps', 'column' => 'tags', 'values' => ['red']], + ])); + } + + private function basic(string $column, string $operator, mixed $value): array + { + return compact('column', 'operator', 'value') + [ + 'type' => 'Basic', + 'boolean' => 'and', + ]; + } + + private function schema(array $fields): Schema + { + $schema = Mockery::mock(Schema::class); + $schema->shouldReceive('field')->andReturnUsing(fn (string $name) => $fields[$name] ?? null); + $schema->shouldReceive('filterable')->andReturnUsing(fn (string $name) => isset($fields[$name])); + $schema->shouldReceive('sortable')->andReturnUsing( + fn (string $name) => ($fields[$name]['sort'] ?? false) === true + ); + + return $schema; + } +} diff --git a/tests/Search/TypesenseIndexTest.php b/tests/Search/TypesenseIndexTest.php new file mode 100644 index 00000000000..8ec1eafdb22 --- /dev/null +++ b/tests/Search/TypesenseIndexTest.php @@ -0,0 +1,470 @@ +makeTypesenseIndex($config, $locale, $name)[0]; + } + + #[Test] + public function it_reindexes_by_atomically_swapping_an_alias() + { + [$index, $http] = $this->makeTypesenseIndex(name: 'articles'); + + $http + ->respond(['name' => 'articles', 'collection_name' => 'articles_old']) + ->respond(['name' => 'articles_old', 'synonym_sets' => ['dashboard'], 'curation_sets' => ['featured']]) + ->respond(['name' => 'articles_new']) + ->respond(['name' => 'articles', 'collection_name' => 'articles_new']) + ->respond([]); + + $index->update(); + + $this->assertCount(5, $http->requests); + $this->assertSame('GET', $http->request(0)['method']); + $this->assertStringContainsString('/aliases/articles', $http->request(0)['uri']); + $this->assertSame('POST', $http->request(2)['method']); + $this->assertStringEndsWith('/collections', $http->request(2)['uri']); + + $candidate = $http->json(2)['name']; + $this->assertSame(['dashboard'], $http->json(2)['synonym_sets']); + $this->assertSame(['featured'], $http->json(2)['curation_sets']); + $this->assertStringStartsWith('articles_', $candidate); + $this->assertSame($candidate, $http->json(3)['collection_name']); + $this->assertStringContainsString('/collections/articles_old', $http->request(4)['uri']); + } + + #[Test] + public function it_migrates_a_legacy_plain_collection_to_an_alias() + { + [$index, $http] = $this->makeTypesenseIndex(name: 'articles'); + + $http + ->respond(['message' => 'Alias not found'], 404) + ->respond(['name' => 'articles']) + ->respond(['name' => 'articles']) + ->respond(['name' => 'articles_new']) + ->respond([]) + ->respond(['name' => 'articles']); + + $index->update(); + + $this->assertCount(6, $http->requests); + $this->assertSame('DELETE', $http->request(4)['method']); + $this->assertStringContainsString('/collections/articles', $http->request(4)['uri']); + $this->assertSame('PUT', $http->request(5)['method']); + $this->assertStringContainsString('/aliases/articles', $http->request(5)['uri']); + } + + #[Test] + public function it_does_not_delete_a_candidate_when_a_failed_alias_response_was_applied() + { + [$index, $http] = $this->makeTypesenseIndex(name: 'articles'); + Carbon::setTestNow('2026-10-23 10:00:00'); + Str::createRandomStringsUsing(fn () => 'ABCDEF'); + + $http + ->respond(['name' => 'articles', 'collection_name' => 'articles_old']) + ->respond(['name' => 'articles_old', 'synonym_sets' => ['dashboard'], 'curation_sets' => ['featured']]) + ->respond(['name' => 'articles_new']) + ->respond(['message' => 'Upstream timed out'], 500) + ->respond([ + 'name' => 'articles', + 'collection_name' => 'articles_20261023100000_abcdef', + ]); + + try { + $index->update(); + $this->fail('The failed alias response should be surfaced.'); + } catch (TypesenseClientError) { + // + } finally { + Str::createRandomStringsNormally(); + Carbon::setTestNow(); + } + + $this->assertCount(5, $http->requests); + $this->assertSame('GET', $http->request(4)['method']); + $this->assertStringContainsString('/aliases/articles', $http->request(4)['uri']); + } + + #[Test] + public function configured_sets_override_dashboard_sets_during_a_rebuild() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => ['schema' => ['synonym_sets' => [], 'curation_sets' => ['configured']]], + ]); + $http + ->respond(['collection_name' => 'test_old']) + ->respond(['synonym_sets' => ['dashboard'], 'curation_sets' => ['featured']]) + ->respond([]) + ->respond([]) + ->respond([]); + + $index->update(); + + $this->assertSame([], $http->json(2)['synonym_sets']); + $this->assertSame(['configured'], $http->json(2)['curation_sets']); + } + + #[Test] + public function a_failed_collection_creation_does_not_delete_an_existing_collection() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http + ->respond(['collection_name' => 'test_old']) + ->respond(['name' => 'test_old']) + ->respond(['message' => 'A collection with this name already exists'], 409); + + try { + $index->update(); + $this->fail('The collection conflict should be surfaced.'); + } catch (TypesenseClientError) { + $this->assertCount(3, $http->requests); + } + } + + #[Test] + public function a_failed_import_keeps_the_existing_alias_and_removes_the_candidate() + { + \Statamic\Facades\Collection::make('blog')->save(); + EntryFactory::collection('blog')->id('one')->data(['title' => 'One'])->create(); + [$index, $http] = $this->makeTypesenseIndex(['searchables' => 'collection:blog']); + $http + ->respond(['collection_name' => 'test_old']) + ->respond(['name' => 'test_old']) + ->respond([]) + ->respond('{"success":false,"error":"Invalid document"}') + ->respond(['collection_name' => 'test_old']) + ->respond([]); + + try { + $index->update(); + $this->fail('The import failure should be surfaced.'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('Invalid document', $e->getMessage()); + } + + $candidate = $http->json(2)['name']; + $this->assertStringContainsString('/collections/'.$candidate.'/documents/import', $http->request(3)['uri']); + $this->assertSame('entry::one', $http->json(3)['reference']); + $this->assertSame('DELETE', $http->request(5)['method']); + $this->assertStringEndsWith('/collections/'.$candidate, $http->request(5)['uri']); + $this->assertNotContains('PUT', array_column($http->requests, 'method')); + } + + #[Test] + public function it_can_reindex_in_place_when_aliases_are_not_wanted() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => ['reindex' => 'in_place'], + ]); + + $http + ->respond(['message' => 'Alias not found'], 404) + ->respond(['message' => 'Collection not found'], 404) + ->respond(['name' => 'test']); + + $index->update(); + + $this->assertCount(3, $http->requests); + $this->assertSame('test', $http->json(2)['name']); + } + + #[Test] + public function it_only_checks_existence_once_per_request() + { + [$index, $http] = $this->makeTypesenseIndex(name: 'existence'); + + $http + ->respond(['message' => 'Alias not found'], 404) + ->respond(['name' => 'existence']); + + $this->assertTrue($index->exists()); + $this->assertTrue($index->exists()); + $this->assertCount(2, $http->requests); + } + + #[Test] + public function it_imports_documents_using_url_safe_ids_and_checks_each_response() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond('{"success":true}'); + + $index->insertDocuments(new Documents([ + 'asset::images::photos/cat.jpg' => ['title' => 'Cat'], + ])); + + $document = json_decode($http->request(0)['body'], true, flags: JSON_THROW_ON_ERROR); + + $this->assertSame( + hash('sha256', 'asset::images::photos/cat.jpg'), + $document['id'], + ); + $this->assertSame('asset::images::photos/cat.jpg', $document['reference']); + $this->assertStringContainsString('action=upsert', $http->request(0)['uri']); + $this->assertStringContainsString('dirty_values=coerce_or_drop', $http->request(0)['uri']); + } + + #[Test] + public function it_adds_typed_statamic_metadata_to_documents() + { + [$index] = $this->makeTypesenseIndex(); + $entry = EntryFactory::id('one') + ->collection('blog') + ->data(['title' => 'One']) + ->make(); + + $this->assertSame([ + 'title' => 'One', + 'reference' => 'entry::one', + 'type' => 'entry', + 'status' => 'published', + 'site' => 'en', + 'collection' => 'blog', + ], $index->fields($entry)); + } + + #[Test] + public function it_surfaces_individual_import_failures() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond('{"success":false,"error":"Field `date` must be an int64."}'); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Field `date` must be an int64.'); + + $index->insertDocuments(new Documents([ + 'entry::one' => ['title' => 'One'], + ])); + } + + #[Test] + public function deleting_an_already_missing_document_is_a_noop() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond(['message' => 'Document not found'], 404); + + $document = Mockery::mock(\Statamic\Contracts\Search\Searchable::class); + $document->shouldReceive('getSearchReference')->andReturn('entry::one'); + + $index->delete($document); + + $this->assertCount(1, $http->requests); + } + + #[Test] + public function it_generates_a_scoped_frontend_key_with_published_site_constraints() + { + Carbon::setTestNow('2026-10-23 12:00:00'); + [$index, , $client] = $this->makeTypesenseIndex([ + 'settings' => [ + 'search_parameters' => [ + 'filter_by' => 'collection:=blog', + ], + ], + ], locale: 'en'); + + $parameters = [ + 'filter_by' => 'status:=published && site:=`en` && (collection:=blog) && (type:=entry)', + 'expires_at' => now()->addMinute()->timestamp, + ]; + + $this->assertSame( + $client->keys->generateScopedSearchKey('search-key', $parameters), + $index->scopedSearchKey(['filter_by' => 'type:=entry'], 60), + ); + } + + #[Test] + public function a_single_site_scoped_key_does_not_filter_an_unlocalized_index_by_site() + { + Carbon::setTestNow('2026-10-23 12:00:00'); + [$index, , $client] = $this->makeTypesenseIndex(); + + $parameters = [ + 'filter_by' => 'status:=published', + 'expires_at' => now()->addHour()->timestamp, + ]; + + $this->assertSame( + $client->keys->generateScopedSearchKey('search-key', $parameters), + $index->scopedSearchKey(), + ); + } + + #[Test] + public function documents_missing_the_default_sorting_field_still_import() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'schema' => ['default_sorting_field' => 'date'], + ], + ]); + $http->respond('{"success":true}'); + + $index->insertDocuments(new Documents([ + 'entry::one' => ['title' => 'One'], + ])); + + $this->assertSame(0, $http->json(0)['date']); + } + + #[Test] + public function it_creates_and_caches_a_natural_language_model() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'natural_language' => [ + 'model' => 'statamic_articles', + 'model_config' => [ + 'model_name' => 'openai/gpt-4.1-mini', + 'api_key' => 'llm-key', + ], + ], + ], + ]); + + $http + ->respond(['message' => 'Model not found'], 404) + ->respond(['id' => 'statamic_articles']); + + $this->assertSame('statamic_articles', $index->ensureNaturalLanguageModel()); + $this->assertSame('statamic_articles', $index->ensureNaturalLanguageModel()); + $this->assertCount(2, $http->requests); + $this->assertSame([ + 'model_name' => 'openai/gpt-4.1-mini', + 'api_key' => 'llm-key', + 'id' => 'statamic_articles', + ], $http->json(1)); + } + + #[Test] + public function it_exposes_all_highlights_snippets_and_text_match_data() + { + [$index] = $this->makeTypesenseIndex(); + $result = Mockery::mock(Result::class); + $result->shouldReceive('getRawResult')->andReturn([ + 'search_text_match' => 123, + 'search_highlights' => [ + ['field' => 'title', 'snippet' => 'Red shirt'], + 'content' => ['snippet' => 'A shirt'], + ], + ]); + + $this->assertSame([ + 'search_highlights' => [ + 'title' => ['field' => 'title', 'snippet' => 'Red shirt'], + 'content' => ['snippet' => 'A shirt'], + ], + 'search_snippets' => [ + 'title' => 'Red shirt', + 'content' => 'A shirt', + ], + 'search_text_match' => 123, + ], $index->extraAugmentedResultData($result)); + } + + #[Test] + public function it_flattens_grouped_hits_from_dashboard_presets() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond([ + 'results' => [ + [ + 'found' => 2, + 'grouped_hits' => [ + [ + 'group_key' => ['USA'], + 'hits' => [ + [ + 'document' => ['reference' => 'entry::one', 'title' => 'One'], + 'text_match' => 90, + ], + ], + ], + [ + 'group_key' => ['UK'], + 'hits' => [ + [ + 'document' => ['reference' => 'entry::two', 'title' => 'Two'], + 'text_match' => 80, + ], + ], + ], + ], + ], + ], + ]); + + $response = $index->searchUsingApi('red'); + + $this->assertSame(2, $response['found']); + $this->assertSame(['entry::one', 'entry::two'], $response['hits']->pluck('reference')->all()); + $this->assertSame([2, 1], $response['hits']->pluck('search_score')->all()); + } + + #[Test] + public function it_maps_missing_collection_errors_to_statamics_exception() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond([ + 'results' => [ + [ + 'code' => 404, + 'error' => 'Could not find a collection named `test`.', + ], + ], + ]); + + $this->expectException(IndexNotFoundException::class); + $this->expectExceptionMessage('Index [test] does not exist.'); + + $index->searchUsingApi('foo'); + } + + #[Test] + public function it_maps_unknown_search_fields_to_a_helpful_exception() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond([ + 'results' => [ + [ + 'code' => 400, + 'error' => 'Could not find a field named `subtitle` in the schema.', + ], + ], + ]); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Field [subtitle] does not exist in this Typesense index.'); + + $index->searchUsingApi('foo'); + } +} diff --git a/tests/Search/TypesenseQueryTest.php b/tests/Search/TypesenseQueryTest.php new file mode 100644 index 00000000000..53327286676 --- /dev/null +++ b/tests/Search/TypesenseQueryTest.php @@ -0,0 +1,376 @@ +makeTypesenseIndex(); + $http->respond($this->response([ + $this->hit('entry::one', 'One', 900), + $this->hit('entry::two', 'Two', 800), + $this->hit('entry::three', 'Three', 700), + ], 3)); + + $results = $index->search('red')->withoutData()->get(); + + $this->assertSame( + ['entry::one', 'entry::two', 'entry::three'], + $results->map(fn ($result) => $result->get('reference'))->all(), + ); + $this->assertSame([3, 2, 1], $results->map->getScore()->all()); + $this->assertSame(900, $results->first()->get('search_text_match')); + + $search = $http->json(0)['searches'][0]; + $this->assertSame('red', $search['q']); + $this->assertSame('title', $search['query_by']); + $this->assertSame(0, $search['offset']); + $this->assertSame(250, $search['limit']); + } + + #[Test] + public function it_pushes_supported_filters_sorting_and_limits_to_typesense() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http + ->respond([ + 'fields' => [ + ['name' => 'status', 'type' => 'string'], + ['name' => 'date', 'type' => 'int64', 'sort' => true], + ], + ]) + ->respond($this->response([$this->hit('entry::one', 'One')], 1)); + + $results = $index + ->search('red') + ->withoutData() + ->whereStatus('published') + ->orderBy('date', 'desc') + ->offset(5) + ->limit(20) + ->get(); + + $this->assertCount(1, $results); + $search = $http->json(1)['searches'][0]; + $this->assertSame('status:=`published`', $search['filter_by']); + $this->assertSame('date:desc', $search['sort_by']); + $this->assertSame(5, $search['offset']); + $this->assertSame(20, $search['limit']); + } + + #[Test] + public function it_falls_back_to_statamic_filtering_when_any_clause_cannot_be_translated() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http + ->respond([ + 'fields' => [ + ['name' => 'title', 'type' => 'string'], + ], + ]) + ->respond($this->response([ + $this->hit('entry::one', 'Red shirt'), + $this->hit('entry::two', 'Blue jeans'), + $this->hit('entry::three', 'Dark red hat'), + ], 3)); + + $results = $index + ->search('clothes') + ->withoutData() + ->where('title', 'like', '%red%') + ->get(); + + $this->assertSame( + ['entry::one', 'entry::three'], + $results->map(fn ($result) => $result->get('reference'))->all(), + ); + + $search = $http->json(1)['searches'][0]; + $this->assertArrayNotHasKey('filter_by', $search); + $this->assertSame(250, $search['limit']); + } + + #[Test] + public function pagination_uses_typesenses_found_count_and_server_side_window() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http + ->respond([ + 'fields' => [ + ['name' => 'status', 'type' => 'string'], + ], + ]) + ->respond($this->response([ + $this->hit('entry::twenty-one', 'Twenty one'), + $this->hit('entry::twenty-two', 'Twenty two'), + ], 42)); + + $paginator = $index + ->search('red') + ->withoutData() + ->whereStatus('published') + ->paginate(10, page: 3); + + $this->assertSame(42, $paginator->total()); + $this->assertCount(2, $paginator); + + $search = $http->json(1)['searches'][0]; + $this->assertSame(20, $search['offset']); + $this->assertSame(10, $search['limit']); + } + + #[Test] + public function pagination_is_capped_without_skipping_results() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond($this->response([], 1000)); + + $paginator = $index + ->search('red') + ->withoutData() + ->paginate(500, page: 2); + + $search = $http->json(0)['searches'][0]; + $this->assertSame(250, $paginator->perPage()); + $this->assertSame(250, $search['offset']); + $this->assertSame(250, $search['limit']); + } + + #[Test] + public function count_uses_typesenses_found_count_and_honors_offset_and_limit() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http->respond($this->response([], 42)); + + $count = $index + ->search('red') + ->withoutData() + ->offset(10) + ->limit(5) + ->count(); + + $this->assertSame(5, $count); + $this->assertSame(1, $http->json(0)['searches'][0]['limit']); + } + + #[Test] + public function it_configures_hybrid_search_and_excludes_embeddings_from_results() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'search_parameters' => [ + 'query_by' => ['title', 'author->name'], + 'include_fields' => ['title'], + 'exclude_fields' => ['reference', 'private->notes'], + ], + 'embedding' => [ + 'from' => ['title'], + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + 'alpha' => 0.4, + ], + ], + ]); + $http->respond($this->response([], 0)); + + $index->search('red')->withoutData()->get(); + + $search = $http->json(0)['searches'][0]; + $this->assertSame('title,author.name,embedding', $search['query_by']); + $this->assertSame('title,reference', $search['include_fields']); + $this->assertSame('private.notes,embedding', $search['exclude_fields']); + $this->assertSame('embedding:([], alpha: 0.4)', $search['vector_query']); + } + + #[Test] + public function presets_can_supply_query_by_for_dashboard_managed_relevance() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'search_parameters' => [ + 'query_by' => null, + 'preset' => 'articles_default', + ], + ], + ]); + $http->respond($this->response([], 0)); + + $index->search('red')->withoutData()->get(); + + $search = $http->json(0)['searches'][0]; + $this->assertSame('articles_default', $search['preset']); + $this->assertArrayNotHasKey('query_by', $search); + } + + #[Test] + public function natural_language_search_reuses_generated_parameters_on_later_pages() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'natural_language' => [ + 'model' => 'statamic_articles', + ], + ], + ]); + + $http + ->respond([ + 'results' => [ + [ + 'found' => 12, + 'hits' => [$this->hit('entry::one', 'Red shirt')], + 'parsed_nl_query' => [ + 'generated_params' => [ + 'filter_by' => 'color:=`red` && price:<50', + 'sort_by' => 'price:asc', + ], + 'augmented_params' => [ + 'q' => '*', + ], + ], + ], + ], + ]) + ->respond($this->response([$this->hit('entry::six', 'Another red shirt')], 12)); + + $first = (new Query($index)) + ->query('show me red shirts under 50') + ->naturalLanguage() + ->withoutData(); + $first->paginate(5, page: 1); + + $second = (new Query($index)) + ->query('show me red shirts under 50') + ->naturalLanguage() + ->withoutData(); + $second->paginate(5, page: 2); + + $firstSearch = $http->json(0)['searches'][0]; + $this->assertTrue($firstSearch['nl_query']); + $this->assertSame('statamic_articles', $firstSearch['nl_model_id']); + + $secondSearch = $http->json(1)['searches'][0]; + $this->assertArrayNotHasKey('nl_query', $secondSearch); + $this->assertSame('*', $secondSearch['q']); + $this->assertSame('color:=`red` && price:<50', $secondSearch['filter_by']); + $this->assertSame('price:asc', $secondSearch['sort_by']); + $this->assertSame(5, $secondSearch['offset']); + $this->assertSame([ + 'filter_by' => 'color:=`red` && price:<50', + 'sort_by' => 'price:asc', + 'q' => '*', + ], $second->naturalLanguageParams()); + } + + #[Test] + public function searches_with_different_index_settings_do_not_share_cached_results() + { + [$first, $firstHttp] = $this->makeTypesenseIndex([ + 'settings' => ['search_parameters' => ['filter_by' => 'collection:=blog']], + ]); + [$second, $secondHttp] = $this->makeTypesenseIndex([ + 'settings' => ['search_parameters' => ['filter_by' => 'collection:=news']], + ]); + $firstHttp->respond($this->response([$this->hit('entry::blog', 'Blog')], 1)); + $secondHttp->respond($this->response([$this->hit('entry::news', 'News')], 1)); + + $this->assertSame('Blog', $first->search('red')->withoutData()->get()->first()->get('title')); + $this->assertSame('News', $second->search('red')->withoutData()->get()->first()->get('title')); + } + + #[Test] + public function importing_documents_invalidates_cached_search_results() + { + [$index, $http] = $this->makeTypesenseIndex(); + $http + ->respond($this->response([$this->hit('entry::one', 'Old')], 1)) + ->respond('{"success":true}') + ->respond($this->response([$this->hit('entry::one', 'New')], 1)); + + $this->assertSame('Old', $index->search('red')->withoutData()->get()->first()->get('title')); + $index->insertDocuments(new \Statamic\Search\Documents([ + 'entry::one' => ['title' => 'New'], + ])); + $this->assertSame('New', $index->search('red')->withoutData()->get()->first()->get('title')); + } + + #[Test] + public function replayed_natural_language_sort_does_not_override_configured_sorting() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'search_parameters' => ['sort_by' => 'date:desc'], + 'natural_language' => ['model' => 'articles'], + ], + ]); + $http + ->respond(['results' => [[ + 'found' => 12, + 'hits' => [], + 'parsed_nl_query' => [ + 'generated_params' => ['sort_by' => 'price:asc'], + 'augmented_params' => ['q' => '*', 'sort_by' => 'date:desc'], + ], + ]]]) + ->respond($this->response([], 12)); + + $index->search('cheapest shirts')->naturalLanguage()->withoutData()->paginate(5, page: 1); + $index->search('cheapest shirts')->naturalLanguage()->withoutData()->paginate(5, page: 2); + + $this->assertSame('date:desc', $http->json(0)['searches'][0]['sort_by']); + $this->assertSame('date:desc', $http->json(1)['searches'][0]['sort_by']); + $this->assertArrayNotHasKey('nl_query', $http->json(1)['searches'][0]); + } + + #[Test] + public function replayed_natural_language_preserves_the_effective_sort_from_a_preset() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'settings' => [ + 'search_parameters' => ['preset' => 'latest'], + 'natural_language' => ['model' => 'articles'], + ], + ]); + $http + ->respond(['results' => [[ + 'found' => 12, + 'hits' => [], + 'parsed_nl_query' => [ + 'generated_params' => ['sort_by' => 'price:asc'], + 'augmented_params' => ['q' => '*', 'sort_by' => 'date:desc'], + ], + ]]]) + ->respond($this->response([], 12)); + + $index->search('cheapest shirts')->naturalLanguage()->withoutData()->paginate(5, page: 1); + $index->search('cheapest shirts')->naturalLanguage()->withoutData()->paginate(5, page: 2); + + $this->assertSame('date:desc', $http->json(1)['searches'][0]['sort_by']); + } + + private function response(array $hits, int $found): array + { + return [ + 'results' => [ + compact('found', 'hits'), + ], + ]; + } + + private function hit(string $reference, string $title, int $textMatch = 100): array + { + return [ + 'document' => compact('reference', 'title'), + 'text_match' => $textMatch, + 'highlights' => [], + ]; + } +} diff --git a/tests/Search/TypesenseSchemaTest.php b/tests/Search/TypesenseSchemaTest.php new file mode 100644 index 00000000000..725b4f1cb7e --- /dev/null +++ b/tests/Search/TypesenseSchemaTest.php @@ -0,0 +1,149 @@ +makeTypesenseIndex([ + 'fields' => ['title', 'tags', 'date', 'author->name', 'price', 'categories'], + 'settings' => [ + 'schema' => [ + 'default_sorting_field' => 'date', + 'synonym_sets' => ['blog'], + 'curation_sets' => ['featured'], + 'fields' => [ + ['name' => 'tags', 'type' => 'string[]', 'facet' => true], + ['name' => 'author->name', 'type' => 'string', 'facet' => true], + ['name' => 'popularity', 'type' => 'int32', 'sort' => true], + ], + ], + 'embedding' => [ + 'name' => 'vector', + 'from' => ['title', 'author->name'], + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + 'alpha' => 0.4, + ], + ], + ]); + + $schema = $index->schema()->definition('articles_123'); + $fields = collect($schema['fields'])->keyBy('name'); + + $this->assertSame('articles_123', $schema['name']); + $this->assertTrue($schema['enable_nested_fields']); + $this->assertSame('date', $schema['default_sorting_field']); + $this->assertSame(['blog'], $schema['synonym_sets']); + $this->assertSame(['featured'], $schema['curation_sets']); + $this->assertSame('string', $fields['title']['type']); + $this->assertSame('auto', $fields['price']['type']); + $this->assertSame('auto', $fields['categories']['type']); + $this->assertSame('string[]', $fields['tags']['type']); + $this->assertTrue($fields['tags']['facet']); + $this->assertTrue($fields['author.name']['facet']); + $this->assertSame('int64', $fields['date']['type']); + $this->assertTrue($fields['date']['sort']); + $this->assertFalse($fields['date']['optional']); + $this->assertSame('int32', $fields['popularity']['type']); + $this->assertSame([ + 'from' => ['title', 'author.name'], + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + ], $fields['vector']['embed']); + $this->assertSame('auto', $fields['.*']['type']); + } + + #[Test] + public function it_builds_query_by_from_only_searchable_live_fields() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'fields' => ['title', 'summary', 'price', 'tags', 'author->name'], + 'settings' => [ + 'search_parameters' => ['query_by' => null], + 'embedding' => [ + 'from' => ['title'], + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + ], + ], + ]); + + $http->respond([ + 'fields' => [ + ['name' => 'title', 'type' => 'string'], + ['name' => 'summary', 'type' => 'string', 'index' => false], + ['name' => 'price', 'type' => 'float'], + ['name' => 'tags', 'type' => 'string[]'], + ['name' => 'author.name', 'type' => 'string'], + ['name' => 'embedding', 'type' => 'float[]'], + ], + ]); + + $this->assertSame('title,tags,author.name,embedding', $index->schema()->queryBy()); + $this->assertSame('title,tags,author.name,embedding', $index->schema()->queryBy()); + $this->assertCount(1, $http->requests); + } + + #[Test] + public function it_uses_the_internal_reference_when_no_configured_fields_are_searchable() + { + [$index, $http] = $this->makeTypesenseIndex([ + 'fields' => [], + 'settings' => [ + 'search_parameters' => ['query_by' => null], + ], + ]); + $http->respond([ + 'fields' => [ + ['name' => 'reference', 'type' => 'string'], + ], + ]); + + $this->assertSame('reference', $index->schema()->queryBy()); + } + + #[Test] + public function it_normalizes_dates_collections_handles_and_stringables() + { + $handle = new class + { + public function handle() + { + return 'pages'; + } + }; + + $stringable = new class implements \Stringable + { + public function __toString(): string + { + return 'string value'; + } + }; + + $normalizer = new Normalizer; + + $this->assertSame([ + 'date' => 1761213600, + 'items' => ['one', 'two'], + 'author.name' => 'Jack', + 'handle' => 'pages', + 'stringable' => 'string value', + ], $normalizer->document([ + 'date' => Carbon::parse('2025-10-23 10:00:00', 'UTC'), + 'items' => collect(['one', 'two']), + 'author->name' => 'Jack', + 'handle' => $handle, + 'stringable' => $stringable, + 'unsupported' => new \stdClass, + ])); + } +} diff --git a/tests/Tags/SearchTest.php b/tests/Tags/SearchTest.php index 15c316bb41b..6def4d24e2b 100644 --- a/tests/Tags/SearchTest.php +++ b/tests/Tags/SearchTest.php @@ -7,6 +7,7 @@ use Statamic\Facades\Parse; use Statamic\Facades\Search; use Statamic\Search\QueryBuilder; +use Statamic\Search\Typesense\Index as TypesenseIndex; use Tests\PreventSavingStacheItemsToDisk; use Tests\TestCase; @@ -26,8 +27,9 @@ public function it_outputs_results() $entryB = EntryFactory::id('b')->collection('test')->data(['title' => 'entry b'])->create(); $builder = $this->mock(QueryBuilder::class); - $builder->shouldReceive('ensureExists', 'search', 'withData', 'limit', 'offset', 'where')->andReturnSelf(); + $builder->shouldReceive('ensureExists', 'search', 'withData', 'naturalLanguage', 'limit', 'offset', 'where')->andReturnSelf(); $builder->shouldReceive('get')->andReturn(collect([$entryA, $entryB])); + $builder->shouldReceive('naturalLanguageParams')->andReturn([]); Search::shouldReceive('index')->with(null)->once()->andReturn($builder); @@ -48,8 +50,9 @@ public function it_outputs_results_using_alias() $entryB = EntryFactory::id('b')->collection('test')->data(['title' => 'entry b'])->create(); $builder = $this->mock(QueryBuilder::class); - $builder->shouldReceive('ensureExists', 'search', 'withData', 'limit', 'offset', 'where')->andReturnSelf(); + $builder->shouldReceive('ensureExists', 'search', 'withData', 'naturalLanguage', 'limit', 'offset', 'where')->andReturnSelf(); $builder->shouldReceive('get')->andReturn(collect([$entryA, $entryB])); + $builder->shouldReceive('naturalLanguageParams')->andReturn([]); Search::shouldReceive('index')->with(null)->once()->andReturn($builder); @@ -62,4 +65,50 @@ public function it_outputs_results_using_alias() ) ); } + + #[Test] + public function it_enables_natural_language_search_and_exposes_generated_parameters() + { + $entry = EntryFactory::id('a')->collection('test')->data(['title' => 'entry a'])->create(); + + $builder = $this->mock(QueryBuilder::class); + $builder->shouldReceive('ensureExists', 'search', 'withData', 'limit', 'offset', 'where')->andReturnSelf(); + $builder->shouldReceive('naturalLanguage')->with(true)->once()->andReturnSelf(); + $builder->shouldReceive('get')->andReturn(collect([$entry])); + $builder->shouldReceive('naturalLanguageParams')->andReturn([ + 'q' => 'shirts', + 'filter_by' => 'color:=red', + ]); + + Search::shouldReceive('index')->with(null)->once()->andReturn($builder); + + $this->assertEquals( + 'shirts', + $this->tag( + '{{ search:results for="show me red shirts" natural_language="true" as="stuff" }}'. + '{{ natural_language:q }}{{ stuff }}<{{ title }}>{{ /stuff }}'. + '{{ /search:results }}' + ) + ); + } + + #[Test] + public function it_outputs_a_typesense_scoped_search_key() + { + $index = $this->mock(TypesenseIndex::class); + $index + ->shouldReceive('scopedSearchKey') + ->with(['filter_by' => 'type:=entry'], 60) + ->once() + ->andReturn('scoped-key'); + + Search::shouldReceive('index')->with('blog')->once()->andReturn($index); + + $this->assertEquals( + 'scoped-key', + $this->tag( + '{{ search:scoped_key index="blog" expires="60" filter_by="type:=entry" }}' + ) + ); + } }