From b8a1460a30ff7172ccb6c3dfad928d23165ba2ae Mon Sep 17 00:00:00 2001 From: Dave Clarke Date: Thu, 2 Jul 2026 16:03:08 +0100 Subject: [PATCH] Typesense v1 --- plugins/Typesense/v1/configValidation.json | 11 ++ .../v1/dataStreams/documentSearch.json | 57 +++++++ .../Typesense/v1/dataStreams/multiSearch.json | 32 ++++ .../v1/dataStreams/scripts/documentSearch.js | 5 + .../v1/dataStreams/scripts/multiSearch.js | 6 + .../v1/dataStreams/scripts/searchSummary.js | 9 ++ .../v1/dataStreams/searchSummary.json | 44 ++++++ .../v1/dataStreams/searchValidation.json | 20 +++ .../Typesense/v1/defaultContent/manifest.json | 3 + .../overviewDashboard.dash.json | 139 ++++++++++++++++++ .../Typesense/v1/defaultContent/scopes.json | 1 + plugins/Typesense/v1/docs/README.md | 61 ++++++++ plugins/Typesense/v1/icon.svg | 132 +++++++++++++++++ plugins/Typesense/v1/metadata.json | 36 +++++ plugins/Typesense/v1/ui.json | 36 +++++ 15 files changed, 592 insertions(+) create mode 100644 plugins/Typesense/v1/configValidation.json create mode 100644 plugins/Typesense/v1/dataStreams/documentSearch.json create mode 100644 plugins/Typesense/v1/dataStreams/multiSearch.json create mode 100644 plugins/Typesense/v1/dataStreams/scripts/documentSearch.js create mode 100644 plugins/Typesense/v1/dataStreams/scripts/multiSearch.js create mode 100644 plugins/Typesense/v1/dataStreams/scripts/searchSummary.js create mode 100644 plugins/Typesense/v1/dataStreams/searchSummary.json create mode 100644 plugins/Typesense/v1/dataStreams/searchValidation.json create mode 100644 plugins/Typesense/v1/defaultContent/manifest.json create mode 100644 plugins/Typesense/v1/defaultContent/overviewDashboard.dash.json create mode 100644 plugins/Typesense/v1/defaultContent/scopes.json create mode 100644 plugins/Typesense/v1/docs/README.md create mode 100644 plugins/Typesense/v1/icon.svg create mode 100644 plugins/Typesense/v1/metadata.json create mode 100644 plugins/Typesense/v1/ui.json diff --git a/plugins/Typesense/v1/configValidation.json b/plugins/Typesense/v1/configValidation.json new file mode 100644 index 00000000..d2b2e4cf --- /dev/null +++ b/plugins/Typesense/v1/configValidation.json @@ -0,0 +1,11 @@ +{ + "steps": [ + { + "displayName": "Authenticate and check collection", + "dataStream": { "name": "searchValidation" }, + "required": true, + "error": "Could not query the collection. Check the Host URL, that the Search API Key is valid, and that it is authorised for the configured Collection.", + "success": "Connected successfully." + } + ] +} diff --git a/plugins/Typesense/v1/dataStreams/documentSearch.json b/plugins/Typesense/v1/dataStreams/documentSearch.json new file mode 100644 index 00000000..248cb434 --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/documentSearch.json @@ -0,0 +1,57 @@ +{ + "name": "documentSearch", + "displayName": "Document Search", + "description": "Documents from the configured collection matching a search query", + "tags": ["Search"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "collections/{{dataSource.collection}}/documents/search", + "getArgs": [ + { "key": "q", "value": "{{q || '*'}}" }, + { "key": "query_by", "value": "{{query_by || null}}" }, + { "key": "filter_by", "value": "{{filter_by || null}}" }, + { "key": "sort_by", "value": "{{sort_by || null}}" }, + { "key": "per_page", "value": "{{per_page || 50}}" } + ], + "postRequestScript": "documentSearch.js" + }, + "matches": "none", + "ui": [ + { + "type": "text", + "name": "q", + "label": "Query (q)", + "defaultValue": "*" + }, + { + "type": "text", + "name": "query_by", + "label": "Query by fields", + "help": "Comma-separated fields to search; required unless q is *" + }, + { + "type": "text", + "name": "filter_by", + "label": "Filter by" + }, + { + "type": "text", + "name": "sort_by", + "label": "Sort by" + }, + { + "type": "number", + "name": "per_page", + "label": "Max results", + "defaultValue": 50, + "help": "Typesense max is 250" + } + ], + "manualConfigApply": true, + "metadata": [ + { "name": "_relevance", "displayName": "Relevance", "shape": "number" }, + { "pattern": ".*" } + ], + "timeframes": false +} diff --git a/plugins/Typesense/v1/dataStreams/multiSearch.json b/plugins/Typesense/v1/dataStreams/multiSearch.json new file mode 100644 index 00000000..5c770872 --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/multiSearch.json @@ -0,0 +1,32 @@ +{ + "name": "multiSearch", + "displayName": "Multi Search", + "description": "Documents from a raw Typesense multi_search request, flattened across all sub-searches", + "tags": ["Search"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "multi_search", + "getArgs": [{ "key": "collection", "value": "{{dataSource.collection}}" }], + "postBody": "{{query}}", + "postRequestScript": "multiSearch.js" + }, + "matches": "none", + "ui": [ + { + "type": "code", + "name": "query", + "label": "Query (JSON)", + "language": "json", + "defaultValue": { "searches": [{ "q": "*" }] }, + "help": "A Typesense multi_search body. Each search may set its own collection; otherwise the connection's configured collection is used." + } + ], + "manualConfigApply": true, + "metadata": [ + { "name": "_search", "displayName": "Search #", "shape": "number" }, + { "name": "_relevance", "displayName": "Relevance", "shape": "number" }, + { "pattern": ".*" } + ], + "timeframes": false +} diff --git a/plugins/Typesense/v1/dataStreams/scripts/documentSearch.js b/plugins/Typesense/v1/dataStreams/scripts/documentSearch.js new file mode 100644 index 00000000..6ad932da --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/scripts/documentSearch.js @@ -0,0 +1,5 @@ +// dataStreams/scripts/documentSearch.js +// Typesense search response: { found, out_of, hits: [ { document: {...}, text_match, highlight, highlights } ] } +// Documents have an arbitrary, unknown schema per collection, so flatten each hit's +// document fields to top-level columns and add the relevance score. +result = (data.hits || []).map((h) => ({ ...h.document, _relevance: h.text_match })); diff --git a/plugins/Typesense/v1/dataStreams/scripts/multiSearch.js b/plugins/Typesense/v1/dataStreams/scripts/multiSearch.js new file mode 100644 index 00000000..f19c4640 --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/scripts/multiSearch.js @@ -0,0 +1,6 @@ +// dataStreams/scripts/multiSearch.js +// Typesense multi_search response: { results: [ { found, hits: [ { document:{...}, text_match } ] } ] } +// Flatten hits across all sub-searches, tagging which sub-search each row came from; +// our meta fields must win over document fields. +result = (data.results || []).flatMap((r, i) => + (r.hits || []).map((h) => ({ ...h.document, _search: i, _relevance: h.text_match }))); diff --git a/plugins/Typesense/v1/dataStreams/scripts/searchSummary.js b/plugins/Typesense/v1/dataStreams/scripts/searchSummary.js new file mode 100644 index 00000000..ee3a9926 --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/scripts/searchSummary.js @@ -0,0 +1,9 @@ +// dataStreams/scripts/searchSummary.js +// Typesense search response root is an OBJECT: { found, out_of, search_time_ms, page, hits: [...] } +// Build a single summary row from the top-level counts, ignoring hits. +result = [{ + found: data.found, + out_of: data.out_of, + search_time_ms: data.search_time_ms, + page: data.page +}]; diff --git a/plugins/Typesense/v1/dataStreams/searchSummary.json b/plugins/Typesense/v1/dataStreams/searchSummary.json new file mode 100644 index 00000000..c8b48457 --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/searchSummary.json @@ -0,0 +1,44 @@ +{ + "name": "searchSummary", + "displayName": "Search Summary", + "description": "Match count, total document count and search time for a search query", + "tags": ["Search"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "collections/{{dataSource.collection}}/documents/search", + "getArgs": [ + { "key": "q", "value": "{{q || '*'}}" }, + { "key": "query_by", "value": "{{query_by || null}}" }, + { "key": "filter_by", "value": "{{filter_by || null}}" }, + { "key": "per_page", "value": "0" } + ], + "postRequestScript": "searchSummary.js" + }, + "matches": "none", + "ui": [ + { + "type": "text", + "name": "q", + "label": "Query (q)", + "defaultValue": "*" + }, + { + "type": "text", + "name": "query_by", + "label": "Query by fields" + }, + { + "type": "text", + "name": "filter_by", + "label": "Filter by" + } + ], + "metadata": [ + { "name": "found", "displayName": "Matching Documents", "shape": "number", "role": "value" }, + { "name": "out_of", "displayName": "Total Documents", "shape": "number" }, + { "name": "search_time_ms", "displayName": "Search Time", "shape": "milliseconds" }, + { "name": "page", "displayName": "Page", "shape": "number", "visible": false } + ], + "timeframes": false +} diff --git a/plugins/Typesense/v1/dataStreams/searchValidation.json b/plugins/Typesense/v1/dataStreams/searchValidation.json new file mode 100644 index 00000000..93daa48d --- /dev/null +++ b/plugins/Typesense/v1/dataStreams/searchValidation.json @@ -0,0 +1,20 @@ +{ + "name": "searchValidation", + "displayName": "Search Validation", + "description": "Up to one document from the configured collection", + "tags": ["Search"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "collections/{{dataSource.collection}}/documents/search", + "getArgs": [ + { "key": "q", "value": "*" }, + { "key": "per_page", "value": "1" } + ], + "pathToData": "hits" + }, + "matches": "none", + "metadata": [{ "pattern": ".*" }], + "timeframes": false, + "visibility": { "type": "hidden" } +} diff --git a/plugins/Typesense/v1/defaultContent/manifest.json b/plugins/Typesense/v1/defaultContent/manifest.json new file mode 100644 index 00000000..0807f964 --- /dev/null +++ b/plugins/Typesense/v1/defaultContent/manifest.json @@ -0,0 +1,3 @@ +{ + "items": [{ "name": "overviewDashboard", "type": "dashboard" }] +} diff --git a/plugins/Typesense/v1/defaultContent/overviewDashboard.dash.json b/plugins/Typesense/v1/defaultContent/overviewDashboard.dash.json new file mode 100644 index 00000000..55a83f40 --- /dev/null +++ b/plugins/Typesense/v1/defaultContent/overviewDashboard.dash.json @@ -0,0 +1,139 @@ +{ + "name": "Overview", + "schemaVersion": "1.5", + "timeframe": "last24hours", + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "1e102028-7d08-4220-b038-9acab1d00e7a", + "x": 0, + "y": 0, + "w": 4, + "h": 1, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/text", + "title": "", + "description": "", + "visualisation": { + "config": { + "content": "Showing results from the configured Typesense collection.", + "fontSize": 14, + "align": "left", + "autoSize": true + } + } + } + }, + { + "i": "cc0e7c1c-a05f-47b8-8838-0597691526c3", + "x": 0, + "y": 1, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Total Documents", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.[searchSummary]}}", + "name": "searchSummary", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "q": "*" + } + }, + "timeframe": "none", + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "found", + "comparisonColumn": "none", + "label": "Matching Documents" + } + } + } + } + }, + { + "i": "693f6164-c6e5-4906-b77b-4c1da5222915", + "x": 2, + "y": 1, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Search Time", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.[searchSummary]}}", + "name": "searchSummary", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "q": "*" + } + }, + "timeframe": "none", + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "search_time_ms", + "comparisonColumn": "none", + "label": "Search Time" + } + } + } + } + }, + { + "i": "8b4d3946-efa0-4180-bc02-5a20ed445cef", + "x": 0, + "y": 3, + "w": 4, + "h": 6, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Documents", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.[documentSearch]}}", + "name": "documentSearch", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "q": "*", + "per_page": 25 + } + }, + "timeframe": "none", + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false + } + } + } + } + } + ] + } +} diff --git a/plugins/Typesense/v1/defaultContent/scopes.json b/plugins/Typesense/v1/defaultContent/scopes.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/plugins/Typesense/v1/defaultContent/scopes.json @@ -0,0 +1 @@ +[] diff --git a/plugins/Typesense/v1/docs/README.md b/plugins/Typesense/v1/docs/README.md new file mode 100644 index 00000000..2a492fed --- /dev/null +++ b/plugins/Typesense/v1/docs/README.md @@ -0,0 +1,61 @@ +# Typesense + +Query documents from your [Typesense](https://typesense.org) collections directly in SquaredUp. This plugin runs searches against a Typesense Cloud (or self-hosted) instance and returns matching documents as rows you can chart, table, and build dashboards from. + +It is designed to work with a **search-only API key**, so it never needs admin access to your cluster. + +## What this plugin does + +- Runs structured searches against a single collection and returns one row per matching document. +- Runs raw `multi_search` queries from a JSON blob for federated / multi-collection / advanced queries. +- Returns search result counts (matching documents, total documents scanned, timing). + +Because a search-only key cannot list collections (`collections:list` requires an admin key), this plugin does **not** import your collections or documents as objects into the SquaredUp graph. You choose the collection to query in the plugin configuration. + +## Prerequisites — getting a search API key + +1. Log in to the [Typesense Cloud dashboard](https://cloud.typesense.org) (or your self-hosted admin tooling). +2. Note your cluster's **nodes hostname** — it looks like `xxxxxxxxx.a1.typesense.net`. Your base URL is `https://`. +3. Create a **search-only API key**: + - Using an admin key, call the [Create API Key](https://typesense.org/docs/latest/api/api-keys.html) endpoint with the `documents:search` action, scoped to the collection(s) you want to expose. Example: + ``` + curl "https:///keys" \ + -X POST \ + -H "X-TYPESENSE-API-KEY: " \ + -H "Content-Type: application/json" \ + -d '{"description":"SquaredUp search key","actions":["documents:search"],"collections":[""]}' + ``` + - Copy the returned `value` — this is your search-only key. It is shown only once. +4. Confirm the key works: + ``` + curl "https:///collections//documents/search?q=*&per_page=1" \ + -H "X-TYPESENSE-API-KEY: " + ``` + You should get a JSON response containing a `found` count. + +## Configuration fields + +| Field | Required | What it is / where to find it | +|-------|----------|-------------------------------| +| **Host URL** | Yes | Your Typesense base URL including scheme, e.g. `https://xxxxxxxxx.a1.typesense.net`. No trailing slash. From the Typesense Cloud dashboard (nodes hostname) or your self-hosted address. | +| **Search API key** | Yes | A search-only API key (`documents:search` action). See "getting a search API key" above. Stored securely. | +| **Collection** | Yes | The name of the collection this connection queries, e.g. `products`. The key must be authorised for this collection. Add another connection for another collection. | + +When you save the configuration, the plugin runs a lightweight `q=*` search against the collection to confirm the host, key, and collection are all valid. + +## What gets indexed + +Nothing. This is a query-only plugin — it imports no objects into the SquaredUp graph. Data is retrieved live each time a tile runs. + +## Data streams + +- **Document Search** — structured search against the configured collection. Tile parameters: `q` (query, default `*`), `query_by`, `filter_by`, `sort_by`, `per_page`. Returns one row per matching document, with the document's own fields as columns plus a relevance score. +- **Multi Search** — runs a raw `multi_search` JSON blob (e.g. `{"searches":[{"q":"*"}]}`) for federated or advanced queries. Returns one row per hit across all sub-searches. +- **Search Summary** — returns a single row with the number of matching documents, total documents scanned, and search time. + +## Known limitations + +- **Search-only scope.** No cluster health, metrics, API-key or alias management, and no collection/document import — these require an admin key. +- **Result size.** A single search returns at most 250 documents per page (`per_page`). Search is designed to return the most relevant results, not to bulk-export a collection. +- **Response size.** Very large result sets can exceed SquaredUp's ~6MB per-request limit — keep `per_page` and returned field counts reasonable, or use `filter_by` to narrow results. +- **No time range.** Typesense search has no built-in time-range parameter, so these streams return current results with no timeframe picker. To restrict by time, add a `filter_by` on a timestamp field in your documents. diff --git a/plugins/Typesense/v1/icon.svg b/plugins/Typesense/v1/icon.svg new file mode 100644 index 00000000..3c1fcf1a --- /dev/null +++ b/plugins/Typesense/v1/icon.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/Typesense/v1/metadata.json b/plugins/Typesense/v1/metadata.json new file mode 100644 index 00000000..78306c87 --- /dev/null +++ b/plugins/Typesense/v1/metadata.json @@ -0,0 +1,36 @@ +{ + "name": "typesense", + "displayName": "Typesense", + "version": "1.0.0", + "author": { "name": "@clarkd", "type": "community" }, + "description": "Query documents from your Typesense collections using a search-only API key.", + "category": "Database", + "type": "hybrid", + "schemaVersion": "2.1", + "importNotSupported": true, + "restrictedToPlatforms": [], + "keywords": ["typesense", "search", "documents", "query"], + "objectTypes": [], + "links": [ + { + "category": "documentation", + "url": "https://github.com/squaredup/plugins/blob/main/plugins/Typesense/v1/docs/README.md", + "label": "Help adding this plugin" + }, + { + "category": "source", + "url": "https://github.com/squaredup/plugins/tree/main/plugins/Typesense", + "label": "Repository" + } + ], + "base": { + "plugin": "WebAPI", + "majorVersion": "1", + "config": { + "baseUrl": "{{host}}", + "authMode": "none", + "headers": [{ "key": "X-TYPESENSE-API-KEY", "value": "{{apiKey}}" }], + "queryArgs": [] + } + } +} diff --git a/plugins/Typesense/v1/ui.json b/plugins/Typesense/v1/ui.json new file mode 100644 index 00000000..166f4a16 --- /dev/null +++ b/plugins/Typesense/v1/ui.json @@ -0,0 +1,36 @@ +[ + { + "type": "markdown", + "name": "info", + "content": "Connect to a Typesense instance using a **search-only API key** (`documents:search` action). See the plugin documentation for how to create one." + }, + { + "type": "url", + "name": "host", + "label": "Host URL", + "placeholder": "https://xxxxxxxxx.a1.typesense.net", + "help": "Your Typesense base URL including `https://` and **no trailing slash**. For Typesense Cloud this is `https://`.", + "validation": { + "required": true, + "pattern": { + "value": "^https?://[^\\s/]+$", + "message": "Enter the base URL including https:// and no trailing slash or path" + } + } + }, + { + "type": "password", + "name": "apiKey", + "label": "Search API Key", + "help": "A search-only API key. This is stored securely and sent in the `X-TYPESENSE-API-KEY` header.", + "validation": { "required": true } + }, + { + "type": "text", + "name": "collection", + "label": "Collection", + "placeholder": "products", + "help": "The collection this connection queries. The API key must be authorised for it. Add another connection to query another collection.", + "validation": { "required": true } + } +]