From 898db00f2fcd6f99157d3aaf937ed5bb3f69cdc0 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 2 Jul 2026 15:32:48 -0400 Subject: [PATCH 01/12] add some sequence diagrams to ARCHITECTURE.md --- ARCHITECTURE.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f27608ffc..8163f80ba 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -9,7 +9,8 @@ In short, SHARE/trove holds metadata records that describe things and makes thos ## Parts -a look at the tangles of communication between different parts of the system: +a slightly simplified look at the tangles of communication between different parts of the system, +as currently implemented: ```mermaid graph LR; @@ -48,6 +49,49 @@ graph LR; subscribers-->oaipmh; ``` +### /trove/ingest +```mermaid +sequenceDiagram + participant ms as metadata source + box shtrove + participant ss as web server + participant sd as db (postgres) + participant sw as worker (celery) + participant sq as queues (rabbitmq) + participant si as indexer + participant se as elasticsearch + end + ms ->> ss: POST /trove/ingest + ss ->> sd: save ResourceIdentifier(s) + ss ->> sd: save Indexcard + ss ->> sd: save ResourceDescription(s) + ss ->> sq: enqueue derive task + ss ->> ms: 201 CREATED (success!) + sq -->> sw: receive derive task + sd <<-->> sw: load metadata record + sw ->> sd: save DerivedIndexcards + sw ->> sq: enqueue indexer message + sq -->> si: bulk receive indexer messages + sd <<-->> si: bulk load metadata + si ->> se: bulk index +``` + +### /trove/index-card-search +```mermaid +sequenceDiagram + participant c as client + box shtrove + participant ss as web server + participant sd as db (postgres) + participant se as elasticsearch + end + c ->> ss: GET /trove/index-card-search + ss ->> se: query (via index strategy) + se ->> ss: result ids (plus context) + ss <<-->> sd: load metadata records + ss ->> c: respond/stream search results (formatted as requested) +``` + ## Code map A brief look at important areas of code as they happen to exist now. From 7336bdd3f7a6542966ababa1405203da54a702e4 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 2 Jul 2026 15:33:14 -0400 Subject: [PATCH 02/12] fill some gaps in trove search api docs --- trove/openapi.py | 2 +- trove/vocab/trove.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/trove/openapi.py b/trove/openapi.py index 89c0bee67..43af9ac1c 100644 --- a/trove/openapi.py +++ b/trove/openapi.py @@ -168,7 +168,7 @@ def _openapi_path(path_iri: str, api_graph: primitive_rdf.RdfGraph) -> Tuple[str def _concept_markdown_blocks(concept_iri: str, api_graph: primitive_rdf.RdfGraph) -> Generator[str, None, None]: for _label in api_graph.q(concept_iri, RDFS.label): - yield f'## {_label.unicode_value}' + yield f'## concept: {_label.unicode_value}' for _comment in api_graph.q(concept_iri, RDFS.comment): yield f'' for _desc in api_graph.q(concept_iri, DCTERMS.description): diff --git a/trove/vocab/trove.py b/trove/vocab/trove.py index 5649db6b8..a402563d4 100644 --- a/trove/vocab/trove.py +++ b/trove/vocab/trove.py @@ -502,6 +502,24 @@ def _literal_markdown(text: str, *, language: str) -> literal: the response will have the http header `Content-Disposition: attachment` with a filename based on the query param value, current date, and response content mediatype +''', language='en')}, + }, + TROVE.iriShorthand: { + RDF.type: {RDF.Property, TROVE.QueryParameter}, + JSONAPI_MEMBERNAME: {literal('iriShorthand', language='en')}, + RDFS.label: {literal('iriShorthand', language='en')}, + RDFS.comment: {literal('define a shorthand namespace or alias for IRIs in this query string', language='en')}, + TROVE.jsonSchema: {literal_json({'type': 'string'})}, + DCTERMS.description: {_literal_markdown('''**iriShorthand** is +a query parameter to define a shorthand name used for parsing IRIs in other query parameters + +for example, a request to `/trove/index-card-search` with these query parameters: +- `iriShorthand[blarg]=https://blarg.example/vocab/` +- `iriShorthand[foo]=https://another.example/vocab/foo` +- `cardSearchFilter[blarg:prop]=foo` + +will find cards with the IRI value `` +at the property `` ''', language='en')}, }, TROVE.cardSearchText: { @@ -709,11 +727,15 @@ def _literal_markdown(text: str, *, language: str) -> literal: DCTERMS.description: {_literal_markdown(f'''a **property-path** is a dot-separated path of short-hand IRIs, used in several api parameters -currently the only supported shorthand is defined by [OSFMAP]({osfmap.OSFMAP_LINK}) - for example, `creator.name` is parsed as a two-step path that follows `creator` (aka `dcterms:creator`, ``) and then `name` (aka `foaf:name`, ``) +currently, the only implied shorthand is that defined by [OSFMAP]({osfmap.OSFMAP_LINK}) +-- to search on other properties, use an `iriShorthand` query param to provide an explicit +alias or namespace (e.g. with `iriShorthand[blarg]=https://blarg.example/vocab/`, +`blarg:prop1.blarg:prop2` in another param will be parsed as a two-step property-path +following `` then ``) + most places that allow one property-path also accept a comma-separated set of paths, like `title,description` (which is parsed as two paths: `title` and `description`) or `affiliation,creator.affiliation,funder` (which is parsed as three paths: `affiliation`, From 4a823e161ca82b62aadd932458b463a7b23dfe88 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 2 Jul 2026 15:59:12 -0400 Subject: [PATCH 03/12] clarify --- ARCHITECTURE.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8163f80ba..60463d703 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -50,6 +50,7 @@ graph LR; ``` ### /trove/ingest +a slightly simplified look at how metadata records are ingested, as currently implemented: ```mermaid sequenceDiagram participant ms as metadata source @@ -68,15 +69,16 @@ sequenceDiagram ss ->> sq: enqueue derive task ss ->> ms: 201 CREATED (success!) sq -->> sw: receive derive task - sd <<-->> sw: load metadata record + sd <<-->> sw: load ResourceDescription(s) sw ->> sd: save DerivedIndexcards sw ->> sq: enqueue indexer message - sq -->> si: bulk receive indexer messages - sd <<-->> si: bulk load metadata + sq -->> si: bulk receive messages + sd <<-->> si: bulk load metadata records si ->> se: bulk index ``` ### /trove/index-card-search +a slightly simplified look at how search requests are served, as currently implemented: ```mermaid sequenceDiagram participant c as client @@ -86,10 +88,9 @@ sequenceDiagram participant se as elasticsearch end c ->> ss: GET /trove/index-card-search - ss ->> se: query (via index strategy) - se ->> ss: result ids (plus context) + ss <<-->> se: query for result ids (and context) ss <<-->> sd: load metadata records - ss ->> c: respond/stream search results (formatted as requested) + ss ->> c: respond/stream search results ``` ## Code map @@ -135,9 +136,7 @@ Uses the [resource description framework](https://www.w3.org/TR/rdf11-primer/#se ### Identifiers -Whenever feasible, use full URI strings to identify resources, concepts, types, and properties that may be exposed outwardly. - -Prefer using open, standard, well-defined namespaces wherever possible ([DCAT](https://www.w3.org/TR/vocab-dcat-3/) is a good place to start; see `trove.vocab.namespaces` for others already in use). When app-specific concepts must be defined, use the `TROVE` namespace (`https://share.osf.io/vocab/2023/trove/`). +Whenever feasible, use full [IRI](https://www.rfc-editor.org/rfc/rfc3987.html) strings (utf-8) to identify resources, concepts, types, and properties that may be exposed outwardly (without converting to URI or using to send requests). Prefer using open, standard, well-defined namespaces wherever possible ([DCAT](https://www.w3.org/TR/vocab-dcat-3/) is a good place to start; see `trove.vocab.namespaces` for others already in use). When app-specific concepts must be defined, use the `TROVE` namespace (`https://share.osf.io/vocab/2023/trove/`). A notable exception (non-URI identifier) is the "source-unique identifier" or "suid" -- essentially a two-tuple `(source, identifier)` that uniquely and persistently identifies a metadata record in a source repository. This `identifier` may be any string value, provided by the external source. From 0d85c1fd0817f5aa8625d3506ae017a0909b2577 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Jul 2026 13:57:38 -0400 Subject: [PATCH 04/12] docs: admin site, index strategies --- README.md | 6 +- how-to/README.md | 11 +++ how-to/relate-to-osf.md | 130 +++++++++++++++++++++++++++++ how-to/use-the-admin-site.md | 94 +++++++++++++++++++++ how-to/use-the-api.md | 2 +- how-to/wrangle-index-strategies.md | 35 ++++++++ 6 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 how-to/README.md create mode 100644 how-to/relate-to-osf.md create mode 100644 how-to/use-the-admin-site.md create mode 100644 how-to/wrangle-index-strategies.md diff --git a/README.md b/README.md index 201adfc2b..e94bdaac5 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,8 @@ SHARE/trove (aka SHARtrove, shtrove) is is a service meant to store (meta)data y note: this codebase is currently (and historically) rather entangled with [osf.io](https://osf.io), which has its shtrove at https://share.osf.io -- stay tuned for more-reusable open-source libraries and tools for working with (meta)data +see [how-to docs](./how-to/) for help setting up and using various things + see [ARCHITECTURE.md](./ARCHITECTURE.md) for help navigating this codebase see [CONTRIBUTING.md](./CONTRIBUTING.md) for info about contributing changes - -see [how-to/use-the-api.md](./how-to/use-the-api.md) for help using the api to add and access (meta)data - -see [how-to/run-locally.md](./how-to/run-locally.md) for help running a shtrove instance for local development diff --git a/how-to/README.md b/how-to/README.md new file mode 100644 index 000000000..0f77a6f3c --- /dev/null +++ b/how-to/README.md @@ -0,0 +1,11 @@ +# shtrove how-to + +see [how-to/use-the-api.md](./use-the-api.md) for help using the api to add and access (meta)data + +see [how-to/run-locally.md](./run-locally.md) for help running a shtrove instance for local development + +see [how-to/use-the-admin-site.md](./use-the-admin-site.md) for help using the included admin site to manage indexing and ingested data + +see [how-to/wrangle-index-strategies.md](./wrangle-index-strategies.md) for help updating + +see [how-to/relate-to-osf.md](./relate-to-osf.md) for help understanding how this relates to OSF at the moment diff --git a/how-to/relate-to-osf.md b/how-to/relate-to-osf.md new file mode 100644 index 000000000..9162893ba --- /dev/null +++ b/how-to/relate-to-osf.md @@ -0,0 +1,130 @@ +# how this relates to osf.io + +how OSF metadata (see [osf.io repo](https://github.com/CenterForOpenScience/osf.io)) gets indexed for search (currently, as of 2026-07): + +1. gather an OSFMAP metadata record (as [linked open data](https://en.wikipedia.org/wiki/Linked_open_data)) +2. send that record to shtrove for ingestion +3. derive multiple representations/serializations +4. index by multiple indexing strategies in parallel + +## 1. gather metadata +in `osf.metadata.osf_gathering`, each metadata property defined in OSF's metadata application +profile ([OSFMAP](https://osf.io/8yczr)) has a "gatherer" function that gets data from OSF's +database for the given item and yields values as portable/interoperable RDF linked data + +(note: this is the same way that an OSF item's metadata is gathered when a user downloads +a metadata record or when sending an update to e.g. Datacite -- the RDF graph is an abstract +data structure than can be serialized/rendered in various ways) + +after an update occurs to an item on OSF that is public and meant to be discoverable, +background tasks (see OSF's `api.share.utils`) gather all available metadata and send +it to SHARE/trove + +## 2. send to shtrove +metadata records are sent in [turtle](https://en.wikipedia.org/wiki/Turtle_RDF) format with +a POST request to shtrove's `/trove/ingest` api -- for details of query params, see +[how-to docs](https://github.com/CenterForOpenScience/SHARE/blob/develop/how-to/use-the-api.md#posting-index-cards) + +in OSF, all descriptive metadata is sent in a "main" record, while administrative +metadata (specific to OSF usage, rather than describing the item itself) and, +in the near future, custom CEDAR metadata records are gathered and sent in separate +"supplementary" records -- all these records are connected by the same `focus_iri` +(identifying the item in OSF) and the knowledge graphs they contain must each be a tree +rooted at that focus iri + +requests are authenticated by a pre-created access token -- for historical reasons, +each OSF "preprint provider" and "registration provider" has a separate access token, +and OSF has an additional static token that is used both for creating provider tokens and +for indexing items that don't belong to a provider (e.g. projects) + +based on request headers and query params, shtrove ensures in the database: +* `share.Source`/`share.SourceConfig` for the authenticated user, with... +* `share.SourceUniqueIdentifier` for given `record_identifier`, with... +* `trove.ResourceIdentifier` for given `focus_iri` + +(see `trove.digestive_tract.sniff`) + +then shtrove parses the request body, and may create (or update) in the database: + +* `trove.ResourceIdentifier` (for each described resource and its types) +* `trove.Indexcard` (with identifiers and type-identifiers for the focus resource) +* "resource descriptions" in turtle format: + * if non-supplementary, `trove.LatestResourceDescription` and `trove.ArchivedResourceDescription` + * if supplementary, only `trove.SupplementaryResourceDescription` + +(see `trove.digestive_tract.extract`) + +shtrove then enqueues a background `task__derive` and responds to OSF with `201 CREATED` success + +(note: from this point on, all background tasks are in SHARE's own queues and workers, +unaffected by OSF's chronic celery-task logjams) + +## 3. derive representations +after an update has been received for an `Indexcard`, `trove.digestive_tract.task__derive` +combines that card's `LatestResourceDescription` and each current `SupplementaryResourceDescription` +and creates a `DerivedIndexcard` for each deriver in `trove.derive.DEFAULT_DERIVER_SET` + +current derivers include: + +* "osfmap_json_mini": a [json-ld](https://en.wikipedia.org/wiki/JSON-LD) format using [OSFMAP](https://osf.io/8yczr) property keys, with some noisier properties (e.g. list of all files in a project) omitted -- this is the format expected by OSF's current search frontends +* "sharev2_elastic": a json format for back-compat with 2016's [SHAREv2 search api](https://staging-share.osf.io/api/v2/search/creativeworks/_search), a raw pass-thru to elasticsearch +* "oaidc_xml": a minimal xml format used in shtrove's [/oai-pmh/](https://staging-share.osf.io/oai-pmh/?verb=ListRecords&metadataPrefix=oai_dc) feed, following the [OAI Protocol for Metadata Harvesting](https://www.openarchives.org/OAI/openarchivesprotocol.html) + +once all relevant `DerivedIndexcard`s are saved, `task__derive` enqueues messages +(using `share.search.index_messenger`) to notify shtrove's "indexer daemon" that the +given `Indexcard` is ready for indexing + +## 4. index multiple ways +shtrove's indexer daemon (see `share.search.daemon`) is meant for efficient bulk-indexing +to multiple indexing strategies in parallel -- each index strategy has its own separate +message queues and daemon threads for urgent and nonurgent updates, where each thread fetches +a chunk of messages and streams updates to elasticsearch's bulk-index api (or according to logic +in the indexing strategy) + +(note: on reflection, there's some logical redundancy between urgent/nonurgent queues and +update/backfill message types that shouldn't cause problems but could likely be simplified) + +there are currently only two indexing strategies: "sharev2_elastic8" and "trovesearch_denorm" + +the "sharev2_elastic8" strategy loads the sharev2_elastic `DerivedIndexcard` and puts the json +into an elasticsearch8 index as-is, to support the legacy sharev2 search (passthru to elasticsearch) + +the "trovesearch_denorm" strategy loads and traverses the combined knowledge graph +(`LatestResourceDescription` and each `SupplementaryResourceDescription`) to populate two +elasticsearch8 indexes: one index that supports requests to `/trove/index-card-search` and +another index that supports `/trove/index-value-search` +(see [search api docs](https://staging-share.osf.io/trove/docs) for details of expected behavior) +-- allows indexing and querying on arbitrary graphs, including on metadata properties that +were not known ahead of time (e.g. from CEDAR template instances) + +(note: this multi-strategy approach allows for experimenting and side-by-side comparison of +different implementations of the trove search api, which offers some complex functionality that +could be implemented a variety of ways -- if there were multiple trovesearch strategies +(as there was before settling on trovesearch_denorm), can select a non-default one with +the `indexStrategy` query param) + +## OSF-to-shtrove ingestion sequence +(slightly simplified) +```mermaid +sequenceDiagram + participant ms as metadata source + box shtrove + participant ss as web server + participant sd as db + participant sw as worker + participant sq as queues + participant si as indexer + participant se as elastic + end + ms ->> ss: POST /trove/ingest + ss ->> sd: save metadata record + ss ->> sq: enqueue derive task + ss ->> ow: success! + sq -->> sw: receive derive task + sd <<-->> sw: load metadata record + sw ->> sd: save DerivedIndexcards + sw ->> sq: enqueue indexer message + sq -->> si: bulk receive indexer messages + sd <<-->> si: bulk load metadata + si ->> se: bulk index +``` diff --git a/how-to/use-the-admin-site.md b/how-to/use-the-admin-site.md new file mode 100644 index 000000000..18ef19c3f --- /dev/null +++ b/how-to/use-the-admin-site.md @@ -0,0 +1,94 @@ +# how to use the admin ui + +shtrove has a [django admin site](https://docs.djangoproject.com/en/stable/ref/contrib/admin/) +at `/admin/` that can be used for interacting with the database and managing search indexes +-- this document describes how to use the admin site for some common tasks + +## prerequisite: creating an admin user +an admin user is automatically created as part of initial setup (database migrations) +when the `SHARE_ADMIN_PASSWORD` environment variable (or django setting) is non-empty, +with username from `SHARE_ADMIN_USERNAME` (default "admin") + +in local development, when [DEBUG](https://docs.djangoproject.com/en/stable/ref/settings/#debug) +is true and you shouldn't have any sensitive data, `SHARE_ADMIN_PASSWORD` defaults to "password" + +other `ShareUser` instances (perhaps created by logging in with OSF OAuth, below) can be granted +admin-site permissions -- find them at `/admin/share/shareuser/`, set "Staff status" +(`is_staff`) to give access to the admin site, and either select specific permissions +or set "Superuser status" (`is_superuser`) to give all permissions. + +### with OSF account +when set up with OSF, you can also click "login with osf" (or go to `/accounts/osf/login/`) +to login with an OSF account -- this will create a `ShareUser` instance that can be granted +admin-site permissions, as above + +## how to view metadata about a specific resource +to find metadata about a specific resource, use the `Indexcard` model as a starting point -- click +on "Indexcards" in the "TROVE" section of `/admin/` (or go to `/admin/trove/indexcard/` directly) +and search by either indexcard uuid or source-unique identifier (e.g. OSF id) + +an Indexcard's detail view has links to: +- current version of core metadata ("latest resource description") +- past versions of core metadata ("archived description set") +- current supplementary metadata ("supplementary description set") +- different serializations of current metadata ("derived indexcard set") + +if an Indexcard has been marked deleted, its "latest" and "derived" data will be removed +but "archived" will remain + +## how to view index status +click on the "elasticsearch indexes" link in the admin-site header (or go to `/admin/search-indexes`) +to see the current status of all configured index strategies, including their queues and indexes + +for each available index strategy: +- status of urgent and non-urgent queues + - depth (count of enqueued messages) + - approximate indexing rate over the past 30 seconds +- for the current version of this strategy: + - backfill status (if any) + - for manual tracking of backfill progress + - click link to view/reset backfill status and error message, if any + - lifecycle controls (each displayed only when doable): + - setup (create indexes -- note only the current version can be set up) + - start keeping live (receive new metadata updates) + - start backfill (schedule task to enqueue indexer messages for all existing cards) + - mark backfill complete (to be done manually, when queue clear) + - make default for searching (each strategy has only one default) + - delete (must confirm by typing "really really") + - status of each specific index within this strategy version: + - key (short name within the strategy) + - created date + - kept-live state (whether it receives new metadata updates) + - doc count (as of last index refresh), with direct link to view index + - link to elasticsearch mappings + - full index name +- for each prior version of this same strategy (that still has existing indexes): + - unique checksum of index mappings/settings + - lifecycle controls (each displayed only when doable): + - start keeping live (receive new metadata updates) + - make default for searching (each strategy has only one default) + - delete (must confirm by typing "really really" -- note prior versions cannot be recreated from current code) + - status of each specific index within this strategy version: + - key (short name within the strategy) + - created date + - kept-live state (whether it receives new metadata updates) + - doc count (as of last index refresh), with direct link to view index + - link to elasticsearch mappings + - full index name + +see [how to wrangle index strategies](./wrangle-index-strategies.md) for more context + +## how to diagnose ingestion problems + +if something is missing from search: +1. does it have an Indexcard (see above for finding it)? if not: + - requests to `/trove/ingest` are either failing or not being sent (check OSF task logs? or shtrove web-server errors?) +2. does the Indexcard have a `deleted` date? if so: + - the source (OSF) apparently sent a DELETE request (check why?) +3. does the Indexcard have `DerivedIndexcard`s? if not: + - `task__derive` may be failing (check for errors at `/admin/share/celerytaskresult/`) + - ...or not running at all (check for a backup in `digestive_tract` queues, or problems with shtrove's `worker`) +4. is there a large queue depth at `/admin/search-indexes`? if so: + - check shtrove's `indexer` logs for errors +5. if all above seems fine: + - compare the search query to the metadata record -- any mismatched assumptions? diff --git a/how-to/use-the-api.md b/how-to/use-the-api.md index c5c33a990..919af33ff 100644 --- a/how-to/use-the-api.md +++ b/how-to/use-the-api.md @@ -11,7 +11,7 @@ (see [openapi docs](https://share.osf.io/trove/docs) for detail and available parameters) -### Posting index-cards +## Posting index-cards > NOTE: currently used only by other COS projects, not yet for public use, authorization required `POST /trove/ingest?focus_iri=...`: diff --git a/how-to/wrangle-index-strategies.md b/how-to/wrangle-index-strategies.md new file mode 100644 index 000000000..5a012f4bd --- /dev/null +++ b/how-to/wrangle-index-strategies.md @@ -0,0 +1,35 @@ +# how to wrangle index strategies +`IndexStrategy` exists to make it easy to index the same trove of metadata in multiple different +ways that may be compared, replaced, and improved over time + +## strategy checksums (versions) +the version of an index strategy is identified by a checksum of the mappings and settings for +each of its indexes -- there can be only one "current" version of an index strategy, which +is statically defined in code, but there may be multiple "prior" versions that were created +from earlier code but which still exist in elasticsearch (and may still be used) + +when a strategy version is "kept live", it will receive new metadata updates in the "current" +document format -- this means any new version of an existing index strategy must have a +back-compatible document structure to avoid breaking prior versions + +if you want to change things in a non-breaking way (like changing index settings, changing field +mappings without changing the source document, or adding new fields (without `dynamic: "strict"`)), +you can change the `IndexStrategy` subclass in-place to create a new "current" version of it + +if you want to change things in a breaking way (like incompatible document structure or a +major-version upgrade of elasticsearch) you should instead create a new index strategy + +## how to set up a new version of an index strategy +1. change the mappings/settings `IndexStrategy` subclass, making sure document structure is still back-compatible +2. update the `IndexStrategy` subclass's `CURRENT_STRATEGY_CHECKSUM` (easy way: follow the error message from trying to run a django service or command) +3. ensure all services are running up-to-date code (in local dev setup, restart `indexer`, `worker`, and `web`) +4. in the [shtrove admin site](./use-the-admin-site.md), use `/admin/search-indexes` to setup and backfill the new current version +5. use the trove api with `indexStrategy` query param (using the full `mystrategy__checksum...` identifier) to test/compare against the prior (default) version +6. use `/admin/search-indexes` to make the updated "current" the new default for searching with that strategy + +## how to set up a new index strategy +1. add an instance of your `IndexStrategy` subclass to `share.search.index_strategy._AvailableStrategies` with a unique shortname (this should someday be configurable in settings, but is currently hard-coded) +2. ensure all services are running up-to-date code (in local dev setup, restart `indexer`, `worker`, and `web`) +3. in the [shtrove admin site](./use-the-admin-site.md), use `/admin/search-indexes` to setup and backfill the current version of your new index strategy +4. use the api with `indexStrategy` query param to test/compare against other strategies +5. to update the default strategy for the trove search api (`/trove/index-card-search`, `/trove/index-value-search`), change the hard-coded default in `share.search.index_strategy.get_strategy_for_trovesearch` (this should also someday be configurable in settings (or admin site), but is not currently) From 6bd38db2a18604d00bef000c58ab4c37e20a6d89 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Jul 2026 13:58:52 -0400 Subject: [PATCH 05/12] fix(admin-site): default-for-searching indicator --- share/admin/search.py | 25 +++------- share/search/index_status.py | 1 - share/search/index_strategy/elastic8.py | 5 -- templates/admin/search-indexes.html | 50 +++++++++++-------- .../index_strategy/_with_real_services.py | 46 +++++++++-------- 5 files changed, 59 insertions(+), 68 deletions(-) diff --git a/share/admin/search.py b/share/admin/search.py index 8894346dd..f35616508 100644 --- a/share/admin/search.py +++ b/share/admin/search.py @@ -9,7 +9,6 @@ from share.search.index_messenger import IndexMessenger from share.search.index_strategy import ( IndexStrategy, - all_strategy_names, each_strategy, parse_strategy_name, parse_specific_index_name, @@ -54,23 +53,12 @@ def _mappings_url_prefix(): def _index_status_by_strategy(): - _backfill_by_checksum: dict[str, IndexBackfill] = { - _backfill.strategy_checksum: _backfill - for _backfill in ( - IndexBackfill.objects - .filter(index_strategy_name__in=all_strategy_names()) - ) - } status_by_strategy = {} _messenger = IndexMessenger() for _index_strategy in each_strategy(): - _current_backfill = ( - _backfill_by_checksum.get(str(_index_strategy.CURRENT_STRATEGY_CHECKSUM)) - or _backfill_by_checksum.get(_index_strategy.indexname_prefix) # backcompat - ) status_by_strategy[_index_strategy.strategy_name] = { 'status': _index_strategy.pls_get_strategy_status(), - 'backfill': _serialize_backfill(_index_strategy, _current_backfill), + 'backfill': _serialize_backfill(_index_strategy), 'queues': [ { 'name': _queue_name, @@ -85,13 +73,14 @@ def _index_status_by_strategy(): return status_by_strategy -def _serialize_backfill( - strategy: IndexStrategy, - backfill: IndexBackfill | None, -): +def _serialize_backfill(strategy: IndexStrategy): if not strategy.is_current: return {} - if not backfill: + try: + backfill = IndexBackfill.objects.get(index_strategy_name=strategy.strategy_name) + except IndexBackfill.DoesNotExist: + backfill = None + if not backfill or (backfill.strategy_checksum != str(strategy.CURRENT_STRATEGY_CHECKSUM)): return { 'can_start_backfill': strategy.pls_check_exists(), } diff --git a/share/search/index_status.py b/share/search/index_status.py index 1ed16f9b7..f62fab1ff 100644 --- a/share/search/index_status.py +++ b/share/search/index_status.py @@ -9,7 +9,6 @@ class IndexStatus: specific_indexname: str doc_count: int = 0 is_kept_live: bool = False - is_default_for_searching: bool = False @dataclasses.dataclass diff --git a/share/search/index_strategy/elastic8.py b/share/search/index_strategy/elastic8.py index bc4b30f2e..2c61e771e 100644 --- a/share/search/index_strategy/elastic8.py +++ b/share/search/index_strategy/elastic8.py @@ -360,7 +360,6 @@ def pls_get_status(self) -> IndexStatus: index_subname=self.subname, specific_indexname=self.full_index_name, is_kept_live=False, - is_default_for_searching=False, doc_count=0, creation_date='', ) @@ -385,10 +384,6 @@ def pls_get_status(self) -> IndexStatus: self.index_strategy._alias_for_keeping_live in index_aliases ), - is_default_for_searching=( - self.index_strategy._alias_for_searching - in index_aliases - ), creation_date=creation_date, doc_count=doc_count, ) diff --git a/templates/admin/search-indexes.html b/templates/admin/search-indexes.html index 9f2e6bb30..358d3cb83 100644 --- a/templates/admin/search-indexes.html +++ b/templates/admin/search-indexes.html @@ -38,6 +38,9 @@

queues

current: {{ strategy_info.status.strategy_id }}

+ {% if strategy_info.status.is_default_for_searching %} +

{% blocktranslate %}(default for searching with {{ index_strategy_name }}){% endblocktranslate %}

+ {% endif %} - + - @@ -106,7 +99,6 @@

current: {{ strategy_info.status.strategy_id }}

- {% endfor %}
{% trans "index" %}{% trans "key" %} {% trans "created" %} {% trans "is kept live" %}{% trans "is default for searching" %} {% trans "doc count" %} {% trans "links" %} {% trans "full index name" %} {{ current_index_status.index_subname }} {{ current_index_status.creation_date|default:"--" }} {% if current_index_status.is_kept_live %}✓{% endif %}{% if current_index_status.is_default_for_searching %}✓{% endif %} {{ current_index_status.doc_count }} {% if current_index_status.creation_date %} @@ -117,10 +109,23 @@

current: {{ strategy_info.status.strategy_id }}

+ {% if strategy_info.status.is_set_up %} +
+ {% csrf_token %} + + + + +
+ {% endif %}
{% for prior_strategy_status in strategy_info.status.existing_prior_strategies %}

prior: {{ prior_strategy_status.strategy_id }}

+ {% if prior_strategy_status.is_default_for_searching %} +

{% blocktranslate %}(default for searching with {{ index_strategy_name }}){% endblocktranslate %}

+ {% endif %} - + - @@ -168,7 +165,6 @@

prior: {{ prior_strategy_status.strategy_id }}

- {% endfor %}
{% trans "index" %}{% trans "key" %} {% trans "created" %} {% trans "is kept live" %}{% trans "is default for searching" %} {% trans "doc count" %} {% trans "links" %} {% trans "full index name" %} {{ index_status.index_subname }} {{ index_status.creation_date }} {% if index_status.is_kept_live %}✓{% endif %}{% if index_status.is_default_for_searching %}✓{% endif %} {{ index_status.doc_count }} {% if index_status.creation_date %}

{% trans "mappings" %}

@@ -178,6 +174,16 @@

prior: {{ prior_strategy_status.strategy_id }}

+ {% if not prior_strategy_status.is_kept_live %} +
+ {% csrf_token %} + + + + +
+ {% endif %}
{% endfor %} diff --git a/tests/share/search/index_strategy/_with_real_services.py b/tests/share/search/index_strategy/_with_real_services.py index ec4076668..2114aeaf0 100644 --- a/tests/share/search/index_strategy/_with_real_services.py +++ b/tests/share/search/index_strategy/_with_real_services.py @@ -87,33 +87,35 @@ def _assert_setup_happypath(self): # initial (no indexes exist) for _index in self.index_strategy.each_subnamed_index(): assert not _index.pls_check_exists() - index_status = _index.pls_get_status() - assert not index_status.creation_date - assert not index_status.is_kept_live - assert not index_status.is_default_for_searching - assert not index_status.doc_count + _status = self.index_strategy.pls_get_strategy_status() + assert not _status.is_kept_live + assert not _status.is_default_for_searching + for _index_status in _status.index_statuses: + assert not _index_status.creation_date + assert not _index_status.doc_count # create each index for _index in self.index_strategy.each_subnamed_index(): _index.pls_create() assert _index.pls_check_exists() # new! - index_status = _index.pls_get_status() - assert index_status.creation_date # new! - assert not index_status.is_kept_live - assert not index_status.is_default_for_searching - assert not index_status.doc_count + _status = self.index_strategy.pls_get_strategy_status() + assert not _status.is_kept_live + assert not _status.is_default_for_searching + for _index_status in _status.index_statuses: + assert _index_status.creation_date # new! + assert not _index_status.doc_count # start keeping each index live (with ingested updates) self.index_strategy.pls_start_keeping_live() - for _index in self.index_strategy.each_subnamed_index(): - index_status = _index.pls_get_status() - assert index_status.creation_date - assert index_status.is_kept_live # new! - assert not index_status.is_default_for_searching - assert not index_status.doc_count + _status = self.index_strategy.pls_get_strategy_status() + assert _status.is_kept_live # new! + assert not _status.is_default_for_searching + for _index_status in _status.index_statuses: + assert _index_status.creation_date + assert not _index_status.doc_count # make this version of the strategy the default for searching self.index_strategy.pls_make_default_for_searching() - for _index in self.index_strategy.each_subnamed_index(): - index_status = _index.pls_get_status() - assert index_status.creation_date - assert index_status.is_kept_live - assert index_status.is_default_for_searching # new! - assert not index_status.doc_count # (still empty) + _status = self.index_strategy.pls_get_strategy_status() + assert _status.is_kept_live + assert _status.is_default_for_searching # new! + for _index_status in _status.index_statuses: + assert _index_status.creation_date + assert not _index_status.doc_count # (still empty) From 63940e7588dbb6f22c4acb0d9c85a3735e2c61e9 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Jul 2026 13:59:20 -0400 Subject: [PATCH 06/12] fix: import error in shell_plus command --- share/shell_util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/share/shell_util.py b/share/shell_util.py index 223f13304..258bb4882 100644 --- a/share/shell_util.py +++ b/share/shell_util.py @@ -3,7 +3,6 @@ that is, the shell you get from `python manage.py shell_plus` """ -from share import tasks from share.search import IndexMessenger, index_strategy from share.util import IDObfuscator @@ -12,5 +11,4 @@ 'IDObfuscator', 'IndexMessenger', 'index_strategy', - 'tasks', ) From dd02d89a381f8c57d934117ce44973f145634503 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Jul 2026 14:00:03 -0400 Subject: [PATCH 07/12] fix(admin-site): timeout on indexcard search --- trove/admin.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/trove/admin.py b/trove/admin.py index 4cf9cdff1..34738c825 100644 --- a/trove/admin.py +++ b/trove/admin.py @@ -1,11 +1,13 @@ from __future__ import annotations from typing import Any from django.contrib import admin +from django.db.models import Q from django.utils.html import format_html from share.admin import admin_site from share.admin.util import TimeLimitedPaginator, linked_fk, linked_many from share.search.index_messenger import IndexMessenger +from share.models import SourceUniqueIdentifier from trove.models import ( ArchivedResourceDescription, DerivedIndexcard, @@ -49,11 +51,19 @@ class IndexcardAdmin(admin.ModelAdmin): paginator = TimeLimitedPaginator list_display = ('uuid', 'source_record_suid', 'created', 'modified') show_full_result_count = False - search_fields = ('uuid', 'source_record_suid__identifier') + search_fields = ('uuid', 'source_record_suid__identifier') # see get_search_results list_select_related = ('source_record_suid',) list_filter = ('deleted', 'source_record_suid__source_config') actions = ('_freshen_index',) + def get_search_results(self, request, queryset, search_term): + # allow search on source_record_suid__identifier more efficiently than with search_fields + _suid_qs = SourceUniqueIdentifier.objects.filter(identifier=search_term) + _indexcard_qs = queryset.filter( + Q(uuid=search_term) | Q(source_record_suid__in=_suid_qs) + ) + return _indexcard_qs, False # no duplicates + def _freshen_index(self, request, queryset: list[Indexcard]) -> None: IndexMessenger().notify_indexcard_update(queryset) _freshen_index.short_description = 'freshen indexcard in search index' # type: ignore[attr-defined] From 8c6f5801ecd6d62ca5c8657ded010adc780b00f9 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Fri, 10 Jul 2026 14:53:47 -0400 Subject: [PATCH 08/12] tidy admin/search-indexes --- how-to/use-the-admin-site.md | 12 ++--- how-to/wrangle-index-strategies.md | 5 +- share/admin/search.py | 5 -- share/search/index_strategy/__init__.py | 6 +-- share/search/index_strategy/_base.py | 9 ++-- share/search/index_strategy/elastic8.py | 4 +- templates/admin/search-indexes.html | 66 ++++++++++++------------- 7 files changed, 52 insertions(+), 55 deletions(-) diff --git a/how-to/use-the-admin-site.md b/how-to/use-the-admin-site.md index 18ef19c3f..fabdefca5 100644 --- a/how-to/use-the-admin-site.md +++ b/how-to/use-the-admin-site.md @@ -48,13 +48,13 @@ for each available index strategy: - backfill status (if any) - for manual tracking of backfill progress - click link to view/reset backfill status and error message, if any - - lifecycle controls (each displayed only when doable): + - lifecycle controls (each displayed only when safe to do): - setup (create indexes -- note only the current version can be set up) - start keeping live (receive new metadata updates) - start backfill (schedule task to enqueue indexer messages for all existing cards) - mark backfill complete (to be done manually, when queue clear) - - make default for searching (each strategy has only one default) - - delete (must confirm by typing "really really") + - make default for searching within this strategy (only when backfill marked complete) + - delete -- must confirm by typing "really really" - status of each specific index within this strategy version: - key (short name within the strategy) - created date @@ -64,10 +64,10 @@ for each available index strategy: - full index name - for each prior version of this same strategy (that still has existing indexes): - unique checksum of index mappings/settings - - lifecycle controls (each displayed only when doable): + - lifecycle controls (each displayed only when safe to do): - start keeping live (receive new metadata updates) - - make default for searching (each strategy has only one default) - - delete (must confirm by typing "really really" -- note prior versions cannot be recreated from current code) + - make default for searching within this strategy (only when backfill marked complete) + - delete (only when not default for searching) -- must confirm by typing "really really" - status of each specific index within this strategy version: - key (short name within the strategy) - created date diff --git a/how-to/wrangle-index-strategies.md b/how-to/wrangle-index-strategies.md index 5a012f4bd..7ad2c7e9c 100644 --- a/how-to/wrangle-index-strategies.md +++ b/how-to/wrangle-index-strategies.md @@ -25,7 +25,10 @@ major-version upgrade of elasticsearch) you should instead create a new index st 3. ensure all services are running up-to-date code (in local dev setup, restart `indexer`, `worker`, and `web`) 4. in the [shtrove admin site](./use-the-admin-site.md), use `/admin/search-indexes` to setup and backfill the new current version 5. use the trove api with `indexStrategy` query param (using the full `mystrategy__checksum...` identifier) to test/compare against the prior (default) version -6. use `/admin/search-indexes` to make the updated "current" the new default for searching with that strategy +6. when ready to switch over to use the new version by default, use `/admin/search-indexes`: + - +6. when ready to switch over use `/admin/search-indexes` to make the updated "current" the new default for searching with that strategy +7. when ready to delete a prior version, use `/admin/search-indexes` ## how to set up a new index strategy 1. add an instance of your `IndexStrategy` subclass to `share.search.index_strategy._AvailableStrategies` with a unique shortname (this should someday be configurable in settings, but is currently hard-coded) diff --git a/share/admin/search.py b/share/admin/search.py index f35616508..db5c83f39 100644 --- a/share/admin/search.py +++ b/share/admin/search.py @@ -102,10 +102,6 @@ def _pls_start_keeping_live(index_strategy: IndexStrategy, request_kwargs): index_strategy.pls_start_keeping_live() -def _pls_stop_keeping_live(index_strategy: IndexStrategy, request_kwargs): - index_strategy.pls_stop_keeping_live() - - def _pls_start_backfill(index_strategy: IndexStrategy, request_kwargs): assert index_strategy.is_current index_strategy.pls_start_backfill() @@ -130,6 +126,5 @@ def _pls_delete(index_strategy: IndexStrategy, request_kwargs): 'start_backfill': _pls_start_backfill, 'mark_backfill_complete': _pls_mark_backfill_complete, 'make_default_for_searching': _pls_make_default_for_searching, - 'stop_keeping_live': _pls_stop_keeping_live, 'delete': _pls_delete, } diff --git a/share/search/index_strategy/__init__.py b/share/search/index_strategy/__init__.py index ff5100d35..fc8ce8564 100644 --- a/share/search/index_strategy/__init__.py +++ b/share/search/index_strategy/__init__.py @@ -64,9 +64,9 @@ def get_strategy( if strategy_check: _strategy = _strategy.with_strategy_check(strategy_check) return ( - _strategy.pls_get_default_for_searching() - if (for_search and not strategy_check) - else _strategy + _strategy + if strategy_check or not for_search + else _strategy.pls_get_default_for_searching() or _strategy ) diff --git a/share/search/index_strategy/_base.py b/share/search/index_strategy/_base.py index e25153398..c2eeb3b58 100644 --- a/share/search/index_strategy/_base.py +++ b/share/search/index_strategy/_base.py @@ -130,10 +130,9 @@ def pls_setup(self, *, skip_backfill=False) -> None: for _index in self.each_subnamed_index(): _index.pls_create() _index.pls_start_keeping_live() - if skip_backfill: - _backfill = self.get_or_create_backfill() - _backfill.backfill_status = _backfill.COMPLETE - _backfill.save() + _backfill = self.get_or_create_backfill() + _backfill.backfill_status = (_backfill.COMPLETE if skip_backfill else _backfill.INITIAL) + _backfill.save() def pls_teardown(self) -> None: for _index in self.each_existing_index(): @@ -245,7 +244,7 @@ def pls_make_default_for_searching(self) -> None: raise NotImplementedError @abc.abstractmethod - def pls_get_default_for_searching(self) -> IndexStrategy: + def pls_get_default_for_searching(self) -> IndexStrategy | None: raise NotImplementedError ### diff --git a/share/search/index_strategy/elastic8.py b/share/search/index_strategy/elastic8.py index 2c61e771e..85ada812b 100644 --- a/share/search/index_strategy/elastic8.py +++ b/share/search/index_strategy/elastic8.py @@ -228,12 +228,12 @@ def pls_make_default_for_searching(self): ) # abstract method from IndexStrategy - def pls_get_default_for_searching(self) -> IndexStrategy: + def pls_get_default_for_searching(self) -> IndexStrategy | None: _searchnames = self._get_indexnames_for_alias(self._alias_for_searching) try: (_indexname, *_) = _searchnames except ValueError: - return self # no default set, this one's fine + return None # no default set (_strategyname, _strategycheck, *_) = parse_indexname_parts(_indexname) assert _strategyname == self.strategy_name _strategycheck = _strategycheck.rstrip('*') # may be a wildcard alias diff --git a/templates/admin/search-indexes.html b/templates/admin/search-indexes.html index 358d3cb83..e04411b79 100644 --- a/templates/admin/search-indexes.html +++ b/templates/admin/search-indexes.html @@ -3,8 +3,12 @@ {% block extrastyle %}