From ac699fe5f94dbde8d58a3ff3fad457e8544d7ad3 Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 11:40:25 -0400 Subject: [PATCH 1/4] First work towards supporting tokenization parameters --- app/controllers/search_controller.rb | 3 ++- app/helpers/application_helper.rb | 3 ++- app/models/enhancer.rb | 3 ++- app/models/query_builder.rb | 3 ++- app/models/timdex_search.rb | 6 ++++++ 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 58e02d8e..a97821b5 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -225,7 +225,8 @@ def query_primo(per_page, offset) end def execute_geospatial_query(query) - query = query.except('queryMode') + query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold', + 'semanticShortQueryMaxTokens') if query['geobox'] == 'true' && query[:geodistance] == 'true' TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index b085ceb8..9be0d2fc 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -20,7 +20,8 @@ def index_page_title def results_page_title(query, character_limit = 50) return index_page_title unless query.present? - ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode] + ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold + semanticMustBoostThreshold semanticShortQueryMaxTokens] terms = query.reject { |term| ignored_terms.include? term }.values.join(' ') terms = "#{terms.first(character_limit)}..." if terms.length > character_limit "#{terms} | #{index_page_title}" diff --git a/app/models/enhancer.rb b/app/models/enhancer.rb index b21099bf..7ee32a4a 100644 --- a/app/models/enhancer.rb +++ b/app/models/enhancer.rb @@ -1,7 +1,8 @@ class Enhancer attr_accessor :enhanced_query - QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze + QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title + queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 76491ca0..5095583a 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -1,7 +1,8 @@ class QueryBuilder attr_reader :query - QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType].freeze + QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType + semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb index d8141baf..c8ceea43 100644 --- a/app/models/timdex_search.rb +++ b/app/models/timdex_search.rb @@ -28,6 +28,9 @@ class TimdexSearch < TimdexBase $sourceFilter: [String!] $subjectsFilter: [String!] $useGlobalScoring: Boolean + $semanticDropBoostThreshold: Float + $semanticMustBoostThreshold: Float + $semanticShortQueryMaxTokens: Int ) { search( searchterm: $q @@ -54,6 +57,9 @@ class TimdexSearch < TimdexBase sourceFilter: $sourceFilter subjectsFilter: $subjectsFilter useGlobalScoring: $useGlobalScoring + semanticDropBoostThreshold: $semanticDropBoostThreshold + semanticMustBoostThreshold: $semanticMustBoostThreshold + semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens ) { hits records { From b3adbae4bb82e92202046699087db389a85f850e Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 14:53:19 -0400 Subject: [PATCH 2/4] Remove tokenizer tuning parameters --- app/models/timdex_search.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb index c8ceea43..d8141baf 100644 --- a/app/models/timdex_search.rb +++ b/app/models/timdex_search.rb @@ -28,9 +28,6 @@ class TimdexSearch < TimdexBase $sourceFilter: [String!] $subjectsFilter: [String!] $useGlobalScoring: Boolean - $semanticDropBoostThreshold: Float - $semanticMustBoostThreshold: Float - $semanticShortQueryMaxTokens: Int ) { search( searchterm: $q @@ -57,9 +54,6 @@ class TimdexSearch < TimdexBase sourceFilter: $sourceFilter subjectsFilter: $subjectsFilter useGlobalScoring: $useGlobalScoring - semanticDropBoostThreshold: $semanticDropBoostThreshold - semanticMustBoostThreshold: $semanticMustBoostThreshold - semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens ) { hits records { From 85b912065b8dc0238efe6ebd15ac84a3c94c7bdd Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 16:16:01 -0400 Subject: [PATCH 3/4] Move tokenization parameters to their own extraction method --- app/models/query_builder.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 5095583a..c86f5ef3 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -1,13 +1,13 @@ class QueryBuilder attr_reader :query - QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType - semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze + QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude geodistanceLongitude geodistanceDistance].freeze VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze + TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze def initialize(enhanced_query) @query = {} @@ -23,6 +23,7 @@ def initialize(enhanced_query) extract_geosearch(enhanced_query) extract_filters(enhanced_query) evaluate_query_mode(enhanced_query) + extract_tokenization_params(enhanced_query) @query['index'] = ENV.fetch('TIMDEX_INDEX', nil) @query['booleanType'] = enhanced_query[:booleanType] @query.compact! @@ -60,6 +61,12 @@ def extract_filters(enhanced_query) end end + def extract_tokenization_params(enhanced_query) + TOKENIZATION_PARAMS.each do |tp| + @query[tp] = enhanced_query[tp.to_sym]&.strip.to_f if enhanced_query[tp.to_sym].present? + end + end + # The GraphQL API requires that lat/long in geospatial fields be floats def coerce_to_float?(geo_param) geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude') From 8fa35efc3de98baed5e35870dc71ff6a87c39438 Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 29 Jul 2026 22:07:16 -0400 Subject: [PATCH 4/4] Build separate pathway for tuning queries This is necessary to bypass the schema validation that comes for free with the graphql-client gem. No tests yet because this is super rough. --- app/controllers/search_controller.rb | 56 +++++++- app/models/timdex_tuning.rb | 151 +++++++++++++++++++++ test/controllers/search_controller_test.rb | 20 +++ 3 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 app/models/timdex_tuning.rb diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index a97821b5..5b37a0b1 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -200,16 +200,19 @@ def query_timdex(query) Rails.cache.fetch("#{cache_key}/#{@active_tab}", expires_in: 12.hours) do raw = if Feature.enabled?(:geodata) execute_geospatial_query(query) + elsif tuning_request?(query) + execute_tuning_query(query) else TimdexBase::Client.query(TimdexSearch::BaseQuery, variables: query) end # The response type is a GraphQL::Client::Response, which is not directly serializable, so we # convert it to a hash. - { - data: raw.data.to_h, - errors: raw.errors.details.to_h - } + # { + # data: raw.data.to_h, + # errors: raw.errors.details.to_h + # } + process_timdex_response(raw, query) end end @@ -239,6 +242,29 @@ def execute_geospatial_query(query) end end + # We rely on undocumented parameters in TIMDEX for tuning purposes, which cannot be used with the graphql-client gem + # because they will fail during validation. As a result, when tuning parameters are present in the web request, we + # fall back to NET::HTTP to communicate with TIMDEX. + def execute_tuning_query(query_vars) + uri = URI(ENV.fetch('TIMDEX_GRAPHQL', '')) + + req = Net::HTTP::Post.new(uri) + req['Content-Type'] = 'application/json' + req['Accept'] = 'application/json' + req['User-Agent'] = 'MIT Libraries Client' + + req.body = JSON.generate( + query: TimdexTuning::TuningQuery, + variables: query_vars + ) + + res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(req) + end + + JSON.parse(res.body) + end + def extract_errors(response) response[:errors]['data'] if response.is_a?(Hash) && response.key?(:errors) && response[:errors].key?('data') end @@ -454,4 +480,26 @@ def valid_token? def show_nls_warning? @natural_language_search_optin && primo_tabs.include?(@active_tab) end + + def tuning_request?(query) + tuning_params = %w(semanticMustBoostThreshold semanticDropBoostThreshold semanticShortQueryMaxTokens) + + tuning_params.any? do |term| + query.keys.include?(term) + end + end + + def process_timdex_response(raw, query) + if tuning_request?(query) + { + data: (raw["data"].nil? ? {} : raw["data"]), + errors: (raw["errors"].nil? ? {} : raw["errors"]) + } + else + { + data: raw.data.to_h, + errors: raw.errors.details.to_h + } + end + end end diff --git a/app/models/timdex_tuning.rb b/app/models/timdex_tuning.rb new file mode 100644 index 00000000..8e2e455b --- /dev/null +++ b/app/models/timdex_tuning.rb @@ -0,0 +1,151 @@ +class TimdexTuning + TuningQuery = <<-GRAPHQL + query TimdexPlaygroundQuery( + $q: String, + $citation: String, + $contributors: String, + $fundingInformation: String, + $identifiers: String, + $locations: String, + $subjects: String, + $title: String, + $index: String, + $from: String, + $booleanType: String, + $queryMode: String, + $fulltext: Boolean, + $perPage: Int, + $accessToFilesFilter: [String!], + $contentTypeFilter: [String!], + $contributorsFilter: [String!], + $formatFilter: [String!], + $languagesFilter: [String!], + $literaryFormFilter: String, + $placesFilter: [String!], + $sourceFilter: [String!], + $subjectsFilter: [String!], + $useGlobalScoring: Boolean, + $semanticDropBoostThreshold: Float, + $semanticMustBoostThreshold: Float + ) { + search( + searchterm: $q + citation: $citation + contributors: $contributors + fundingInformation: $fundingInformation + identifiers: $identifiers + locations: $locations + subjects: $subjects + title: $title + index: $index + from: $from + booleanType: $booleanType + queryMode: $queryMode + fulltext: $fulltext + perPage: $perPage + accessToFilesFilter: $accessToFilesFilter + contentTypeFilter: $contentTypeFilter + contributorsFilter: $contributorsFilter + formatFilter: $formatFilter + languagesFilter: $languagesFilter + literaryFormFilter: $literaryFormFilter + placesFilter: $placesFilter + sourceFilter: $sourceFilter + subjectsFilter: $subjectsFilter + useGlobalScoring: $useGlobalScoring + semanticDropBoostThreshold: $semanticDropBoostThreshold + semanticMustBoostThreshold: $semanticMustBoostThreshold + ) { + hits + records { + timdexRecordId + identifiers { + kind + value + } + title + source + contentType + contributors { + kind + value + } + publicationInformation + dates { + kind + value + range { + gte + lte + } + } + links { + kind + restrictions + text + url + } + notes { + kind + value + } + highlight { + matchedField + matchedPhrases + } + provider + rights { + kind + description + uri + } + sourceLink + summary + subjects { + kind + value + } + citation + } + aggregations { + accessToFiles { + key + docCount + } + contentType { + key + docCount + } + contributors { + key + docCount + } + format { + key + docCount + } + languages { + key + docCount + } + literaryForm { + key + docCount + } + places { + key + docCount + } + source { + key + docCount + } + subjects { + key + docCount + } + } + } + } + GRAPHQL +end diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb index 010c527e..c4af5ed5 100644 --- a/test/controllers/search_controller_test.rb +++ b/test/controllers/search_controller_test.rb @@ -1183,6 +1183,26 @@ def source_filter_count(controller) assert_select '.pagination-container .current', text: /21 - 40 of 800/ end + # test 'results can include tuning parameters' do + # query = 'optical networks in space' + # must_default = 0.7 + # drop_default = 0.1 + # must_alt = 0.9 + # drop_alt = 0.4 + + # VCR.use_cassette('default tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_default}&semanticDropBoostThreshold=#{drop_default}&tab=timdex" + # assert_response :success + # end + + # VCR.use_cassette('alternate tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_alt}&semanticDropBoostThreshold=#{drop_alt}&tab=timdex" + # assert_response :success + # end + + # # Assert result counts are different + # end + test 'results can be returned in JSON format when env is set and valid token is provided' do secret_value = 'sooper_sekret' quepid_ua = 'Quepid/1.0 (Web Scraper)'