diff --git a/fts_base/README.md b/fts_base/README.md new file mode 100644 index 0000000..3f36891 --- /dev/null +++ b/fts_base/README.md @@ -0,0 +1,16 @@ +# fts_base + +Base to add FullText Search to Odoo models. + +## Breaking changes + +### 16.0.1.1.0 + +The method `_get_fts_proxy_values()` has been removed. It was previously called by +`_proxy_search()` to build the values inserted into `fts_proxy`, but is no longer used +since `_proxy_search()` now uses a single `INSERT...SELECT` statement instead of batched +ORM `create()` calls. + +If you have a custom module that overrides `_get_fts_proxy_values()`, that override will +silently have no effect after this update. Migrate your customisation to +`_proxy_search_select_expressions()` instead. diff --git a/fts_base/__init__.py b/fts_base/__init__.py new file mode 100644 index 0000000..50327b4 --- /dev/null +++ b/fts_base/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2012-2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +from . import models + +# from . import wizards diff --git a/fts_base/__manifest__.py b/fts_base/__manifest__.py new file mode 100644 index 0000000..61f2921 --- /dev/null +++ b/fts_base/__manifest__.py @@ -0,0 +1,21 @@ +# Copyright 2012-2026 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). + +{ + "name": "Fulltext search", + "version": "18.0.1.0.0", + "depends": ["base"], + "author": "Therp BV", + "website": "https://github.com/Therp/fulltextsearch", + "license": "AGPL-3", + "category": "Searching", + "data": [ + "security/ir.model.access.csv", + "views/fts_proxy.xml", + "views/ir_actions_server.xml", + # "wizard/fts_config.xml", + ], + "demo_xml": [], + "installable": True, + "active": False, +} diff --git a/fts_base/i18n/fts_base.pot b/fts_base/i18n/fts_base.pot new file mode 100644 index 0000000..dc70619 --- /dev/null +++ b/fts_base/i18n/fts_base.pot @@ -0,0 +1,244 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fts_base +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2024-08-02 15:21+0000\n" +"PO-Revision-Date: 2024-08-02 15:21+0000\n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "A word - will find all entries containing that word." +msgstr "" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_mixin +msgid "Add this to each model that needs FullText search functions." +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Choose some filters and then some text for Full Text search.\n" +" No search will be done, before a text is entered." +msgstr "" + +#. module: fts_base +#. odoo-python +#: code:addons/fts_base/wizards/fts_config.py:0 +#, python-format +msgid "Configuration" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__create_uid +msgid "Created by" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__create_date +msgid "Created on" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__date +msgid "Date" +msgstr "" + +#. module: fts_base +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_search +msgid "Date Created" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__display_name +msgid "Display Name" +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words between double quotation marks,\n" +" like \"cats and dogs\" will find only the entries that\n" +" contain that exact frase." +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words will find entries containing any\n" +" of those words. So entering 'cat' and 'dog' will find all\n" +" entries that have the word cat, or the word dog, or both." +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words, each followed by SHIFT+ENTER will\n" +" find those documents that contain all of the words." +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__extra +msgid "Extra" +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"For each model a maximum of 1024 entries will be found. If those do\n" +" not contain what you are looking for, it is either not there, or\n" +" you can add additional search criteria." +msgstr "" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_proxy +msgid "" +"Front end to show results of FT searches, together with rank and summary." +msgstr "" + +#. module: fts_base +#: model:ir.actions.act_window,name:fts_base.action_fts_proxy_search +#: model:ir.ui.menu,name:fts_base.menu_fts_proxy_search +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_search +msgid "Fulltext search" +msgstr "" + +#. module: fts_base +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_tree +msgid "Fulltext search - results" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__id +msgid "ID" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy____last_update +msgid "Last Modified on" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__write_uid +msgid "Last Updated by" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__write_date +msgid "Last Updated on" +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Note: to prevent unneeded searches, first enter all other criteria, like\n" +" models to search, or date, and only then the words to search for." +msgstr "" + +#. module: fts_base +#: model:ir.actions.server,name:fts_base.action_open_document +msgid "Open Document" +msgstr "" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_query_helper +msgid "" +"Provides functions to assist in FT Search.\n" +"\n" +" Define as AbstractModel for easy extention and being able\n" +" to load this through the registry.\n" +" " +msgstr "" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_debug_helper +msgid "" +"Provides logging function to analyse FT problems.\n" +"\n" +" This class is needed to not flood the log with FTS debug\n" +" messages, but only log stuff when actually having an Odoo\n" +" session in debug mode.\n" +"\n" +" We will use info level messages, so this will also work\n" +" on production servers where normally you will not have\n" +" debug level logging.\n" +" " +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__rank +msgid "Rank" +msgstr "" + +#. module: fts_base +#. odoo-python +#: code:addons/fts_base/wizards/fts_config.py:0 +#, python-format +msgid "Recreate search index" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_id +msgid "Resource ID" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_model +msgid "Resource Model" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_name +msgid "Resource Name" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__searchstring +msgid "Searchstring" +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Some text follwed by a \"*\" - will find all entries that have\n" +" a word starting with that text." +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__summary +msgid "Summary" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,help:fts_base.field_fts_proxy__res_model +msgid "The model this search works on. Required." +msgstr "" + +#. module: fts_base +#: model:ir.actions.act_window,name:fts_base.action_fts_proxy_search_with_summary +#: model:ir.ui.menu,name:fts_base.menu_fts_proxy_search_with_summary +msgid "With summary" +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"You can select one or more of the configured data collections\n" +" (models), like emails or attachments. If you do not select any,\n" +" all data collections will be searched." +msgstr "" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "You can specify criteria this way:" +msgstr "" diff --git a/fts_base/i18n/nl.po b/fts_base/i18n/nl.po new file mode 100644 index 0000000..28d1271 --- /dev/null +++ b/fts_base/i18n/nl.po @@ -0,0 +1,268 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fts_base +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2024-08-02 15:21+0000\n" +"PO-Revision-Date: 2024-08-02 15:21+0000\n" +"Last-Translator: NL66278\n" +"Language-Team: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "A word - will find all entries containing that word." +msgstr "Een woord - vindt alle records die dat woord bevatten." + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_mixin +msgid "Add this to each model that needs FullText search functions." +msgstr "Voeg dit toe aan ieder model waarin zoeken op tekst nodig is." + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Choose some filters and then some text for Full Text search.\n" +" No search will be done, before a text is entered." +msgstr "" +"Kies enkele filters en dan de tekst om op te zoeken.\n" +" Er zal niet worden gezocht voor er tekst is ingevoerd." + +#. module: fts_base +#. odoo-python +#: code:addons/fts_base/wizards/fts_config.py:0 +#, python-format +msgid "Configuration" +msgstr "Instellingen" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__create_uid +msgid "Created by" +msgstr "Aangemaakt door" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__create_date +msgid "Created on" +msgstr "Aangemaakt op" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__date +msgid "Date" +msgstr "Datum" + +#. module: fts_base +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_search +msgid "Date Created" +msgstr "Datum aangemaakt" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__display_name +msgid "Display Name" +msgstr "Weergave naam" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words between double quotation marks,\n" +" like \"cats and dogs\" will find only the entries that\n" +" contain that exact frase." +msgstr "" +"Het invoeren van meerdere woorden tussen dubbele aanhalingstekens,\n" +" zoals \"katten en honden\" zal alleen die records vinden die\n" +" die exacte frase bevatten." + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words will find entries containing any\n" +" of those words. So entering 'cat' and 'dog' will find all\n" +" entries that have the word cat, or the word dog, or both." +msgstr "" +"Het invoeren van meerdere woorden zal records vinden die\n" +" enige van deze woorden bevatten. Dus het invoeren van 'kat' en 'hond' zal alle\n" +" records vinden die of het woord 'kat' bevatten, of het woord 'hond' of beide" +" woorden." + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Entering multiple words, each followed by SHIFT+ENTER will\n" +" find those documents that contain all of the words." +msgstr "" +"Het invoeren van meerdere woorden, ieder woord gevolgd door SHIFT+ENTER zal\n" +" die records vinden die al deze woorden bevatten." + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__extra +msgid "Extra" +msgstr "Extra" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"For each model a maximum of 1024 entries will be found. If those do\n" +" not contain what you are looking for, it is either not there, or\n" +" you can add additional search criteria." +msgstr "" +"Voor ieder model zal een maximum van 1024 records worden gevonden. Als deze niet\n" +" bevatten waar je naar op zoek bent, dan is het gezochte niet aanwezig, of\n" +" het is mogelijk om extra zoek-criteria op te geven." + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_proxy +msgid "" +"Front end to show results of FT searches, together with rank and summary." +msgstr "" +"Hulpmodel om het resultaat van zoeken in tekst opdrachten weer te geven," +" samen met relevantie en samenvatting." + +#. module: fts_base +#: model:ir.actions.act_window,name:fts_base.action_fts_proxy_search +#: model:ir.ui.menu,name:fts_base.menu_fts_proxy_search +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_search +msgid "Fulltext search" +msgstr "Zoeken in tekst" + +#. module: fts_base +#: model_terms:ir.ui.view,arch_db:fts_base.fts_proxy_tree +msgid "Fulltext search - results" +msgstr "Zoeken in tekst - resultaten" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__id +msgid "ID" +msgstr "ID" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy____last_update +msgid "Last Modified on" +msgstr "Laatst gewijzigd op" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__write_uid +msgid "Last Updated by" +msgstr "Laatst gewijzigd door" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__write_date +msgid "Last Updated on" +msgstr "Laatst bijgewerkt op" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Note: to prevent unneeded searches, first enter all other criteria, like\n" +" models to search, or date, and only then the words to search for." +msgstr "" +"Opmerking: om onnodige zoekacties te voorkomen is het het beste eerste alle\n" +" andere criteria op te geven, zoals te doorzoeken modellen, of datum, en pas dan" +" de woorden om op te zoeken." + +#. module: fts_base +#: model:ir.actions.server,name:fts_base.action_open_document +msgid "Open Document" +msgstr "Open Document" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_query_helper +msgid "" +"Provides functions to assist in FT Search.\n" +"\n" +" Define as AbstractModel for easy extention and being able\n" +" to load this through the registry.\n" +" " +msgstr "" + +#. module: fts_base +#: model:ir.model,name:fts_base.model_fts_debug_helper +msgid "" +"Provides logging function to analyse FT problems.\n" +"\n" +" This class is needed to not flood the log with FTS debug\n" +" messages, but only log stuff when actually having an Odoo\n" +" session in debug mode.\n" +"\n" +" We will use info level messages, so this will also work\n" +" on production servers where normally you will not have\n" +" debug level logging.\n" +" " +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__rank +msgid "Rank" +msgstr "Relevantie" + +#. module: fts_base +#. odoo-python +#: code:addons/fts_base/wizards/fts_config.py:0 +#, python-format +msgid "Recreate search index" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_id +msgid "Resource ID" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_model +msgid "Resource Model" +msgstr "" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__res_name +msgid "Resource Name" +msgstr "Naam record" + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__searchstring +msgid "Searchstring" +msgstr "Zoektekst" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"Some text follwed by a \"*\" - will find all entries that have\n" +" a word starting with that text." +msgstr "" +"Enige tekst gevolgd door een \"*\" zal die records vinden die\n" +" een woord bevatten dat met deze tekst begint." + +#. module: fts_base +#: model:ir.model.fields,field_description:fts_base.field_fts_proxy__summary +msgid "Summary" +msgstr "Samenvatting" + +#. module: fts_base +#: model:ir.model.fields,help:fts_base.field_fts_proxy__res_model +msgid "The model this search works on. Required." +msgstr "Het model waarop gezocht wordt - verplicht." + +#. module: fts_base +#: model:ir.actions.act_window,name:fts_base.action_fts_proxy_search_with_summary +#: model:ir.ui.menu,name:fts_base.menu_fts_proxy_search_with_summary +msgid "With summary" +msgstr "Met samenvatting" + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "" +"You can select one or more of the configured data collections\n" +" (models), like emails or attachments. If you do not select any,\n" +" all data collections will be searched." +msgstr "" +"Je kunt een of meerdere van de geconfigureerde data verzamelingen selecteren\n" +" om te doorzoeken, zoals emails of documenten. Als je geen enkele dataverzameling" +" selecteert, dan zullen alle verzamelingen worden doorzocht. + +#. module: fts_base +#: model_terms:ir.actions.act_window,help:fts_base.action_fts_proxy_search +msgid "You can specify criteria this way:" +msgstr "Dit zijn de manieren om zoekcriteria op te geven:" diff --git a/fts_base/images/fulltextsearch-hover.png b/fts_base/images/fulltextsearch-hover.png new file mode 100644 index 0000000..d985a47 Binary files /dev/null and b/fts_base/images/fulltextsearch-hover.png differ diff --git a/fts_base/images/fulltextsearch.png b/fts_base/images/fulltextsearch.png new file mode 100644 index 0000000..08803e6 Binary files /dev/null and b/fts_base/images/fulltextsearch.png differ diff --git a/fts_base/models/__init__.py b/fts_base/models/__init__.py new file mode 100644 index 0000000..2b8e365 --- /dev/null +++ b/fts_base/models/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2012-2025 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +from . import fts_query_helper +from . import fts_debug_helper +from . import fts_mixin +from . import fts_content +from . import fts_proxy diff --git a/fts_base/models/fts_content.py b/fts_base/models/fts_content.py new file mode 100644 index 0000000..54a9bf1 --- /dev/null +++ b/fts_base/models/fts_content.py @@ -0,0 +1,37 @@ +# Copyright 2025 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +from odoo import api, fields, models +from odoo.tools.sql import SQL + +from ..tsvector_field import TSVector + + +class FtsContent(models.Model): + """For the moment used for testing. + + This model might become the central repository for all indexed + content. + """ + + _name = "fts.content" + _inherit = ["fts.mixin"] + _description = "Content used for testing Full Text Search" + _order = "date DESC" + _proxy_search_field = "content_tsvector" + + name = fields.Char("Resource Name", required=True) + date = fields.Date(required=True) + content = fields.Text("Content to index", required=True) + content_tsvector = TSVector( + indexed_columns=[ + "name", + "content", + ], + help="FT Search on content", + ) + + @api.model + def _proxy_search_select_expressions(self): + exprs = super()._proxy_search_select_expressions() + exprs["date"] = SQL("date") + return exprs diff --git a/fts_base/models/fts_debug_helper.py b/fts_base/models/fts_debug_helper.py new file mode 100644 index 0000000..30fc90e --- /dev/null +++ b/fts_base/models/fts_debug_helper.py @@ -0,0 +1,28 @@ +# Copyright 2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +import logging + +from odoo import models + +_logger = logging.getLogger(__name__) + + +class FtsDebugHelper(models.AbstractModel): + """Provides logging function to analyse FT problems. + + This class is needed to not flood the log with FTS debug + messages, but only log stuff when actually having an Odoo + session in debug mode. + + We will use info level messages, so this will also work + on production servers where normally you will not have + debug level logging. + """ + + _name = "fts.debug.helper" + _description = __doc__ + + def log_message(self, message, message_dict): + """Log message when odoo session in debug mode.""" + if self.env.user.has_groups("base.group_no_one"): + _logger.info(message, message_dict) diff --git a/fts_base/models/fts_mixin.py b/fts_base/models/fts_mixin.py new file mode 100644 index 0000000..cd3c2b9 --- /dev/null +++ b/fts_base/models/fts_mixin.py @@ -0,0 +1,213 @@ +# Copyright 2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +import logging + +from odoo import api, fields, models +from odoo.osv.expression import is_leaf +from odoo.tools.sql import SQL + +_logger = logging.getLogger(__name__) + + +class FtsMixin(models.AbstractModel): + """Add this to each model that needs FullText search functions.""" + + _name = "fts.mixin" + _description = __doc__ + + # For the moment we only support searching on one field per model to be + # included in the general search interface. The field here must be a + # ts_vector type field in this model. + _proxy_search_field = None + _title_column = "name" # Will be used to set res_name in fts.proxy + _extra_columns = [] # Add extra columns + + def _valid_field_parameter(self, field, name): + return name == "indexed_columns" or super()._valid_field_parameter(field, name) + + @api.model + def _search(self, domain, offset=0, limit=None, order=None): + """Searches on tsvector fields will be done with FT Search. + + In 18 _search always returns a Query object . + The result will be used to create proxy + records whose ids are returned by fts.proxy._search. + """ + # Split domain in normal parts and FT leaves. + query_helper = self.env["fts.query.helper"] + domain = self._replace_res_name(domain) + fulltext_leaves, patched_domain = query_helper.fts_patch_domain(self, domain) + if not fulltext_leaves: + return super()._search(domain, offset=offset, limit=limit, order=order) + return self._search_with_fulltext( + patched_domain, + fulltext_leaves, + offset=offset, + limit=limit, + order=order, + ) + + def _replace_res_name(self, domain): + """If records will be filtered on res_name, use actual name field.""" + new_domain = [] + for part in domain: + if is_leaf(part): + if part[0] == "res_name": + new_domain.append((self._title_column, part[1], part[2])) + continue + new_domain.append(part) + return new_domain + + def _search_with_fulltext( + self, patched_domain, fulltext_leaves, offset=0, limit=None, order=None + ): + """We now know we have to process the fulltext search.""" + # This is a Query object by default + query = super()._search(patched_domain, offset=offset, limit=limit, order=order) + self._fts_modify_query(query, fulltext_leaves) + select_sql = query.select() + self.env["fts.debug.helper"].log_message( + "SQL: %(sql)s, params=%(params)s", + { + "sql": select_sql.code, + "params": str(select_sql.params), + }, + ) + return query + + def _fts_modify_query(self, query, fulltext_leaves): + """Replace ORM-generated equality clauses with @@ FT-search operator. + + query._where_clauses holds SQL objects. We reconstruct + each clause by substituting the string representation, then re-wrap it + as a raw SQL object for postgress + """ + query_helper = self.env["fts.query.helper"] + new_clauses = [] + for clause in query._where_clauses: + patched_code = clause.code + patched_params = list(clause.params) + for table_name, field_name in fulltext_leaves: + patched_code, patched_params = query_helper.patch_where_clause_sql( + patched_code, patched_params, table_name, field_name + ) + # Re-wrap as a raw SQL object + new_clauses.append(SQL(patched_code, *patched_params)) + query._where_clauses = new_clauses + + @api.model + def _proxy_search_select_expressions(self): + """Return SQL expressions for the INSERT...SELECT columns. + + Subclasses can override individual expressions to customise what gets + inserted into fts_proxy without having to rewrite _proxy_search. + + Returns a dict with keys: + - res_name: expression for the display name column + - date: expression for the date column + - extra: expression for the extra column + All values must be SQL objects wrapping SQL-safe expressions. + """ + return { + # if title column is NULL or empty, use 'record '. + "res_name": SQL( + f"COALESCE(NULLIF({self._title_column}::text, ''), 'record ' || id::text)" + ), + "date": SQL("create_date::date"), + # Multiple _extra_columns are concatenated into a single text value. + "extra": SQL( + " || ' ' || ".join( + f"COALESCE({c}::text, '')" for c in self._extra_columns + ) + ) + if self._extra_columns + else SQL("NULL"), + } + + @api.model + def _proxy_search(self, domain, searchstring, limit=None, offset=0, **_kwargs): + """Search matching records and insert results into fts_proxy. + + Finds matching records via the GIN index and inserts them + into fts_proxy using a single INSERT...SELECT statement. + + Column expressions (res_name, date, extra) can be customised per model + by overriding _proxy_search_select_expressions(). + + In 18 _search no longer accepts a count parameter; counting is + handled by search_count at a higher level, so _proxy_search always + performs the INSERT and returns the list of inserted ids. + """ + with_summary = self.env.context.get("fts_summary", False) + search_limit = 80 if with_summary else 1024 + query = self._search(domain, limit=search_limit, offset=0) + # Execute the query to get matching ids. + self.env.cr.execute(query.select()) + res = [row[0] for row in self.env.cr.fetchall()] + if not res: + return res + query_helper = self.env["fts.query.helper"] + searchstring_parsed = query_helper.parse_searchstring(searchstring) + now = fields.Datetime.now() + uid = self.env.uid + indexed_columns = self._fields[ + self._proxy_search_field + ].get_indexed_columns_definition(self) + exprs = self._proxy_search_select_expressions() + # Flush pending ORM writes before raw SQL reads the source table, + # so that e.g. a write() before search sees up-to-date column values. + self.flush_model() + if with_summary: + # Wrap ts_headline in regexp_replace to normalise whitespace — + # multi-line content (HTML bodies, documents) would otherwise + # produce raw newlines in the UI. + summary_sql = SQL( + "regexp_replace(ts_headline('simple', %s," + " to_tsquery('simple', %s)," + " 'StartSel = *, StopSel = *'), '\\s+', ' ', 'g')", + SQL(indexed_columns), + searchstring_parsed, + ) + else: + summary_sql = SQL("NULL") + # Single INSERT...SELECT keeps all data inside Postgres. + # RETURNING id gives us exactly the ids inserted by this call, + # avoiding duplicates when searching across multiple models. + self.env.cr.execute( + SQL( + """ + INSERT INTO fts_proxy + (create_date, create_uid, write_date, write_uid, + res_model, res_id, rank, res_name, date, summary, extra) + SELECT + %s, %s, %s, %s, + %s, + id, + ts_rank(%s, to_tsquery('simple', %s)), + %s, + %s, + %s, + %s + FROM %s + WHERE id IN %s + RETURNING id + """, + now, + uid, + now, + uid, + self._name, + SQL(self._proxy_search_field), + searchstring_parsed, + exprs["res_name"], + exprs["date"], + summary_sql, + exprs["extra"], + SQL(self._table), + tuple(res), + ) + ) + inserted_ids = [row[0] for row in self.env.cr.fetchall()] + # Invalidate ORM cache so no stale fts.proxy records survive the INSERT. + self.env["fts.proxy"].invalidate_model() + return inserted_ids diff --git a/fts_base/models/fts_proxy.py b/fts_base/models/fts_proxy.py new file mode 100644 index 0000000..37636c7 --- /dev/null +++ b/fts_base/models/fts_proxy.py @@ -0,0 +1,174 @@ +# Copyright 2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +import logging + +from odoo import api, fields, models +from odoo.modules.registry import Registry +from odoo.osv.expression import TRUE_LEAF, is_leaf + +_logger = logging.getLogger(__name__) + + +class FtsProxy(models.TransientModel): + """Front end to show results of FT searches, together with rank and summary.""" + + _name = "fts.proxy" + _description = __doc__ + _rec_name = "res_name" + _order = "date DESC, rank DESC" + + def _search_searchstring(self, operator, value): + """Add searchstring to domain. Operator will be ignored.""" + return [("searchstring", "like", value)] + + res_name = fields.Char("Resource Name", readonly=True, required=True) + res_model = fields.Selection( + # Other supported models should register with _selection_add. + selection=[("fts.content", "Content Store")], + string="Resource Model", + readonly=True, + required=True, + help="The model this search works on. Required.", + ) + res_id = fields.Many2oneReference( + string="Resource ID", + model_field="res_model", + readonly=True, + required=True, + ) + rank = fields.Float(string="Rank", digits=(8, 4), compute=False, readonly=True) + summary = fields.Text("Summary", readonly=True) + date = fields.Date() + extra = fields.Char() + searchstring = fields.Char( + compute=lambda self: None, + search="_search_searchstring", + ) + + @api.model + def _search(self, domain, offset=0, limit=None, order=None): + """Searches in some or all models.""" + if not any( + is_leaf(part) and isinstance(part[0], str) and part[0] == "searchstring" + for part in domain + ): + return super()._search(domain, offset=offset, limit=limit, order=order) + self._delete_previous_search_results() + ( + searchstring, + new_domain, + model_objs, + ) = self._analyze_domain(domain) + # If no search criteria, return an empty query. + if not searchstring: + _logger.debug("doing nothing because I got no search string") + return super()._search( + [("id", "=", -1)], offset=offset, limit=limit, order=order + ) + if ( + not model_objs + ): # Should not happen, only when no additional module installed. + _logger.debug("doing nothing because I got no models to search") + return super()._search( + [("id", "=", -1)], offset=offset, limit=limit, order=order + ) + # For all models, populate fts_proxy via INSERT...SELECT. + for model_obj in model_objs: + model_obj._proxy_search( + new_domain, searchstring, limit=limit, offset=offset + ) + # Return ordered results for this user. + return super()._search( + [("create_uid", "=", self.env.uid)], + offset=offset, + limit=limit, + order=order, + ) + + def _delete_previous_search_results(self): + """Delete previous search results for this user.""" + # Use SQL as unlink does not delete all the records that need deleting. + with Registry(self.env.cr.dbname).cursor() as new_cursor: + new_cursor.execute( + "DELETE FROM fts_proxy WHERE create_uid = %s", (self.env.uid,) + ) + new_cursor.commit() + self.invalidate_model() # Clear all caches for this model. + + def _analyze_domain(self, domain): + """Get searchstring, modified domain, models used, fields used.""" + debug_helper = self.env["fts.debug.helper"] + searchstring = None + models = [] + query_fields = set() + new_domain = [] + for part in domain: + if is_leaf(part) and isinstance(part[0], str): + if part[0] == "searchstring": + searchstring = part[2] + elif part[0] == "res_model": + models.append(part[2]) + part = TRUE_LEAF # Replace with dummy. + elif part[0] == "date": + part[0] = "create_date" + elif part[0] == "res_name": + pass # Replacing res_name by actual field done later. + else: + # Add first (or only) part of fieldname to set. + query_fields.add(part[0].split(".")[0]) + new_domain.append(part) + model_objs = self._get_applicable_models(models, query_fields) + debug_helper.log_message( + "_analyze_domain returning domain %(domain)s," " models %(models)s", + { + "domain": str(new_domain), + "models": str([model_obj._name for model_obj in model_objs]), + }, + ) + return (searchstring, new_domain, model_objs) + + def _get_applicable_models(self, models, query_fields): + """Return selected or all models that contain the fields to query.""" + # If not models, search in all registered models (for all ts_vector fields) + models = models or [ + selection[0] for selection in self._fields["res_model"].selection + ] + # Remove model if query contains any field that is not in the model, + # or if the model does not define the main text search field. + model_objs = [] + for model in models: + model_obj = self.env[model] + if self._model_missing_field(model_obj, query_fields): + continue + if not model_obj._proxy_search_field: + _logger.debug( + "_proxy_search_field not set on model %(model)s", + {"model": model_obj._name}, + ) + continue + model_objs.append(model_obj) + return model_objs + + def _model_missing_field(self, model_obj, query_fields): + """If domain contains field not in model, ignore model.""" + for field_name in query_fields: + if field_name not in model_obj._fields: + _logger.debug( + "_field %(field_name)s not present on model %(model)s", + { + "model": model_obj._name, + "field_name": field_name, + }, + ) + return True + return False + + def action_open_document(self): + """Open related document.""" + self.ensure_one() + return { + "type": "ir.actions.act_window", + "res_model": self.res_model, + "view_mode": "form,list", + "res_id": self.res_id, + } diff --git a/fts_base/models/fts_query_helper.py b/fts_base/models/fts_query_helper.py new file mode 100644 index 0000000..8de3366 --- /dev/null +++ b/fts_base/models/fts_query_helper.py @@ -0,0 +1,124 @@ +# Copyright 2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +import shlex + +from odoo import _, models +from odoo.exceptions import UserError +from odoo.osv.expression import FALSE_LEAF, TRUE_LEAF, is_leaf + + +class FtsQueryHelper(models.AbstractModel): + """Provides functions to assist in FT Search. + + Define as AbstractModel for easy extention and being able + to load this through the registry. + """ + + _name = "fts.query.helper" + _description = __doc__ + + def parse_searchstring(self, searchstring): + """Convert entered searchstring to to_tsquery compatible format. + + This function does a lot of things webqsearch_to_tsquery does, + but that does not support substring searches (with 'sometext:*'). + + What the function does: + - Change quoted strings to <->: "bla text" -> bla<->text; + - And multiple words: bla text -> bla & text; + - Properly format substrings: bla* -> bla:*; + - Change tekstual operators into symbols: bla or text -> bla | text. + """ + parsed_string = " & ".join( + [ + "<->".join(part.split()) # Join adjacent words in quoted string. + # Splitting with shlex keeps quoted strings together. + for part in shlex.split( + searchstring.lower().replace(" and ", " & ").replace(" or ", " | ") + ) + if part != "&" # Will be added back later if needed. + ] + ) + # TODO: Find nicer solution with regular expressions... + parsed_string = ( + parsed_string.replace("*", ":*") + .replace("::", ":") + .replace(" & | & ", " | ") + ) + return parsed_string + + def patch_where_clause_sql(self, code, params, table_name, field_name): + """Replace an equals (=) selection with a FT search selection. + + Works on the raw SQL code string and params list extracted from an + Odoo 18 SQL object (where _where_clauses stores SQL instances). + + "".""::text = %s ==> + ""."" @@ to_tsquery('simple', %s) + + There might be a '::text' cast after the field name; both variants + are handled. + """ + needle_cast = f'"{table_name}"."{field_name}"::text = %s' + needle_plain = f'"{table_name}"."{field_name}" = %s' + replacement = ( + f'"{table_name}"."{field_name}" @@ to_tsquery(\'simple\', %s)' + ) + if needle_cast in code: + code = code.replace(needle_cast, replacement) + elif needle_plain in code: + code = code.replace(needle_plain, replacement) + return code, params + + def get_model_and_field(self, start_model, dotted_field_name): + """Get model and field for dotted_field_name.""" + DOT = "." + if DOT not in dotted_field_name: + return (start_model, dotted_field_name) + parts = dotted_field_name.split(DOT, 1) + field_name = parts[0] + if not start_model._fields[parts[0]].comodel_name: + raise UserError( + _( + "Field %(field_name)s in model %(model_name)s" + " is not an X2x field." + ) + % { + "field_name": field_name, + "model_name": start_model._name, + } + ) + next_model = self.env[start_model._fields[parts[0]].comodel_name] + next_field_name = parts[1] + return self.get_model_and_field(next_model, next_field_name) + + def fts_patch_domain(self, model, domain): + """Will find all tsvector fields in domain and replace searchstrings.""" + patched_domain = [] # Unfortunately we can not do in place substitutions. + fulltext_leaves = [] # Might contain duplicates, to unlikely to care. + for part in domain: + if (not is_leaf(part)) or part in (TRUE_LEAF, FALSE_LEAF): + patched_domain.append(part) # append as is + continue + actual_name = ( + model._proxy_search_field if part[0] == "searchstring" else part[0] + ) + field_model, field_name = self.get_model_and_field(model, actual_name) + field = field_model._fields[field_name] + if not field.column_type or field.column_type[0] != "tsvector": + patched_domain.append(part) # append as is + continue + patched_domain.append( + ( + actual_name, + "=", # Use '=' to prevent default search adding '%' characters. + self.parse_searchstring(part[2]), + ) + ) + fulltext_leaves.append( + ( + field_model._table, + field_name, + ) + ) + return fulltext_leaves, patched_domain diff --git a/fts_base/security/ir.model.access.csv b/fts_base/security/ir.model.access.csv new file mode 100644 index 0000000..5f63077 --- /dev/null +++ b/fts_base/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_fts_proxy,access_fts_proxy,model_fts_proxy,base.group_user,1,1,1,1 +access_fts_content,access_fts_content,model_fts_content,base.group_user,1,1,1,1 diff --git a/fts_base/static/src/img/icon.png b/fts_base/static/src/img/icon.png new file mode 100644 index 0000000..d985a47 Binary files /dev/null and b/fts_base/static/src/img/icon.png differ diff --git a/fts_base/static/src/js/fts_base.js b/fts_base/static/src/js/fts_base.js new file mode 100644 index 0000000..272218f --- /dev/null +++ b/fts_base/static/src/js/fts_base.js @@ -0,0 +1,64 @@ +/* -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2012 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################*/ + +openerp.fts_base = function(openerp) +{ + openerp.web.ListView.include( + { + init: function() + { + var result = this._super.apply(this, arguments); + if(this.model == 'fts.proxy') + { + this.options.selectable = false; + } + return result; + }, + select_record:function (index, view) + { + if(this.model == 'fts.proxy') + { + var self = this; + this.dataset.read_index( + ['model', 'res_id', 'name'], + {}) + .then(function(row) + { + self.do_action({ + type: 'ir.actions.act_window', + name: row.name, + res_model: row.model, + target: 'current', + res_id: row.res_id, + views: [[false, 'form']], + flags: { + 'initial_mode': 'view', + }, + }); + }); + } + else + { + return this._super.apply(this, arguments); + } + }, + }); +} diff --git a/fts_base/tests/__init__.py b/fts_base/tests/__init__.py new file mode 100644 index 0000000..6737f99 --- /dev/null +++ b/fts_base/tests/__init__.py @@ -0,0 +1,3 @@ +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +from . import test_fts_query_helper +from . import test_search diff --git a/fts_base/tests/test_fts_query_helper.py b/fts_base/tests/test_fts_query_helper.py new file mode 100644 index 0000000..2aad063 --- /dev/null +++ b/fts_base/tests/test_fts_query_helper.py @@ -0,0 +1,89 @@ +# Copyright 2024 Therp BV . +# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html +from odoo.tests import tagged +from odoo.tests.common import TransactionCase + + +@tagged("post_install", "-at_install") +class TestFtsQueryHelper(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.query_helper = cls.env["fts.query.helper"] + + def test_single_word(self): + parsed_string = self.query_helper.parse_searchstring("Rainbow") + # Simple word should only be converted to lowercase. + self.assertEqual(parsed_string, "rainbow") + + def test_multiple_words(self): + parsed_string = self.query_helper.parse_searchstring("Rainbow Warrior") + # Multiple words should be linked with '&'. + self.assertEqual(parsed_string, "rainbow & warrior") + + def test_quoted_strings(self): + parsed_string = self.query_helper.parse_searchstring("The Film 'Finding Nemo'") + # Words in quoted strings should be joined with '<->'. + self.assertEqual(parsed_string, "the & film & finding<->nemo") + + def test_substrings(self): + parsed_string = self.query_helper.parse_searchstring("Rainbow* Warrior:* Ship") + # Words in quoted strings should be joined with '<->'. + self.assertEqual(parsed_string, "rainbow:* & warrior:* & ship") + + def test_and_or_words(self): + parsed_string = self.query_helper.parse_searchstring( + "Rainbow or Warrior and Film & Afghanistan" + ) + # Multiple words should be linked with '&'. + self.assertEqual(parsed_string, "rainbow | warrior & film & afghanistan") + + def test_patch_where_clause(self): + # In Odoo 18, _where_clauses holds SQL objects; patch_where_clause_sql + # operates on the raw code string and params list extracted from them. + ORIGINAL_CODE = ( + """("ir_attachment"."res_field" IS NULL)""" + """ AND ("ir_attachment"."content_tsvector"::text """ + """= %s)""" + ) + MODIFIED_CODE = ( + """("ir_attachment"."res_field" IS NULL)""" # same + """ AND ("ir_attachment"."content_tsvector" """ # no more ::text + """@@ to_tsquery('simple', %s))""" # replaced + ) + params = ["search_term"] + modified_code, modified_params = self.query_helper.patch_where_clause_sql( + ORIGINAL_CODE, params, "ir_attachment", "content_tsvector" + ) + self.assertEqual(modified_code, MODIFIED_CODE) + self.assertEqual(modified_params, params) + # Replace should also succeed if it does not contain ::text + original_no_cast = ORIGINAL_CODE.replace("::text", "") + modified_code, _ = self.query_helper.patch_where_clause_sql( + original_no_cast, params, "ir_attachment", "content_tsvector" + ) + self.assertEqual(modified_code, MODIFIED_CODE) + + def test_get_model_and_field(self): + start_model = self.env["res.users"] + dotted_field_name = "partner_id.parent_id.company_id.name" + end_model, field_name = self.query_helper.get_model_and_field( + start_model, dotted_field_name + ) + self.assertEqual(end_model, self.env["res.company"]) + self.assertEqual(field_name, "name") + + def test_fts_patch_domain(self): + model = self.env["fts.content"] + domain = [ + ("date", ">=", "2001-01-01"), + "|", + ("name", "ilike", "Herman"), + ("content_tsvector", "=", "Piet or Jan"), + ] + fulltext_leaves, modified_domain = self.query_helper.fts_patch_domain( + model, domain + ) + self.assertEqual(fulltext_leaves, [("fts_content", "content_tsvector")]) + self.assertEqual(modified_domain[1], "|") + self.assertEqual(modified_domain[3], ("content_tsvector", "=", "piet | jan")) diff --git a/fts_base/tests/test_search.py b/fts_base/tests/test_search.py new file mode 100644 index 0000000..9e43a4f --- /dev/null +++ b/fts_base/tests/test_search.py @@ -0,0 +1,149 @@ +# Copyright 2025 Therp BV . +# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html +from odoo.tests import tagged +from odoo.tests.common import TransactionCase + +TEST_DATA = [ + ("Herman Gorter", "Een nieuwe lente, een nieuw geluid", "2024-12-12"), + ( + "Henriëtte Roland-Holst", + "De stilte der natuur heeft veel geluiden", + "2024-12-13", + ), + ("Nikolay Chernyshevsky", "The spark will ignite the flame!", "1997-06-13"), + ("PLSR motto", "Through struggle you will attain your rights!", "2025-01-23"), +] + + +@tagged("post_install", "-at_install") +class TestFtsQueryHelper(TransactionCase): + @classmethod + def _create_test_data(cls): + """Make sure we have data to test.""" + vals_list = [ + { + "name": tpl[0], + "content": tpl[1], + "date": tpl[2], + } + for tpl in TEST_DATA + ] + cls.content_model.create(vals_list) + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.content_model = cls.env["fts.content"] + cls._create_test_data() + + def test_search_basic(self): + """Test search on name and on content.""" + records = self.content_model.search([("content_tsvector", "like", "geluid")]) + self.assertTrue(records) + self.assertEqual(records[0].name, "Herman Gorter") + + def test_search_or(self): + """Test search that should find two records through or condition.""" + domain = [("content_tsvector", "like", "flame or rights")] + records = self.content_model.search(domain) + self.assertTrue(records) + for record in records: + self.assertIn(record.name, ("Nikolay Chernyshevsky", "PLSR motto")) + count = self.content_model.search_count(domain) + self.assertEqual(count, 2) + + def test_proxy_search(self): + proxy_model = self.env["fts.proxy"] + domain = [ + ("res_model", "=", "fts.content"), + ("searchstring", "like", "flame or rights"), + ] + records = proxy_model.search(domain, order="extra desc") + self.assertTrue(records) + for record in records: + self.assertIn(record.res_name, ("Nikolay Chernyshevsky", "PLSR motto")) + + def test_proxy_rec_name_search(self): + """Test combined search on text and rec_name.""" + proxy_model = self.env["fts.proxy"] + domain = [ + ("res_model", "=", "fts.content"), + ("searchstring", "like", "geluid*"), + ("res_name", "like", "Henriëtte Roland-Holst"), + ] + records = proxy_model.search(domain, order="extra desc") + self.assertTrue(records) + self.assertEqual(records[0].res_name, "Henriëtte Roland-Holst") + + def test_ordered_search(self): + proxy_model = self.env["fts.proxy"] + domain = [ + ("res_model", "=", "fts.content"), + ("searchstring", "like", "geluid or flame or rights"), + ] + records = proxy_model.search(domain, order="date desc") + self.assertTrue(records) + self.assertEqual(len(records), 3) + self.assertEqual(records[0].res_name, "PLSR motto") + self.assertEqual(records[1].res_name, "Herman Gorter") + self.assertEqual(records[2].res_name, "Nikolay Chernyshevsky") + + def test_wildcard_search(self): + """Test search on partial content.""" + # Should not find record without wildcard + records = self.content_model.search([("content_tsvector", "like", "strugg")]) + self.assertFalse(records) + records = self.content_model.search([("content_tsvector", "like", "strugg*")]) + self.assertTrue(records) + self.assertEqual(records[0].name, "PLSR motto") + + def test_proxy_search_single_insert(self): + """_proxy_search should use a single INSERT...SELECT, not batched ORM create(). + Verifies that: + 1. Results are correct (same records found as before). + 2. Exactly one INSERT statement is executed against fts_proxy. + 3. The date comes from the record's own date field, not create_date + (via _proxy_search_select_expressions override in FtsContent). + """ + proxy_model = self.env["fts.proxy"] + domain = [ + ("res_model", "=", "fts.content"), + ("searchstring", "like", "flame or rights"), + ] + insert_count = 0 + original_execute = self.env.cr.execute + + def counting_execute(query, params=None): + nonlocal insert_count + query_str = query.code if hasattr(query, "code") else query + if "INSERT INTO" in query_str and "fts_proxy" in query_str: + insert_count += 1 + return original_execute(query, params) + + self.env.cr.execute = counting_execute + records = proxy_model.search(domain) + self.env.cr.execute = original_execute + + # Correct results + self.assertTrue(records) + self.assertEqual(len(records), 2) + for record in records: + self.assertIn(record.res_name, ("Nikolay Chernyshevsky", "PLSR motto")) + # Single insert + self.assertEqual( + insert_count, + 1, + f"Expected 1 INSERT into fts_proxy, got {insert_count}. " + "The INSERT...SELECT optimisation may not be active.", + ) + + # Date comes from the record's own date field, not create_date + # (FtsContent overrides _proxy_search_select_expressions for this) + for record in records: + source = self.content_model.search([("name", "=", record.res_name)]) + self.assertEqual( + record.date, + source.date, + f"Proxy date for '{record.res_name}' should match the record's " + f"own date field, not create_date.", + ) diff --git a/fts_base/tsvector_field.py b/fts_base/tsvector_field.py new file mode 100644 index 0000000..932cdfa --- /dev/null +++ b/fts_base/tsvector_field.py @@ -0,0 +1,84 @@ +# Copyright 2024 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +import logging + +from psycopg2.extensions import AsIs + +from odoo.fields import Field + +_schema = logging.getLogger("odoo.schema") + +CREATE_TSVECTOR_COLUMN = ( + "ALTER TABLE %(table_name)s ADD COLUMN %(column_name)s tsvector" + " GENERATED ALWAYS AS (to_tsvector('simple', %(indexed_columns)s)) STORED" +) +CHECK_TSVECTOR_COLUMN = ( + "SELECT attgenerated FROM pg_attribute" + " WHERE attrelid = '%(table_name)s'::regclass AND attname = '%(column_name)s' " +) +DROP_TSVECTOR_COLUMN = "ALTER TABLE %(table_name)s DROP COLUMN %(column_name)s" +CREATE_TSVECTOR_INDEX = ( + "CREATE INDEX %(table_name)s_%(column_name)s_index" + " ON %(table_name)s USING GIN(%(column_name)s)" +) + + +class TSVector(Field): + """Define field of tsvector type.""" + + type = "tsvector" + column_type = ("tsvector", "tsvector") + readonly = True + copy = False + + def update_db_column(self, model, column): + """Create/update the column corresponding to ``self``. + + :param model: an instance of the field's model + :param column: the column's configuration (dict) if it exists, or ``None`` + """ + param_dict = { + "table_name": AsIs(model._table), + "column_name": AsIs(self.name), + "indexed_columns": AsIs(self.get_indexed_columns_definition(model)), + } + if column and column["udt_name"] == "tsvector": + # When column created together with table, it will miss the + # GENERATED AS... part. In that case, also need to drop the + # column and create it afresh. + model._cr.execute(CHECK_TSVECTOR_COLUMN, param_dict) + attgenerated = model._cr.fetchone() if model._cr.rowcount else "" + if attgenerated and attgenerated[0] == "s": + # Column is generated, assume with the right definition. + return + if column: # Column exist, but is of wrong type. + # Drop column to recreate it. + param_dict["udt_name"] = column["udt_name"] + model._cr.execute(DROP_TSVECTOR_COLUMN, param_dict) + _schema.debug( + "Table %(table_name)s:" + " dropped column %(column_name)s of type %(udt_name)s", + param_dict, + ) + column = None + if not column: + # the column does not exist, create it + model._cr.execute(CREATE_TSVECTOR_COLUMN, param_dict) + _schema.debug( + "Table %(table_name)s: added column %(column_name)s of type tsvector", + param_dict, + ) + # Now create index on column. + model._cr.execute(CREATE_TSVECTOR_INDEX, param_dict) + + def get_indexed_columns_definition(self, model): + """Get formula to generate content for indexed columns""" + indexed_columns = self.indexed_columns + if isinstance(indexed_columns, str): + if hasattr(model, indexed_columns): + return getattr(model, indexed_columns)() # Call model method. + return "COALESCE(%s, '')" % indexed_columns # Single column + # We need the COALESCE to prevent problems with NULL values in indexed fields. + return " || ' ' || ".join( + ["COALESCE(%s, '')" % fieldname for fieldname in indexed_columns] + ) diff --git a/fts_base/views/fts_proxy.xml b/fts_base/views/fts_proxy.xml new file mode 100644 index 0000000..3fc969e --- /dev/null +++ b/fts_base/views/fts_proxy.xml @@ -0,0 +1,110 @@ + + + + fts.proxy.search + fts.proxy + + + + + + + + + + + + fts.proxy.tree + fts.proxy + + +