Hide topics with 2 or fewer sources from A-Z, category browse, and Google [sc-45176] (#3496)

* feat(topics): hide low-source topics from A-Z, category browse, and Google search [sc-45176]

Topics with 2 or fewer sources are no longer surfaced on the Topic A-Z page,
category/TOC browse cards, or Google (excluded from the sitemap and marked
noindex on their own page). They remain fully intact in the database and
still appear in other topics' "Related Topics" sidebars.

should_display() gains an optional min_sources param (default preserves
existing behavior) so the stricter threshold can be applied only at the
three affected call sites, leaving the sidebar's annotate_topic_link() path
untouched.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* test(topics): parametrize should_display tests

Collapses four repetitive test methods into a single pytest.mark.parametrize
covering the same cases (default vs raised threshold, curation exemptions,
explicit shouldDisplay=False override).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: nsantacruz

reader/views.py

@@ -25,7 +25,7 @@
 from remote_config.keys import CHATBOT_MAX_INPUT_CHARS, CHATBOT_MAX_PROMPTS, CHATBOT_PROMO_LEARN_MORE_URLS, CHATBOT_PROMO_MAYBE_LATER_JSON, SHOW_JOIN_CHATBOT_BANNER, CHATBOT_PROMO_SESSION_LENGTH_SECONDS
 from sefaria.system.context_processors import _is_user_in_experiment
 from sefaria.utils.util import get_redirect_to_help_center
-from sefaria.constants.model import LIBRARY_MODULE, VOICES_MODULE
+from sefaria.constants.model import LIBRARY_MODULE, VOICES_MODULE, MIN_SOURCES_FOR_TOPIC_DISPLAY
 from rest_framework.decorators import api_view, permission_classes
 from rest_framework.permissions import IsAuthenticated
 from django.template.loader import render_to_string
@@ -3519,6 +3519,7 @@ def topic_page(request, slug, test_version=None):
     return render_template(request, 'base.html', props, {
         "title":          title,
         "desc":           desc,
+        "noindex":        not topic_obj.should_display(min_sources=MIN_SOURCES_FOR_TOPIC_DISPLAY),
     })
 
 @catch_error_as_json
@@ -3528,7 +3529,7 @@ def topics_list_api(request):
     """
     limit = int(request.GET.get("limit", 1000))
     minify = bool(int(request.GET.get("minify", 1)))
-    all_topics = get_all_topics(limit, active_module=request.active_module)
+    all_topics = get_all_topics(limit, active_module=request.active_module, min_sources=MIN_SOURCES_FOR_TOPIC_DISPLAY)
     all_topics_json = []
     for topic in all_topics:
         topic_json = topic.contents(minify=minify, with_html=True)

sefaria/constants/model.py

@@ -33,4 +33,7 @@
 
 # Module constants that correspond to DOMAIN_MODULES keys
 LIBRARY_MODULE = "library"
-VOICES_MODULE = "voices"
+VOICES_MODULE = "voices"
+
+# Topics with fewer sources than this are hidden from the A-Z listing, category browse pages, and Google search
+MIN_SOURCES_FOR_TOPIC_DISPLAY = 3

sefaria/helper/tests/topic_test.py

@@ -3,6 +3,7 @@
 from sefaria.model.text import library
 from sefaria.model.place import Place
 from sefaria.helper import topic
+from sefaria.conftest import mock_topics_pool
 
 
 @pytest.fixture(autouse=True, scope='module') 
@@ -215,6 +216,50 @@ def test_update_topic(some_topic):
 		topic.update_topic(some_topic, slug='abc')
 
 
+@pytest.fixture(scope='module')
+def topics_with_varying_source_counts(django_db_setup, django_db_blocker):
+	# Topic.get_pools() is globally mocked in sefaria/conftest.py to read from mock_topics_pool
+	# rather than the real DjangoTopic pool tables, so pool membership for test topics is granted
+	# the same way: by adding their slugs to that dict.
+	with django_db_blocker.unblock():
+		specs = [
+			('zero', 0, {}),
+			('two', 2, {}),
+			('three', 3, {}),
+			('curated', 1, {"description": {"en": "A hand-written description"}}),
+		]
+		by_suffix = {}
+		for suffix, num_sources, extra_attrs in specs:
+			t = Topic({'slug': "", "numSources": num_sources, **extra_attrs})
+			title = f"get_all_topics test {suffix}"
+			t.add_primary_titles(title, title[::-1])
+			t.set_slug_to_primary_title()
+			t.save()
+			mock_topics_pool[t.slug] = ['library']
+			by_suffix[suffix] = t
+		yield by_suffix
+		for t in by_suffix.values():
+			mock_topics_pool.pop(t.slug, None)
+			t.delete()
+
+
+def test_get_all_topics_min_sources(topics_with_varying_source_counts):
+	from django.core.cache import caches
+	caches['default'].clear()  # get_all_topics is cached for 24h; bust it so fixture topics are picked up
+
+	by_suffix = topics_with_varying_source_counts
+	relevant_slugs = {t.slug for t in by_suffix.values()}
+
+	default_result = {t.slug for t in topic.get_all_topics(limit=0) if t.slug in relevant_slugs}
+	# unchanged legacy behavior: raw query only requires numSources > 0, no min_sources filtering applied
+	assert default_result == {by_suffix['two'].slug, by_suffix['three'].slug, by_suffix['curated'].slug}
+
+	strict_result = {t.slug for t in topic.get_all_topics(limit=0, min_sources=3) if t.slug in relevant_slugs}
+	# 'two' has only 2 sources and no exemption, so it's hidden; 'curated' has 1 source but a
+	# published description, so the curation exemption keeps it visible; 'three' clears the bar outright
+	assert strict_result == {by_suffix['three'].slug, by_suffix['curated'].slug}
+
+
 def test_get_topic_omits_orphaned_ref_links():
 	"""A RefTopicLink whose text was deleted from the library (its ref no longer
 	resolves) must be omitted from the get_topic response. Otherwise the orphaned source

sefaria/helper/topic.py

@@ -256,14 +256,17 @@ def annotate_topic_link(link: dict, link_topic_dict: dict) -> Union[dict, None]:
 
 
 @django_cache(timeout=24 * 60 * 60)
-def get_all_topics(limit=1000, displayableOnly=True, active_module=LIBRARY_MODULE):
+def get_all_topics(limit=1000, displayableOnly=True, active_module=LIBRARY_MODULE, min_sources=None):
     query = {"shouldDisplay": {"$ne": False}, "numSources": {"$gt": 0}} if displayableOnly else {}
     topic_list = TopicSet(query, limit=limit, sort=[('numSources', -1)])
-    
+
     # Get the actual pool name that should be used for this active_module
     expected_pool_name = get_topic_pool_name_for_module(active_module)
-    
-    return [t for t in topic_list if expected_pool_name in t.get_pools()]
+
+    topics = [t for t in topic_list if expected_pool_name in t.get_pools()]
+    if min_sources is not None:
+        topics = [t for t in topics if t.should_display(min_sources=min_sources)]
+    return topics
 
 @django_cache(timeout=24 * 60 * 60)
 def get_num_library_topics():

sefaria/model/tests/topic_test.py

@@ -209,6 +209,36 @@ def test_sanitize(self):
         assert "<script>" not in t.description["he"]
         assert "<script>" not in t.slug
 
+    @pytest.mark.parametrize(('num_sources', 'min_sources', 'description', 'data_source', 'should_display_flag', 'expected'), [
+        (1, 1, None, None, None, True),                                   # default threshold matches legacy '> 0' behavior
+        (0, 1, None, None, None, False),                                  # default threshold matches legacy '> 0' behavior
+        (2, 1, None, None, None, True),                                   # below the raised threshold, but default threshold still passes
+        (2, 3, None, None, None, False),                                  # raised threshold hides low-source topics
+        (3, 3, None, None, None, True),                                   # raised threshold: topic clears the bar
+        (1, 3, {"en": "A hand-written description"}, None, None, True),   # curation exemption: published description
+        (0, 3, None, "sefaria", None, True),                              # curation exemption: manually-curated topic
+        (10, 3, None, None, False, False),                                # explicit shouldDisplay=False always wins
+    ], ids=[
+        'default-threshold-shown',
+        'default-threshold-hidden',
+        'below-raised-threshold-shown-at-default',
+        'below-raised-threshold-hidden',
+        'meets-raised-threshold-shown',
+        'description-exemption',
+        'data-source-exemption',
+        'explicit-false-overrides',
+    ])
+    def test_should_display(self, num_sources, min_sources, description, data_source, should_display_flag, expected):
+        t = Topic()
+        t.numSources = num_sources
+        if description is not None:
+            t.description = description
+        if data_source is not None:
+            t.data_source = data_source
+        if should_display_flag is not None:
+            t.shouldDisplay = should_display_flag
+        assert t.should_display(min_sources=min_sources) is expected
+
 
 class TestTopicLinkHelper(object):
 

sefaria/model/text.py

@@ -5230,7 +5230,7 @@ def get_topic_toc_json_recursive(self, topic=None, explored=None, with_descripti
             topic_json = {}
         else:
             children = [] if topic.slug in explored else [l.fromTopic for l in IntraTopicLinkSet({"linkType": "displays-under", "toTopic": topic.slug})]
-            topic_json = topic.contents(minify=True, children=children, with_html=True)
+            topic_json = topic.contents(minify=True, children=children, with_html=True, min_sources_for_display=constants.MIN_SOURCES_FOR_TOPIC_DISPLAY)
             unexplored_top_level = getattr(topic, "isTopLevelDisplay", False) and getattr(topic, "slug",
                                                                                           None) not in explored
             explored.add(topic.slug)

sefaria/model/topic.py

@@ -364,8 +364,8 @@ def has_types(self, search_slug_set) -> bool:
         types = self.get_types(search_slug_set=search_slug_set)
         return len(search_slug_set.intersection(types)) > 0
 
-    def should_display(self) -> bool:
-        return getattr(self, 'shouldDisplay', True) and (getattr(self, 'numSources', 0) > 0 or self.has_description() or getattr(self, "data_source", "") == "sefaria")
+    def should_display(self, min_sources: int = 1) -> bool:
+        return getattr(self, 'shouldDisplay', True) and (getattr(self, 'numSources', 0) >= min_sources or self.has_description() or getattr(self, "data_source", "") == "sefaria")
 
     def has_description(self) -> bool:
         """
@@ -516,9 +516,10 @@ def contents(self, **kwargs):
             d.update(super(Topic, self).contents(**kwargs))
         else:
             children = kwargs.get("children", [])
+            min_sources = kwargs.get("min_sources_for_display", 1)
             d.update({
                 'slug': self.slug,
-                "shouldDisplay": True if len(children) > 0 else self.should_display(),
+                "shouldDisplay": True if len(children) > 0 else self.should_display(min_sources=min_sources),
                 "displayOrder": getattr(self, "displayOrder", 10000),
                 "pools": self.get_pools()})
             if getattr(self, "categoryDescription", False):

sefaria/sitemap.py

@@ -8,6 +8,7 @@
 
 from sefaria.model import *
 from sefaria.system.database import db
+from sefaria.constants.model import MIN_SOURCES_FOR_TOPIC_DISPLAY
 from .settings import STATICFILES_DIRS, STATIC_URL
 
 
@@ -133,7 +134,7 @@ def generate_topics_sitemap(self):
         Creates a sitemap for each topic that has at least one source or source sheet.
         """
         topics = TopicSet()
-        topics = [topic for topic in topics if topic.should_display()]
+        topics = [topic for topic in topics if topic.should_display(min_sources=MIN_SOURCES_FOR_TOPIC_DISPLAY)]
         urls = [self._hostname + "/topics/" + topic.slug for topic in topics]
         self.write_urls(urls, "topics-sitemap.xml")