From 62ec0caf8f3d2957d6b4c0809727331633fedd65 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 18:14:15 +0300 Subject: [PATCH 001/125] feat(Topic): add pools to optional_attrs. --- sefaria/model/topic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index a5dfe56568..48bbfd52f2 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -49,6 +49,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): "data_source", #any topic edited manually should display automatically in the TOC and this flag ensures this 'image', "portal_slug", # slug to relevant Portal object + 'pools', # list of strings, any of them represents a pool that this topic is member of ] attr_schemas = { From 849cb8ae483724f54027834dbfc7d3d75689bd83 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 18:18:32 +0300 Subject: [PATCH 002/125] feat(Topic): normalize pools - remove duplicates, sort, and set to an empty array if missing. --- sefaria/model/topic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 48bbfd52f2..0d796c7121 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -115,6 +115,7 @@ def _normalize(self): displays_under_link = IntraTopicLink().load({"fromTopic": slug, "linkType": "displays-under"}) if getattr(displays_under_link, "toTopic", "") == "authors": self.subclass = "author" + self.pools = sorted(set(getattr(self, 'pools', []))) def _sanitize(self): super()._sanitize() From 2c42b7680183ab3a50834b275bd3f9d50234e5bf Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 18:22:37 +0300 Subject: [PATCH 003/125] feat(Topic): function for adding pool. --- sefaria/model/topic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 0d796c7121..03868e9b95 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -427,6 +427,10 @@ def __str__(self): def __repr__(self): return "{}.init('{}')".format(self.__class__.__name__, self.slug) + def add_pool(self, pool_name): + if pool_name not in self.pools: + self.pools.append(pool_name) + self.save() class PersonTopic(Topic): """ From eec5b9ac0e67fa8e58ed253e7e83792a258d2218 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 18:31:18 +0300 Subject: [PATCH 004/125] feat(Topic): add optional_pools as class attribute. validate pools are in optional_pools. --- sefaria/model/topic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 03868e9b95..78d06a4acf 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -52,6 +52,8 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] + optional_pools = {'sheets', 'textual', 'torahtab'} + attr_schemas = { "image": { "image_uri": { @@ -104,6 +106,7 @@ def _validate(self): if getattr(self, "image", False): img_url = self.image.get("image_uri") if img_url: validate_url(img_url) + assert all(pool in self.optional_pools for pool in self.pools), f'Pools {[pool for pool in self.pools if pool not in self.optional_pools]} is not an optional pool' def _normalize(self): super()._normalize() @@ -432,6 +435,8 @@ def add_pool(self, pool_name): self.pools.append(pool_name) self.save() + def update_sheets_pool(self): + class PersonTopic(Topic): """ Represents a topic which is a person. Not necessarily an author of a book. From e63842e13cd63740049ad5a52ec8fee915a30ff0 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 18:51:45 +0300 Subject: [PATCH 005/125] feat(Topic): add function for updating 'sheets' in pool, in accordance with refLinks to sheet existence. --- sefaria/model/topic.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 78d06a4acf..2833ab8f4f 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -367,6 +367,10 @@ def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): kwargs['record_kwargs'] = {'context_slug': self.slug} return TopicLinkSetHelper.find(intra_link_query, **kwargs) + def get_sheets_links(self, query_kwargs: dict = None, **kwargs): + query_kwargs['is_sheet'] = True + return self.link_set('refTopic', query_kwargs, **kwargs) + def contents(self, **kwargs): mini = kwargs.get('minify', False) d = {'slug': self.slug} if mini else super(Topic, self).contents(**kwargs) @@ -436,6 +440,11 @@ def add_pool(self, pool_name): self.save() def update_sheets_pool(self): + sheets_links = self.get_sheets_links() + if bool(sheets_links) != 'sheets' in self.pools: + self.pools.remove('sheets') if 'sheets' in self.pools else self.pools.append('sheets') + self.save() + class PersonTopic(Topic): """ From 3c377cfa15f94b599918833e468d61f1b92b1df7 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 19:21:37 +0300 Subject: [PATCH 006/125] refactor(Topic): remove magic string from update_sheets_pool. --- sefaria/model/topic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 2833ab8f4f..b8b28e2aa5 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -440,9 +440,10 @@ def add_pool(self, pool_name): self.save() def update_sheets_pool(self): + pool = 'sheets' sheets_links = self.get_sheets_links() - if bool(sheets_links) != 'sheets' in self.pools: - self.pools.remove('sheets') if 'sheets' in self.pools else self.pools.append('sheets') + if bool(sheets_links) != pool in self.pools: + self.pools.remove(pool) if pool in self.pools else self.pools.append(pool) self.save() From 91d9e6c6d0e8e209b4e6f493d6d13e23747ced19 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 19:30:17 +0300 Subject: [PATCH 007/125] refactor(Topic): use pools['torahtab'] rather than good_to_promote'. --- sefaria/helper/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index ae3337d0c7..3119d0ff2a 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -281,7 +281,7 @@ def curated_primacy(order_dict, lang): def get_random_topic(good_to_promote=True) -> Optional[Topic]: - query = {"good_to_promote": True} if good_to_promote else {} + query = {"pools": 'sheets'} if good_to_promote else {} random_topic_dict = list(db.topics.aggregate([ {"$match": query}, {"$sample": {"size": 1}} From c24313d17b86096f86eabde465549a87c4e2a112 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 3 Jul 2024 19:43:14 +0300 Subject: [PATCH 008/125] doc(Topic): remove good_to_promote from openAPI.jsom. --- docs/openAPI.json | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/openAPI.json b/docs/openAPI.json index fe97c22686..62c5aa4d1e 100644 --- a/docs/openAPI.json +++ b/docs/openAPI.json @@ -4187,7 +4187,6 @@ }, "categoryDescription": {}, "numSources": 1767, - "good_to_promote": true, "description_published": true, "data_source": "sefaria", "primaryTitle": { @@ -4591,7 +4590,6 @@ "isTopLevelDisplay": true, "displayOrder": 30, "numSources": 2937, - "good_to_promote": true, "primaryTitle": { "en": "Prayer", "he": "תפילה" @@ -4623,7 +4621,6 @@ "_temp_id": "תורה" }, "numSources": 2333, - "good_to_promote": true, "primaryTitle": { "en": "Torah", "he": "תורה" @@ -4766,7 +4763,6 @@ }, "categoryDescription": {}, "numSources": 1967, - "good_to_promote": true, "description_published": true, "data_source": "sefaria", "primaryTitle": { @@ -4869,7 +4865,6 @@ "categoryDescription": {}, "displayOrder": 0, "numSources": 1662, - "good_to_promote": true, "description_published": true, "data_source": "sefaria", "image": { @@ -5162,7 +5157,6 @@ } }, "numSources": 7, - "good_to_promote": true, "primaryTitle": { "en": "Metushelach", "he": "מתושלח" @@ -9761,7 +9755,6 @@ "categoryDescription": {}, "displayOrder": 0, "numSources": 1662, - "good_to_promote": true, "description_published": true, "data_source": "sefaria", "image": { @@ -10071,9 +10064,6 @@ "format": "int32", "type": "integer" }, - "good_to_promote": { - "type": "boolean" - }, "description_published": { "type": "boolean" }, @@ -10089,7 +10079,7 @@ } } }, - "example": "{\n\"slug\": \"metushelach\",\n\"titles\": [\n{\n\"text\": \"Metushelach\",\n\"lang\": \"en\",\n\"primary\": true,\n\"transliteration\": true\n},\n{\n\"text\": \"מתושלח\",\n\"lang\": \"he\",\n\"primary\": true\n},\n{\n\"text\": \"Methuselah\",\n\"lang\": \"en\"\n},\n{\n\"text\": \"Methushelach\",\n\"lang\": \"en\"\n}\n],\n\"subclass\": \"person\",\n\"alt_ids\": {\n\"_temp_id\": \"מתושלח\",\n\"wikidata\": \"Q156290\"\n},\n\"properties\": {\n\"enWikiLink\": {\n\"value\": \"https://en.wikipedia.org/wiki/Methuselah\",\n\"dataSource\": \"wikidata\"\n},\n\"heWikiLink\": {\n\"value\": \"https://he.wikipedia.org/wiki/מתושלח\",\n\"dataSource\": \"wikidata\"\n},\n\"deWikiLink\": {\n\"value\": \"https://de.wikipedia.org/wiki/Methusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"esWikiLink\": {\n\"value\": \"https://es.wikipedia.org/wiki/Matusalén\",\n\"dataSource\": \"wikidata\"\n},\n\"frWikiLink\": {\n\"value\": \"https://fr.wikipedia.org/wiki/Mathusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"ruWikiLink\": {\n\"value\": \"https://ru.wikipedia.org/wiki/Мафусал_(потомок_Сифа)\",\n\"dataSource\": \"wikidata\"\n}\n},\n\"numSources\": 7,\n\"good_to_promote\": true,\n\"primaryTitle\": {\n\"en\": \"Metushelach\",\n\"he\": \"מתושלח\"\n}\n}" + "example": "{\n\"slug\": \"metushelach\",\n\"titles\": [\n{\n\"text\": \"Metushelach\",\n\"lang\": \"en\",\n\"primary\": true,\n\"transliteration\": true\n},\n{\n\"text\": \"מתושלח\",\n\"lang\": \"he\",\n\"primary\": true\n},\n{\n\"text\": \"Methuselah\",\n\"lang\": \"en\"\n},\n{\n\"text\": \"Methushelach\",\n\"lang\": \"en\"\n}\n],\n\"subclass\": \"person\",\n\"alt_ids\": {\n\"_temp_id\": \"מתושלח\",\n\"wikidata\": \"Q156290\"\n},\n\"properties\": {\n\"enWikiLink\": {\n\"value\": \"https://en.wikipedia.org/wiki/Methuselah\",\n\"dataSource\": \"wikidata\"\n},\n\"heWikiLink\": {\n\"value\": \"https://he.wikipedia.org/wiki/מתושלח\",\n\"dataSource\": \"wikidata\"\n},\n\"deWikiLink\": {\n\"value\": \"https://de.wikipedia.org/wiki/Methusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"esWikiLink\": {\n\"value\": \"https://es.wikipedia.org/wiki/Matusalén\",\n\"dataSource\": \"wikidata\"\n},\n\"frWikiLink\": {\n\"value\": \"https://fr.wikipedia.org/wiki/Mathusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"ruWikiLink\": {\n\"value\": \"https://ru.wikipedia.org/wiki/Мафусал_(потомок_Сифа)\",\n\"dataSource\": \"wikidata\"\n}\n},\n\"numSources\": 7,\n\"primaryTitle\": {\n\"en\": \"Metushelach\",\n\"he\": \"מתושלח\"\n}\n}" }, "url": { "description": "The `Ref` in a format appropriate for a URL, with spaces replaced with `.` etc. ", @@ -10135,7 +10125,6 @@ }, "categoryDescription": {}, "numSources": 217, - "good_to_promote": true, "description_published": true, "primaryTitle": { "en": "Hillel", @@ -10170,10 +10159,6 @@ "description": "A description of the category of this topic", "type": "string" }, - "good_to_promote": { - "description": "A topic which will be included in our results from the `random-by-topic` endpoint. ", - "type": "boolean" - }, "numSources": { "description": "The number of text sources associated with a topic. ", "type": "integer", @@ -10222,7 +10207,6 @@ }, "categoryDescription": {}, "numSources": 120, - "good_to_promote": true, "description_published": true, "data_source": "sefaria", "primaryTitle": { From 796661db79067a9e7c9ab2919f15e5d9633464c8 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Thu, 4 Jul 2024 11:59:34 +0300 Subject: [PATCH 009/125] feat(Topic): remove pools from contents() as default. --- sefaria/model/topic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index b8b28e2aa5..ea6331995c 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -374,6 +374,8 @@ def get_sheets_links(self, query_kwargs: dict = None, **kwargs): def contents(self, **kwargs): mini = kwargs.get('minify', False) d = {'slug': self.slug} if mini else super(Topic, self).contents(**kwargs) + if kwargs.get('remove_pools', True): + d.pop('pools', None) d['primaryTitle'] = {} for lang in ('en', 'he'): d['primaryTitle'][lang] = self.get_primary_title(lang=lang, with_disambiguation=kwargs.get('with_disambiguation', True)) From 604bca7939b09d0e378e1a6f0bee5ebd99f99b05 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Thu, 4 Jul 2024 12:48:12 +0300 Subject: [PATCH 010/125] refactor(Topic): change get_random_topic param good_to_promote to a param pool that defaults to 'torahtab'. --- reader/views.py | 2 +- sefaria/helper/topic.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reader/views.py b/reader/views.py index ef8682dd38..e0fbebf308 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4195,7 +4195,7 @@ def random_by_topic_api(request): Returns Texts API data for a random text taken from popular topic tags """ cb = request.GET.get("callback", None) - random_topic = get_random_topic(good_to_promote=True) + random_topic = get_random_topic() if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index 3119d0ff2a..43a1847c3d 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -280,8 +280,8 @@ def curated_primacy(order_dict, lang): return (bord.get('numDatasource', 0) * bord.get('tfidf', 0)) - (aord.get('numDatasource', 0) * aord.get('tfidf', 0)) -def get_random_topic(good_to_promote=True) -> Optional[Topic]: - query = {"pools": 'sheets'} if good_to_promote else {} +def get_random_topic(pool='torahtab') -> Optional[Topic]: + query = {"pools": pool} if pool else {} random_topic_dict = list(db.topics.aggregate([ {"$match": query}, {"$sample": {"size": 1}} From f90e5ad175dfa33f5b5be6a7e7b724b818a4636c Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 8 Jul 2024 08:53:06 +0300 Subject: [PATCH 011/125] refactor(Topic): add pool to Topic when saving RefTopicLink. --- sefaria/model/topic.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index ea6331995c..11f5b4164c 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -837,6 +837,17 @@ def set_description(self, lang, title, prompt): self.descriptions = d return self + def get_pool(self): + return 'sheets' if self.is_sheet else 'textual' + + def get_topic(self): + return Topic().load({'slug': self.toTopic}) + + def save(self, override_dependencies=False): + super(RefTopicLink, self).save() + topic = self.get_topic() + topic.add_pool(self.get_pool()) + def _sanitize(self): super()._sanitize() for lang, d in getattr(self, "descriptions", {}).items(): From 12569a528e569808331b3d6311a3c12662136019 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 8 Jul 2024 09:05:32 +0300 Subject: [PATCH 012/125] refactor(Topic): refactor the functions for getting sheet links and removeing sheets pool, to apply also for textual links and pool. --- sefaria/model/topic.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 11f5b4164c..ca8f747dd8 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -367,8 +367,8 @@ def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): kwargs['record_kwargs'] = {'context_slug': self.slug} return TopicLinkSetHelper.find(intra_link_query, **kwargs) - def get_sheets_links(self, query_kwargs: dict = None, **kwargs): - query_kwargs['is_sheet'] = True + def get_ref_links(self, is_sheet, query_kwargs: dict = None, **kwargs): + query_kwargs['is_sheet'] = is_sheet return self.link_set('refTopic', query_kwargs, **kwargs) def contents(self, **kwargs): @@ -441,10 +441,13 @@ def add_pool(self, pool_name): self.pools.append(pool_name) self.save() - def update_sheets_pool(self): - pool = 'sheets' - sheets_links = self.get_sheets_links() - if bool(sheets_links) != pool in self.pools: + def update_pool_by_links(self, pool): + """ + updating the pools 'sheets' or 'textual' according to the existence of links + :param pool: 'sheets' or 'textual' + """ + links = self.get_ref_links(pool == 'sheets') + if bool(links) != pool in self.pools: self.pools.remove(pool) if pool in self.pools else self.pools.append(pool) self.save() From 454375aeb0e29c9bb0eb1f1b74b6170258d13f0f Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 8 Jul 2024 09:12:45 +0300 Subject: [PATCH 013/125] feat(Topic): update pool when deleting a RefTopicLink. --- sefaria/model/topic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index ca8f747dd8..b0ea2e4156 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -851,6 +851,13 @@ def save(self, override_dependencies=False): topic = self.get_topic() topic.add_pool(self.get_pool()) + def delete(self, force=False, override_dependencies=False): + topic = self.get_topic() + pool = self.get_pool() + super(RefTopicLink, self).delete() + if topic: + topic.update_pool_by_links(pool) + def _sanitize(self): super()._sanitize() for lang, d in getattr(self, "descriptions", {}).items(): From 10ed04218ff509e32578ae3aaa8358bcb6b57e25 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 8 Jul 2024 09:14:28 +0300 Subject: [PATCH 014/125] chore(Topic): add params to the supers. --- sefaria/model/topic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index b0ea2e4156..5a61f359ff 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -847,14 +847,14 @@ def get_topic(self): return Topic().load({'slug': self.toTopic}) def save(self, override_dependencies=False): - super(RefTopicLink, self).save() + super(RefTopicLink, self).save(override_dependencies) topic = self.get_topic() topic.add_pool(self.get_pool()) def delete(self, force=False, override_dependencies=False): topic = self.get_topic() pool = self.get_pool() - super(RefTopicLink, self).delete() + super(RefTopicLink, self).delete(force, override_dependencies) if topic: topic.update_pool_by_links(pool) From 8488f2f5d4cb0d8e990ca81c750182be841121b9 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 8 Jul 2024 09:15:37 +0300 Subject: [PATCH 015/125] refactor(Topic): rename get_pool to get_related_pool. --- sefaria/model/topic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 5a61f359ff..0efb1272f6 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -840,7 +840,7 @@ def set_description(self, lang, title, prompt): self.descriptions = d return self - def get_pool(self): + def get_related_pool(self): return 'sheets' if self.is_sheet else 'textual' def get_topic(self): @@ -849,11 +849,11 @@ def get_topic(self): def save(self, override_dependencies=False): super(RefTopicLink, self).save(override_dependencies) topic = self.get_topic() - topic.add_pool(self.get_pool()) + topic.add_pool(self.get_related_pool()) def delete(self, force=False, override_dependencies=False): topic = self.get_topic() - pool = self.get_pool() + pool = self.get_related_pool() super(RefTopicLink, self).delete(force, override_dependencies) if topic: topic.update_pool_by_links(pool) From 740d31f057262f4f3f742e68d109b068a2d373ee Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 15 Jul 2024 10:55:37 +0300 Subject: [PATCH 016/125] refactor(Topic): one function for updating topic after saving a refLink. numSources is the total of both kinds of refLinks, to texts and sheets. --- sefaria/model/topic.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 0efb1272f6..6903cf2a41 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -39,7 +39,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'categoryDescription', # dictionary, keys are 2-letter language codes 'isTopLevelDisplay', 'displayOrder', - 'numSources', + 'numSources', # total number of refLinks, to texts and sheets. 'shouldDisplay', 'parasha', # name of parsha as it appears in `parshiot` collection 'ref', # dictionary for topics with refs associated with them (e.g. parashah) containing strings `en`, `he`, and `url`. @@ -436,20 +436,16 @@ def __str__(self): def __repr__(self): return "{}.init('{}')".format(self.__class__.__name__, self.slug) - def add_pool(self, pool_name): - if pool_name not in self.pools: - self.pools.append(pool_name) - self.save() - - def update_pool_by_links(self, pool): + def update_after_link_change(self, pool): """ - updating the pools 'sheets' or 'textual' according to the existence of links + updating the pools 'sheets' or 'textual' according to the existence of links and the numSources :param pool: 'sheets' or 'textual' """ links = self.get_ref_links(pool == 'sheets') if bool(links) != pool in self.pools: self.pools.remove(pool) if pool in self.pools else self.pools.append(pool) - self.save() + self.numSources = self.link_set('refTopic').count() + self.save() class PersonTopic(Topic): @@ -849,14 +845,14 @@ def get_topic(self): def save(self, override_dependencies=False): super(RefTopicLink, self).save(override_dependencies) topic = self.get_topic() - topic.add_pool(self.get_related_pool()) + topic.update_after_link_change(self.get_related_pool()) def delete(self, force=False, override_dependencies=False): topic = self.get_topic() pool = self.get_related_pool() super(RefTopicLink, self).delete(force, override_dependencies) if topic: - topic.update_pool_by_links(pool) + topic.update_after_link_change(pool) def _sanitize(self): super()._sanitize() From a1efe9336a570db5086864a8645314bb285a892a Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 15 Jul 2024 15:41:01 +0300 Subject: [PATCH 017/125] feat(topics): in recalculate_secondary_topic_data the only thing that done is reordering. --- sefaria/helper/topic.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index 43a1847c3d..a84b1bb68a 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -960,33 +960,18 @@ def calculate_popular_writings_for_authors(top_n, min_pr): "order": {"custom_order": rd['pagesheetrank']} }).save() - def recalculate_secondary_topic_data(): - # run before everything else because this creates new links - calculate_popular_writings_for_authors(100, 300) + sheet_source_links = RefTopicLinkSet({'pools': 'textual'}) + sheet_topic_links = RefTopicLinkSet({'pools': 'sheets'}) + sheet_related_links = IntraTopicLinkSet() - sheet_source_links, sheet_related_links, sheet_topic_links = generate_all_topic_links_from_sheets() related_links = update_intra_topic_link_orders(sheet_related_links) - all_ref_links = update_ref_topic_link_orders(sheet_source_links, sheet_topic_links) - - # now that we've gathered all the new links, delete old ones and insert new ones - RefTopicLinkSet({"generatedBy": TopicLinkHelper.generated_by_sheets}).delete() - RefTopicLinkSet({"is_sheet": True}).delete() - IntraTopicLinkSet({"generatedBy": TopicLinkHelper.generated_by_sheets}).delete() - print(f"Num Ref Links {len(all_ref_links)}") - print(f"Num Intra Links {len(related_links)}") - print(f"Num to Update {len(list(filter(lambda x: getattr(x, '_id', False), all_ref_links + related_links)))}") - print(f"Num to Insert {len(list(filter(lambda x: not getattr(x, '_id', False), all_ref_links + related_links)))}") + all_ref_links = update_ref_topic_link_orders(sheet_source_links.array(), sheet_topic_links.array()) db.topic_links.bulk_write([ UpdateOne({"_id": l._id}, {"$set": {"order": l.order}}) - if getattr(l, "_id", False) else - InsertOne(l.contents(for_db=True)) for l in (all_ref_links + related_links) ]) - add_num_sources_to_topics() - make_titles_unique() - def set_all_slugs_to_primary_title(): # reset all slugs to their primary titles, if they have drifted away From 6fe3ec85b25eb61c3cda38b95585aaa023d49312 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 16 Jul 2024 09:14:30 +0300 Subject: [PATCH 018/125] fix(Topic): init query_kwargs in get_ref_links. --- sefaria/model/topic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 6903cf2a41..c558d505b4 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -367,7 +367,9 @@ def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): kwargs['record_kwargs'] = {'context_slug': self.slug} return TopicLinkSetHelper.find(intra_link_query, **kwargs) - def get_ref_links(self, is_sheet, query_kwargs: dict = None, **kwargs): + def get_ref_links(self, is_sheet, query_kwargs=None, **kwargs): + if query_kwargs is None: + query_kwargs = {} query_kwargs['is_sheet'] = is_sheet return self.link_set('refTopic', query_kwargs, **kwargs) From ea881b87d45dcfea62f071b653ec323b758057db Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Wed, 17 Jul 2024 15:13:36 +0300 Subject: [PATCH 019/125] refactor(Topic): changes for 'pools' to be an optional attribute - add the functions get_pools, has_pool, add_pool and remove_pool, and using them in the class. --- sefaria/model/topic.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index c558d505b4..bb4b6590e9 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -106,7 +106,7 @@ def _validate(self): if getattr(self, "image", False): img_url = self.image.get("image_uri") if img_url: validate_url(img_url) - assert all(pool in self.optional_pools for pool in self.pools), f'Pools {[pool for pool in self.pools if pool not in self.optional_pools]} is not an optional pool' + assert all(pool in self.optional_pools for pool in self.get_pools()), f'Pools {[pool for pool in self.get_pools() if pool not in self.optional_pools]} is not an optional pool' def _normalize(self): super()._normalize() @@ -118,7 +118,10 @@ def _normalize(self): displays_under_link = IntraTopicLink().load({"fromTopic": slug, "linkType": "displays-under"}) if getattr(displays_under_link, "toTopic", "") == "authors": self.subclass = "author" - self.pools = sorted(set(getattr(self, 'pools', []))) + if self.get_pools(): + self.pools = sorted(set(self.get_pools())) + elif hasattr(self, 'pools'): + delattr(self, 'pools') def _sanitize(self): super()._sanitize() @@ -128,6 +131,20 @@ def _sanitize(self): p[k] = bleach.clean(v, tags=[], strip=True) setattr(self, attr, p) + def get_pools(self): + return getattr(self, 'pools', []) + + def has_pool(self, pool): + return pool in self.get_pools() + + def add_pool(self, pool): #does not save! + self.pools = self.get_pools() + self.pools.append(pool) + + def remove_pool(self, pool): #does not save! + pools = self.get_pools() + pools.remove(pool) + def set_titles(self, titles): self.title_group = TitleGroup(titles) @@ -444,8 +461,8 @@ def update_after_link_change(self, pool): :param pool: 'sheets' or 'textual' """ links = self.get_ref_links(pool == 'sheets') - if bool(links) != pool in self.pools: - self.pools.remove(pool) if pool in self.pools else self.pools.append(pool) + if bool(links) != pool in self.get_pools(): + self.remove_pool(pool) if pool in self.get_pools() else self.add_pool(pool) self.numSources = self.link_set('refTopic').count() self.save() From 972a8024e9b691ca11142676400925080f27a124 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 30 Jul 2024 11:45:17 +0300 Subject: [PATCH 020/125] refactor(Topic): change assigning by condition to self assigning with or. --- sefaria/model/topic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index bb4b6590e9..e5b432f638 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -385,8 +385,7 @@ def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): return TopicLinkSetHelper.find(intra_link_query, **kwargs) def get_ref_links(self, is_sheet, query_kwargs=None, **kwargs): - if query_kwargs is None: - query_kwargs = {} + query_kwargs = query_kwargs or {} query_kwargs['is_sheet'] = is_sheet return self.link_set('refTopic', query_kwargs, **kwargs) From 377ea5e00c8132042a724f6ef48e46eba624392b Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 30 Jul 2024 12:11:57 +0300 Subject: [PATCH 021/125] refactor(Topic): change conditions for updating the pools after link change to be clearer. --- sefaria/model/topic.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index e5b432f638..a0fee457ff 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -460,8 +460,10 @@ def update_after_link_change(self, pool): :param pool: 'sheets' or 'textual' """ links = self.get_ref_links(pool == 'sheets') - if bool(links) != pool in self.get_pools(): - self.remove_pool(pool) if pool in self.get_pools() else self.add_pool(pool) + if self.has_pool(pool) and not links: + self.remove_pool(pool) + elif not self.has_pool(pool) and links: + self.add_pool(pool) self.numSources = self.link_set('refTopic').count() self.save() From 9f46f430e073443850dd7908029b4ebae2161f6b Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 30 Jul 2024 12:18:49 +0300 Subject: [PATCH 022/125] refactor(Topic): change default of get_random_topic to None. --- reader/views.py | 2 +- sefaria/helper/topic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reader/views.py b/reader/views.py index eb5e2cb75b..141114e8b8 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4196,7 +4196,7 @@ def random_by_topic_api(request): Returns Texts API data for a random text taken from popular topic tags """ cb = request.GET.get("callback", None) - random_topic = get_random_topic() + random_topic = get_random_topic('torahtab') if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index a84b1bb68a..d8d0dfc15a 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -280,7 +280,7 @@ def curated_primacy(order_dict, lang): return (bord.get('numDatasource', 0) * bord.get('tfidf', 0)) - (aord.get('numDatasource', 0) * aord.get('tfidf', 0)) -def get_random_topic(pool='torahtab') -> Optional[Topic]: +def get_random_topic(pool=None) -> Optional[Topic]: query = {"pools": pool} if pool else {} random_topic_dict = list(db.topics.aggregate([ {"$match": query}, From 4e9210fb71ee84de9d0f5f40d93cdf176f40dbd8 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 30 Jul 2024 13:23:56 +0300 Subject: [PATCH 023/125] refactor(Topic): use enum for pools. --- sefaria/model/topic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index a0fee457ff..b7fb5198e2 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import Union, Optional from . import abstract as abst from .schema import AbstractTitledObject, TitleGroup @@ -15,6 +16,11 @@ logger = structlog.get_logger(__name__) +class Pool(Enum): + TEXTUAL = "textual" + SHEETS = "sheets" + + class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): collection = 'topics' history_noun = 'topic' @@ -52,7 +58,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] - optional_pools = {'sheets', 'textual', 'torahtab'} + optional_pools = {pool.value for pool in Pool} | {'torahtab'} attr_schemas = { "image": { @@ -459,7 +465,7 @@ def update_after_link_change(self, pool): updating the pools 'sheets' or 'textual' according to the existence of links and the numSources :param pool: 'sheets' or 'textual' """ - links = self.get_ref_links(pool == 'sheets') + links = self.get_ref_links(pool == Pool.SHEETS.value) if self.has_pool(pool) and not links: self.remove_pool(pool) elif not self.has_pool(pool) and links: @@ -857,7 +863,7 @@ def set_description(self, lang, title, prompt): return self def get_related_pool(self): - return 'sheets' if self.is_sheet else 'textual' + return Pool.SHEETS.value if self.is_sheet else Pool.TEXTUAL.value def get_topic(self): return Topic().load({'slug': self.toTopic}) From 6e42995126393ff4f10642e727ccf1e64f85575f Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Sun, 4 Aug 2024 11:54:27 +0300 Subject: [PATCH 024/125] refactor(Topic): use attr_schemas for validation of pools rather than explicit assertion. --- sefaria/model/topic.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index b7fb5198e2..5b55cdccd9 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -58,7 +58,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] - optional_pools = {pool.value for pool in Pool} | {'torahtab'} + allowed_pools = {pool.value for pool in Pool} | {'torahtab'} attr_schemas = { "image": { @@ -81,6 +81,13 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): } } } + }, + 'pools': { + 'type': 'list', + 'schema': { + 'type': 'string', + 'allowed': allowed_pools + } } } @@ -112,7 +119,6 @@ def _validate(self): if getattr(self, "image", False): img_url = self.image.get("image_uri") if img_url: validate_url(img_url) - assert all(pool in self.optional_pools for pool in self.get_pools()), f'Pools {[pool for pool in self.get_pools() if pool not in self.optional_pools]} is not an optional pool' def _normalize(self): super()._normalize() From c7039c9efd372d711440923e7a847b7c37807006 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Sun, 4 Aug 2024 12:01:53 +0300 Subject: [PATCH 025/125] doc(Topic): documentation for get_random_topic. --- sefaria/helper/topic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index d8d0dfc15a..baaab3499c 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -281,6 +281,10 @@ def curated_primacy(order_dict, lang): def get_random_topic(pool=None) -> Optional[Topic]: + """ + :param pool: name of th requested pool + :return: Returns a random topic from the database. If you provide pool, then the selection is limited to topics in that pool. + """ query = {"pools": pool} if pool else {} random_topic_dict = list(db.topics.aggregate([ {"$match": query}, From d93244940590597c71888c956f17c6fd3b463064 Mon Sep 17 00:00:00 2001 From: Noah Santacruz Date: Mon, 5 Aug 2024 10:08:27 +0300 Subject: [PATCH 026/125] docs(Topic): be more specific about pool param --- sefaria/helper/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index baaab3499c..572b9fc58a 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -282,7 +282,7 @@ def curated_primacy(order_dict, lang): def get_random_topic(pool=None) -> Optional[Topic]: """ - :param pool: name of th requested pool + :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. :return: Returns a random topic from the database. If you provide pool, then the selection is limited to topics in that pool. """ query = {"pools": pool} if pool else {} From d276589fc449b890388db875cfb46160320920e7 Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Mon, 5 Aug 2024 19:13:59 +0300 Subject: [PATCH 027/125] refactor(cerberus): validate whole document with allow_unknown=True rather than only validate some attributes (which allows only validate dicts). change existing attr_schemas to fit. --- sefaria/model/abstract.py | 13 +--- sefaria/model/portal.py | 139 ++++++++++++-------------------------- sefaria/model/topic.py | 29 +++----- 3 files changed, 57 insertions(+), 124 deletions(-) diff --git a/sefaria/model/abstract.py b/sefaria/model/abstract.py index 2057c0e91a..f6d93a8726 100644 --- a/sefaria/model/abstract.py +++ b/sefaria/model/abstract.py @@ -244,16 +244,9 @@ def _validate(self): " not in " + ",".join(self.required_attrs) + " or " + ",".join(self.optional_attrs)) return False """ - for attr, schema in self.attr_schemas.items(): - v = Validator(schema) - try: - value = getattr(self, attr) - if not v.validate(value): - raise InputError(v.errors) - except AttributeError: - # not checking here if value exists, that is done above. - # assumption is if value doesn't exist, it's optional - pass + v = Validator(self.attr_schemas, allow_unknown=True) + if not v.validate(self._saveable_attrs()): + raise InputError(v.errors) return True def _normalize(self): diff --git a/sefaria/model/portal.py b/sefaria/model/portal.py index 36984ceaad..c10e6dde34 100644 --- a/sefaria/model/portal.py +++ b/sefaria/model/portal.py @@ -20,100 +20,51 @@ class Portal(abst.SluggedAbstractMongoRecord): "organization" ] attr_schemas = { - "about": { - "title": { - "type": "dict", - "required": True, - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - "title_url": {"type": "string"}, - "image_uri": {"type": "string"}, - "image_caption": { - "type": "dict", - "schema": { - "en": {"type": "string"}, - "he": {"type": "string"} - } - }, - "description": { - "type": "dict", - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - }, - "mobile": { - "title": { - "type": "dict", - "required": True, - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - "description": { - "type": "dict", - "schema": { - "en": {"type": "string"}, - "he": {"type": "string"} - } - }, - "android_link": {"type": "string"}, - "ios_link": {"type": "string"} - }, - "organization": { - "title": { - "type": "dict", - "required": True, - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - "description": { - "type": "dict", - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - }, - "newsletter": { - "title": { - "type": "dict", - "required": True, - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - "description": { - "type": "dict", - "schema": { - "en": {"type": "string", "required": True}, - "he": {"type": "string", "required": True} - } - }, - "title_url": {"type": "string"}, - "api_schema": { - "type": "dict", - "schema": { - "http_method": {"type": "string", "required": True}, - "payload": { - "type": "dict", - "schema": { - "first_name_key": {"type": "string"}, - "last_name_key": {"type": "string"}, - "email_key": {"type": "string"} - } - }, - } - } - } + 'about': {'type': 'dict', + 'schema': {'title': {'type': 'dict', + 'required': True, + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}, + 'title_url': {'type': 'string'}, + 'image_uri': {'type': 'string'}, + 'image_caption': {'type': 'dict', + 'schema': {'en': {'type': 'string'}, 'he': {'type': 'string'}}}, + 'description': {'type': 'dict', + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}}}, + 'mobile': {'type': 'dict', + 'schema': {'title': {'type': 'dict', + 'required': True, + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}, + 'description': {'type': 'dict', + 'schema': {'en': {'type': 'string'}, 'he': {'type': 'string'}}}, + 'android_link': {'type': 'string'}, + 'ios_link': {'type': 'string'}}}, + 'organization': {'type': 'dict', + 'schema': {'title': {'type': 'dict', + 'required': True, + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}, + 'description': {'type': 'dict', + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}}}, + 'newsletter': {'type': 'dict', + 'schema': {'title': {'type': 'dict', + 'required': True, + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}, + 'description': {'type': 'dict', + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}, + 'title_url': {'type': 'string'}, + 'api_schema': {'type': 'dict', + 'schema': {'http_method': {'type': 'string', 'required': True}, + 'payload': {'type': 'dict', + 'schema': { + 'first_name_key': {'type': 'string'}, + 'last_name_key': {'type': 'string'}, + 'email_key': {'type': 'string'}}}}}}} } def _validate(self): diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 5b55cdccd9..3a44e30f3c 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -58,29 +58,18 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] - allowed_pools = {pool.value for pool in Pool} | {'torahtab'} + allowed_pools = [pool.value for pool in Pool] + ['torahtab'] attr_schemas = { "image": { - "image_uri": { - "type": "string", - "required": True, - "regex": "^https://storage\.googleapis\.com/img\.sefaria\.org/topics/.*?" - }, - "image_caption": { - "type": "dict", - "required": True, - "schema": { - "en": { - "type": "string", - "required": True - }, - "he": { - "type": "string", - "required": True - } - } - } + 'type': 'dict', + 'schema': {'image_uri': {'type': 'string', + 'required': True, + 'regex': '^https://storage\\.googleapis\\.com/img\\.sefaria\\.org/topics/.*?'}, + 'image_caption': {'type': 'dict', + 'required': True, + 'schema': {'en': {'type': 'string', 'required': True}, + 'he': {'type': 'string', 'required': True}}}} }, 'pools': { 'type': 'list', From 450a44e113e79756d720ac07ecb8a883dd088dfd Mon Sep 17 00:00:00 2001 From: YishaiGlasner Date: Tue, 6 Aug 2024 10:17:27 +0300 Subject: [PATCH 028/125] fix(cerberus): allow unknown only in root level (unless the attr_schemas explicitly allow it). --- sefaria/model/abstract.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sefaria/model/abstract.py b/sefaria/model/abstract.py index f6d93a8726..1195b086be 100644 --- a/sefaria/model/abstract.py +++ b/sefaria/model/abstract.py @@ -244,7 +244,10 @@ def _validate(self): " not in " + ",".join(self.required_attrs) + " or " + ",".join(self.optional_attrs)) return False """ - v = Validator(self.attr_schemas, allow_unknown=True) + schema = self.attr_schemas + for key in schema: + schema[key]['allow_unknown'] = schema[key].get('allow_unknown', False) # allow unknowns only in the root + v = Validator(schema, allow_unknown=True) if not v.validate(self._saveable_attrs()): raise InputError(v.errors) return True From eefd205c1d105339122637d1459a5dd1f3882b06 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Wed, 21 Aug 2024 10:36:35 +0300 Subject: [PATCH 029/125] chore(Topic): add back ticks to docs --- sefaria/helper/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index 572b9fc58a..03707b8841 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -283,7 +283,7 @@ def curated_primacy(order_dict, lang): def get_random_topic(pool=None) -> Optional[Topic]: """ :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. - :return: Returns a random topic from the database. If you provide pool, then the selection is limited to topics in that pool. + :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool. """ query = {"pools": pool} if pool else {} random_topic_dict = list(db.topics.aggregate([ From 6bea38ee9b70c7f6905b831c805d8f79b57ebb40 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:11:30 +0200 Subject: [PATCH 030/125] feat(topics): add topic_pool_link model --- admin_tools/__init__.py | 0 admin_tools/migrations/0001_initial.py | 24 ++++++++++++++++++ .../migrations/0002_delete_topicpoollink.py | 18 +++++++++++++ admin_tools/migrations/0003_topicpoollink.py | 25 +++++++++++++++++++ admin_tools/migrations/__init__.py | 0 admin_tools/models/__init__.py | 1 + admin_tools/models/topic_pool_link.py | 25 +++++++++++++++++++ sefaria/settings.py | 1 + 8 files changed, 94 insertions(+) create mode 100644 admin_tools/__init__.py create mode 100644 admin_tools/migrations/0001_initial.py create mode 100644 admin_tools/migrations/0002_delete_topicpoollink.py create mode 100644 admin_tools/migrations/0003_topicpoollink.py create mode 100644 admin_tools/migrations/__init__.py create mode 100644 admin_tools/models/__init__.py create mode 100644 admin_tools/models/topic_pool_link.py diff --git a/admin_tools/__init__.py b/admin_tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/admin_tools/migrations/0001_initial.py b/admin_tools/migrations/0001_initial.py new file mode 100644 index 0000000000..ec43fcb95a --- /dev/null +++ b/admin_tools/migrations/0001_initial.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 17:45 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TopicPoolLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool', models.CharField(max_length=255)), + ('topic_slug', models.CharField(max_length=255)), + ], + ), + ] diff --git a/admin_tools/migrations/0002_delete_topicpoollink.py b/admin_tools/migrations/0002_delete_topicpoollink.py new file mode 100644 index 0000000000..98e95d6eef --- /dev/null +++ b/admin_tools/migrations/0002_delete_topicpoollink.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 18:42 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('admin_tools', '0001_initial'), + ] + + operations = [ + migrations.DeleteModel( + name='TopicPoolLink', + ), + ] diff --git a/admin_tools/migrations/0003_topicpoollink.py b/admin_tools/migrations/0003_topicpoollink.py new file mode 100644 index 0000000000..95558d20a4 --- /dev/null +++ b/admin_tools/migrations/0003_topicpoollink.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 18:43 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('admin_tools', '0002_delete_topicpoollink'), + ] + + operations = [ + migrations.CreateModel( + name='TopicPoolLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool', models.CharField(max_length=255)), + ('topic_slug', models.CharField(max_length=255)), + ], + ), + ] diff --git a/admin_tools/migrations/__init__.py b/admin_tools/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/admin_tools/models/__init__.py b/admin_tools/models/__init__.py new file mode 100644 index 0000000000..6eaf38f2f7 --- /dev/null +++ b/admin_tools/models/__init__.py @@ -0,0 +1 @@ +from .topic_pool_link import TopicPoolLink diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py new file mode 100644 index 0000000000..17a64a726b --- /dev/null +++ b/admin_tools/models/topic_pool_link.py @@ -0,0 +1,25 @@ +from django.db import models + + +class TopicPoolLinkManager(models.Manager): + def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: + query_set = self.get_queryset() + if pool: + query_set = query_set.filter(pool=pool) + query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] + return [x['topic_slug'] for x in query_set] + + +class TopicPoolLink(models.Model): + pool = models.CharField(max_length=255) + topic_slug = models.CharField(max_length=255) + objects = TopicPoolLinkManager() + + def __str__(self): + return f"{self.pool} <> {self.topic_slug}" + + + + + + diff --git a/sefaria/settings.py b/sefaria/settings.py index 7eea25ff70..1a425e5392 100644 --- a/sefaria/settings.py +++ b/sefaria/settings.py @@ -144,6 +144,7 @@ 'reader', 'sourcesheets', 'sefaria.gauth', + 'admin_tools', 'captcha', 'django.contrib.admin', 'anymail', From ac1fa70e41a40603601056722936ba45bebc9f22 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:11:50 +0200 Subject: [PATCH 031/125] refactor(topics): modify random topic api to use topic pool link model --- sefaria/helper/topic.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index 8203a06b31..d69e3aa0cd 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -285,15 +285,12 @@ def get_random_topic(pool=None) -> Optional[Topic]: :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool. """ - query = {"pools": pool} if pool else {} - random_topic_dict = list(db.topics.aggregate([ - {"$match": query}, - {"$sample": {"size": 1}} - ])) - if len(random_topic_dict) == 0: + from admin_tools.models import TopicPoolLink + random_topic_slugs = TopicPoolLink.objects.get_random_topic_slugs(pool=pool, limit=1) + if len(random_topic_slugs) == 0: return None - return Topic(random_topic_dict[0]) + return Topic.init(random_topic_slugs[0]) def get_random_topic_source(topic:Topic) -> Optional[Ref]: From 110b05104043f56aed6808fa097105eca930917a Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:19:00 +0200 Subject: [PATCH 032/125] refactor(topics): change pool to 'promoted' --- reader/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reader/views.py b/reader/views.py index aafc905ef2..35f191eea4 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4230,7 +4230,7 @@ def random_by_topic_api(request): Returns Texts API data for a random text taken from popular topic tags """ cb = request.GET.get("callback", None) - random_topic = get_random_topic('torahtab') + random_topic = get_random_topic('promoted') if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) From cce7daf4c4a59fec42bc1386d42f993ee706209b Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:04:31 +0200 Subject: [PATCH 033/125] refactor(topics): move management of pools to use TopicLinkPool model --- admin_tools/models/topic_pool_link.py | 15 ++++++++++ reader/views.py | 3 +- sefaria/model/topic.py | 43 ++++++++++----------------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py index 17a64a726b..f7426cf872 100644 --- a/admin_tools/models/topic_pool_link.py +++ b/admin_tools/models/topic_pool_link.py @@ -1,4 +1,11 @@ from django.db import models +from enum import Enum + + +class PoolType(Enum): + TEXTUAL = "textual" + SHEETS = "sheets" + PROMOTED = "promoted" class TopicPoolLinkManager(models.Manager): @@ -9,12 +16,20 @@ def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] return [x['topic_slug'] for x in query_set] + @staticmethod + def get_pools_by_topic_slug(topic_slug) -> list[str]: + query_set = TopicPoolLink.objects.filter(topic_slug=topic_slug).values('pool').distinct() + return [x['pool'] for x in query_set] + class TopicPoolLink(models.Model): pool = models.CharField(max_length=255) topic_slug = models.CharField(max_length=255) objects = TopicPoolLinkManager() + class Meta: + unique_together = ('pool', 'topic_slug') + def __str__(self): return f"{self.pool} <> {self.topic_slug}" diff --git a/reader/views.py b/reader/views.py index 35f191eea4..1395cf1ac2 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,8 +4229,9 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ + from admin_tools.models.topic_pool_link import PoolType cb = request.GET.get("callback", None) - random_topic = get_random_topic('promoted') + random_topic = get_random_topic(PoolType.PROMOTED.value) if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index ef48716027..c126c8b2d0 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -1,9 +1,11 @@ from enum import Enum from typing import Union, Optional +from django.db.utils import IntegrityError from . import abstract as abst from .schema import AbstractTitledObject, TitleGroup from .text import Ref, IndexSet, AbstractTextRecord, Index, Term from .category import Category +from admin_tools.models.topic_pool_link import TopicPoolLink, PoolType from sefaria.system.exceptions import InputError, DuplicateRecordError from sefaria.model.timeperiod import TimePeriod, LifePeriod from sefaria.system.validators import validate_url @@ -121,11 +123,6 @@ def __hash__(self): return hash((self.collective_title, self.base_cat_path)) -class Pool(Enum): - TEXTUAL = "textual" - SHEETS = "sheets" - - class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): collection = 'topics' history_noun = 'topic' @@ -163,8 +160,6 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] - allowed_pools = [pool.value for pool in Pool] + ['torahtab'] - attr_schemas = { "image": { 'type': 'dict', @@ -176,14 +171,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'schema': {'en': {'type': 'string', 'required': True}, 'he': {'type': 'string', 'required': True}}}} }, - 'pools': { - 'type': 'list', - 'schema': { - 'type': 'string', - 'allowed': allowed_pools - } - } - } + } ROOT = "Main Menu" # the root of topic TOC is not a topic, so this is a fake slug. we know it's fake because it's not in normal form # this constant is helpful in the topic editor tool functions in this file @@ -200,6 +188,7 @@ def load(self, query, proj=None): def _set_derived_attributes(self): self.set_titles(getattr(self, "titles", None)) + self.pools = TopicPoolLink.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) if self.__class__ != Topic and not getattr(self, "subclass", False): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) @@ -224,10 +213,6 @@ def _normalize(self): displays_under_link = IntraTopicLink().load({"fromTopic": slug, "linkType": "displays-under"}) if getattr(displays_under_link, "toTopic", "") == "authors": self.subclass = "author" - if self.get_pools(): - self.pools = sorted(set(self.get_pools())) - elif hasattr(self, 'pools'): - delattr(self, 'pools') def _sanitize(self): super()._sanitize() @@ -237,17 +222,23 @@ def _sanitize(self): p[k] = bleach.clean(v, tags=[], strip=True) setattr(self, attr, p) - def get_pools(self): + def get_pools(self) -> list[str]: return getattr(self, 'pools', []) - def has_pool(self, pool): + def has_pool(self, pool: str) -> bool: return pool in self.get_pools() - def add_pool(self, pool): #does not save! + def add_pool(self, pool: str) -> None: + try: + link = TopicPoolLink(pool=pool, topic_slug=self.slug) + link.save() + except IntegrityError: + raise DuplicateRecordError(f"'{pool}'<>'{self.slug}' link already exists in TopicPoolLink table.") self.pools = self.get_pools() self.pools.append(pool) - def remove_pool(self, pool): #does not save! + def remove_pool(self, pool) -> None: + TopicPoolLink.objects.filter(pool=pool, topic_slug=self.slug).delete() pools = self.get_pools() pools.remove(pool) @@ -498,8 +489,6 @@ def get_ref_links(self, is_sheet, query_kwargs=None, **kwargs): def contents(self, **kwargs): mini = kwargs.get('minify', False) d = {'slug': self.slug} if mini else super(Topic, self).contents(**kwargs) - if kwargs.get('remove_pools', True): - d.pop('pools', None) d['primaryTitle'] = {} for lang in ('en', 'he'): d['primaryTitle'][lang] = self.get_primary_title(lang=lang, with_disambiguation=kwargs.get('with_disambiguation', True)) @@ -565,7 +554,7 @@ def update_after_link_change(self, pool): updating the pools 'sheets' or 'textual' according to the existence of links and the numSources :param pool: 'sheets' or 'textual' """ - links = self.get_ref_links(pool == Pool.SHEETS.value) + links = self.get_ref_links(pool == PoolType.SHEETS.value) if self.has_pool(pool) and not links: self.remove_pool(pool) elif not self.has_pool(pool) and links: @@ -970,7 +959,7 @@ def set_description(self, lang, title, prompt): return self def get_related_pool(self): - return Pool.SHEETS.value if self.is_sheet else Pool.TEXTUAL.value + return PoolType.SHEETS.value if self.is_sheet else PoolType.TEXTUAL.value def get_topic(self): return Topic().load({'slug': self.toTopic}) From 94dee446314d355ed6adabba8259b1b1495ca6a5 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:04:51 +0200 Subject: [PATCH 034/125] chore(topics): add uniqueness constraint on topicpoollink --- .../migrations/0004_auto_20241111_2328.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 admin_tools/migrations/0004_auto_20241111_2328.py diff --git a/admin_tools/migrations/0004_auto_20241111_2328.py b/admin_tools/migrations/0004_auto_20241111_2328.py new file mode 100644 index 0000000000..866e648b61 --- /dev/null +++ b/admin_tools/migrations/0004_auto_20241111_2328.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-12 03:28 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('admin_tools', '0003_topicpoollink'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='topicpoollink', + unique_together=set([('pool', 'topic_slug')]), + ), + ] From 129529bf456f3fb5c35cf58db31410b939b4c94c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:31:52 +0200 Subject: [PATCH 035/125] chore(topics): add migrate_good_to_promote_to_topic_pools.py --- .../migrate_good_to_promote_to_topic_pools.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scripts/migrations/migrate_good_to_promote_to_topic_pools.py diff --git a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py new file mode 100644 index 0000000000..1259dfa4df --- /dev/null +++ b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py @@ -0,0 +1,15 @@ +import django +django.setup() +from sefaria.model import * +from admin_tools.models.topic_pool_link import PoolType, TopicPoolLink + + +def run(): + ts = TopicSet({'good_to_promote': True}) + for topic in ts: + link = TopicPoolLink(topic_slug=topic.slug, pool=PoolType.PROMOTED.value) + link.save() + + +if __name__ == "__main__": + run() From 1827d9db338d221cfe56ee073452ddd06ed43190 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Wed, 13 Nov 2024 14:04:15 +0200 Subject: [PATCH 036/125] refactor(topics): Refactor to use two models, Topic and TopicPool to represent many to many relationship --- admin_tools/migrations/0001_initial.py | 24 ----------- .../migrations/0002_delete_topicpoollink.py | 18 --------- admin_tools/migrations/0003_topicpoollink.py | 25 ------------ .../migrations/0004_auto_20241111_2328.py | 19 --------- admin_tools/models/__init__.py | 1 - admin_tools/models/topic_pool_link.py | 40 ------------------- sefaria/settings.py | 2 +- {admin_tools => topics}/__init__.py | 0 .../migrations/__init__.py | 0 topics/models/__init__.py | 2 + topics/models/pool.py | 15 +++++++ topics/models/topic.py | 12 ++++++ 12 files changed, 30 insertions(+), 128 deletions(-) delete mode 100644 admin_tools/migrations/0001_initial.py delete mode 100644 admin_tools/migrations/0002_delete_topicpoollink.py delete mode 100644 admin_tools/migrations/0003_topicpoollink.py delete mode 100644 admin_tools/migrations/0004_auto_20241111_2328.py delete mode 100644 admin_tools/models/__init__.py delete mode 100644 admin_tools/models/topic_pool_link.py rename {admin_tools => topics}/__init__.py (100%) rename {admin_tools => topics}/migrations/__init__.py (100%) create mode 100644 topics/models/__init__.py create mode 100644 topics/models/pool.py create mode 100644 topics/models/topic.py diff --git a/admin_tools/migrations/0001_initial.py b/admin_tools/migrations/0001_initial.py deleted file mode 100644 index ec43fcb95a..0000000000 --- a/admin_tools/migrations/0001_initial.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 17:45 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='TopicPoolLink', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('pool', models.CharField(max_length=255)), - ('topic_slug', models.CharField(max_length=255)), - ], - ), - ] diff --git a/admin_tools/migrations/0002_delete_topicpoollink.py b/admin_tools/migrations/0002_delete_topicpoollink.py deleted file mode 100644 index 98e95d6eef..0000000000 --- a/admin_tools/migrations/0002_delete_topicpoollink.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 18:42 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('admin_tools', '0001_initial'), - ] - - operations = [ - migrations.DeleteModel( - name='TopicPoolLink', - ), - ] diff --git a/admin_tools/migrations/0003_topicpoollink.py b/admin_tools/migrations/0003_topicpoollink.py deleted file mode 100644 index 95558d20a4..0000000000 --- a/admin_tools/migrations/0003_topicpoollink.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 18:43 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('admin_tools', '0002_delete_topicpoollink'), - ] - - operations = [ - migrations.CreateModel( - name='TopicPoolLink', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('pool', models.CharField(max_length=255)), - ('topic_slug', models.CharField(max_length=255)), - ], - ), - ] diff --git a/admin_tools/migrations/0004_auto_20241111_2328.py b/admin_tools/migrations/0004_auto_20241111_2328.py deleted file mode 100644 index 866e648b61..0000000000 --- a/admin_tools/migrations/0004_auto_20241111_2328.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-12 03:28 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('admin_tools', '0003_topicpoollink'), - ] - - operations = [ - migrations.AlterUniqueTogether( - name='topicpoollink', - unique_together=set([('pool', 'topic_slug')]), - ), - ] diff --git a/admin_tools/models/__init__.py b/admin_tools/models/__init__.py deleted file mode 100644 index 6eaf38f2f7..0000000000 --- a/admin_tools/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .topic_pool_link import TopicPoolLink diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py deleted file mode 100644 index f7426cf872..0000000000 --- a/admin_tools/models/topic_pool_link.py +++ /dev/null @@ -1,40 +0,0 @@ -from django.db import models -from enum import Enum - - -class PoolType(Enum): - TEXTUAL = "textual" - SHEETS = "sheets" - PROMOTED = "promoted" - - -class TopicPoolLinkManager(models.Manager): - def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: - query_set = self.get_queryset() - if pool: - query_set = query_set.filter(pool=pool) - query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] - return [x['topic_slug'] for x in query_set] - - @staticmethod - def get_pools_by_topic_slug(topic_slug) -> list[str]: - query_set = TopicPoolLink.objects.filter(topic_slug=topic_slug).values('pool').distinct() - return [x['pool'] for x in query_set] - - -class TopicPoolLink(models.Model): - pool = models.CharField(max_length=255) - topic_slug = models.CharField(max_length=255) - objects = TopicPoolLinkManager() - - class Meta: - unique_together = ('pool', 'topic_slug') - - def __str__(self): - return f"{self.pool} <> {self.topic_slug}" - - - - - - diff --git a/sefaria/settings.py b/sefaria/settings.py index 1a425e5392..bdb6dd7460 100644 --- a/sefaria/settings.py +++ b/sefaria/settings.py @@ -144,7 +144,7 @@ 'reader', 'sourcesheets', 'sefaria.gauth', - 'admin_tools', + 'topics', 'captcha', 'django.contrib.admin', 'anymail', diff --git a/admin_tools/__init__.py b/topics/__init__.py similarity index 100% rename from admin_tools/__init__.py rename to topics/__init__.py diff --git a/admin_tools/migrations/__init__.py b/topics/migrations/__init__.py similarity index 100% rename from admin_tools/migrations/__init__.py rename to topics/migrations/__init__.py diff --git a/topics/models/__init__.py b/topics/models/__init__.py new file mode 100644 index 0000000000..3c756d8991 --- /dev/null +++ b/topics/models/__init__.py @@ -0,0 +1,2 @@ +from .topic import Topic +from .pool import TopicPool diff --git a/topics/models/pool.py b/topics/models/pool.py new file mode 100644 index 0000000000..3facbb6b90 --- /dev/null +++ b/topics/models/pool.py @@ -0,0 +1,15 @@ +from django.db import models +from enum import Enum + + +class PoolType(Enum): + TEXTUAL = "textual" + SHEETS = "sheets" + PROMOTED = "promoted" + + +class TopicPool(models.Model): + name = models.CharField(max_length=255, unique=True) + + def __str__(self): + return f"TopicPool('{self.name}')" diff --git a/topics/models/topic.py b/topics/models/topic.py new file mode 100644 index 0000000000..6bba6523d4 --- /dev/null +++ b/topics/models/topic.py @@ -0,0 +1,12 @@ +from django.db import models +from topics.models.pool import TopicPool + + +class Topic(models.Model): + slug = models.CharField(max_length=255, unique=True) + en_title = models.CharField(max_length=255, blank=True, default="") + he_title = models.CharField(max_length=255, blank=True, default="") + pools = models.ManyToManyField(TopicPool, related_name="topics") + + def __str__(self): + return f"Topic('{self.slug}')" From ad18ba188bb3d8a90e425f5ad19cecfdc0e93e32 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 12:12:42 +0200 Subject: [PATCH 037/125] feat(topics): admin interface for topics and topic pools --- topics/admin.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 topics/admin.py diff --git a/topics/admin.py b/topics/admin.py new file mode 100644 index 0000000000..189c98cfe7 --- /dev/null +++ b/topics/admin.py @@ -0,0 +1,72 @@ +from django.contrib import admin, messages +from django.db.models import BooleanField, Case, When +from topics.models import Topic, TopicPool +from topics.models.pool import PoolType + + +def create_add_to_specific_pool_action(pool_name): + def add_to_specific_pool(modeladmin, request, queryset): + try: + pool = TopicPool.objects.get(name=pool_name) + for topic in queryset: + topic.pools.add(pool) + modeladmin.message_user(request, f"Added {queryset.count()} topics to {pool.name}", messages.SUCCESS) + + except TopicPool.DoesNotExist: + modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) + + add_to_specific_pool.short_description = f"Add selected topics to '{pool_name}' pool" + return add_to_specific_pool + + +class TopicAdmin(admin.ModelAdmin): + list_display = ('slug', 'en_title', 'he_title', 'is_in_pool_general', 'is_in_pool_torah_tab') + filter_horizontal = ('pools',) + readonly_fields = ('slug', 'en_title', 'he_title') + actions = [create_add_to_specific_pool_action(pool_name) for pool_name in (PoolType.GENERAL.value, PoolType.TORAH_TAB.value)] + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.annotate( + in_pool_general=Case( + When(pools__name=PoolType.GENERAL.value, then=True), + default=False, + output_field=BooleanField() + ), + in_pool_torah_tab=Case( + When(pools__name=PoolType.TORAH_TAB.value, then=True), + default=False, + output_field=BooleanField() + ) + ) + + def is_in_pool_general(self, obj): + return obj.in_pool_general + is_in_pool_general.boolean = True + is_in_pool_general.short_description = "General?" + is_in_pool_general.admin_order_field = 'in_pool_general' + + def is_in_pool_torah_tab(self, obj): + return obj.in_pool_torah_tab + is_in_pool_torah_tab.boolean = True + is_in_pool_torah_tab.short_description = "TorahTab?" + is_in_pool_torah_tab.admin_order_field = 'in_pool_torah_tab' + + +class TopicPoolAdmin(admin.ModelAdmin): + list_display = ('name', 'topic_names') + filter_horizontal = ('topics',) + readonly_fields = ('name',) + + def topic_names(self, obj): + topic_slugs = obj.topics.all().values_list('slug', flat=True) + str_rep = ', '.join(topic_slugs[:30]) + if len(topic_slugs) > 30: + str_rep = str_rep + '...' + return str_rep + + +admin.site.register(Topic, TopicAdmin) +admin.site.register(TopicPool, TopicPoolAdmin) + + From 9725b6162e3defb2ced57a9f6cd417992a733ed1 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 12:15:47 +0200 Subject: [PATCH 038/125] feat(topics): only show library topics in topic admin view --- topics/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/admin.py b/topics/admin.py index 189c98cfe7..8b8e2ce685 100644 --- a/topics/admin.py +++ b/topics/admin.py @@ -38,7 +38,7 @@ def get_queryset(self, request): default=False, output_field=BooleanField() ) - ) + ).filter(pools__name=PoolType.LIBRARY.value) def is_in_pool_general(self, obj): return obj.in_pool_general From 544df751865d4d38271f16698985ceb7055f6583 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:42:43 +0200 Subject: [PATCH 039/125] chore(topics): update pools migration to fully migrate --- .../migrate_good_to_promote_to_topic_pools.py | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py index 1259dfa4df..c74e2b9ec3 100644 --- a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py +++ b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py @@ -1,14 +1,80 @@ import django +from django.db import IntegrityError + django.setup() -from sefaria.model import * -from admin_tools.models.topic_pool_link import PoolType, TopicPoolLink +from sefaria.model import TopicSet, RefTopicLinkSet +from topics.models.topic import Topic +from topics.models.pool import TopicPool, PoolType -def run(): +def add_to_torah_tab_pool(): + print('Adding topics to torah tab pool') + pool = TopicPool.objects.get(name=PoolType.TORAH_TAB.value) ts = TopicSet({'good_to_promote': True}) for topic in ts: - link = TopicPoolLink(topic_slug=topic.slug, pool=PoolType.PROMOTED.value) - link.save() + t = Topic.objects.get(slug=topic.slug) + t.pools.add(pool) + + +def add_to_library_pool(): + print('Adding topics to library pool') + pool = TopicPool.objects.get(name=PoolType.LIBRARY.value) + ts = TopicSet({'subclass': 'author'}) + for topic in ts: + t = Topic.objects.get(slug=topic.slug) + t.pools.add(pool) + links = RefTopicLinkSet({'is_sheet': False, 'linkType': 'about'}) + topic_slugs = {link.toTopic for link in links} + for slug in topic_slugs: + try: + t = Topic.objects.get(slug=slug) + t.pools.add(pool) + except Topic.DoesNotExist: + print('Could not find topic with slug {}'.format(slug)) + + +def add_to_sheets_pool(): + print('Adding topics to sheets pool') + pool = TopicPool.objects.get(name=PoolType.SHEETS.value) + links = RefTopicLinkSet({'is_sheet': True, 'linkType': 'about'}) + topic_slugs = {link.toTopic for link in links} + for slug in topic_slugs: + try: + t = Topic.objects.get(slug=slug) + t.pools.add(pool) + except Topic.DoesNotExist: + print('Could not find topic with slug {}'.format(slug)) + + +def delete_all_data(): + print("Delete data") + Topic.pools.through.objects.all().delete() + Topic.objects.all().delete() + TopicPool.objects.all().delete() + + +def add_topics(): + print('Adding topics') + for topic in TopicSet({}): + try: + Topic.objects.create(slug=topic.slug, en_title=topic.get_primary_title('en'), he_title=topic.get_primary_title('he')) + except IntegrityError: + print('Duplicate topic', topic.slug) + + +def add_pools(): + print('Adding pools') + for pool_name in [PoolType.LIBRARY.value, PoolType.SHEETS.value, PoolType.GENERAL.value, PoolType.TORAH_TAB.value]: + TopicPool.objects.create(name=pool_name) + + +def run(): + delete_all_data() + add_topics() + add_pools() + add_to_torah_tab_pool() + add_to_library_pool() + add_to_sheets_pool() if __name__ == "__main__": From eed87c609768125f43e34c1fedf38080e4dd664b Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:43:02 +0200 Subject: [PATCH 040/125] feat(topics): add filters and boolean columns --- topics/admin.py | 71 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/topics/admin.py b/topics/admin.py index 8b8e2ce685..a0e765ecba 100644 --- a/topics/admin.py +++ b/topics/admin.py @@ -1,11 +1,10 @@ from django.contrib import admin, messages -from django.db.models import BooleanField, Case, When from topics.models import Topic, TopicPool from topics.models.pool import PoolType -def create_add_to_specific_pool_action(pool_name): - def add_to_specific_pool(modeladmin, request, queryset): +def create_add_to_pool_action(pool_name): + def add_to_pool(modeladmin, request, queryset): try: pool = TopicPool.objects.get(name=pool_name) for topic in queryset: @@ -15,42 +14,70 @@ def add_to_specific_pool(modeladmin, request, queryset): except TopicPool.DoesNotExist: modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) - add_to_specific_pool.short_description = f"Add selected topics to '{pool_name}' pool" - return add_to_specific_pool + add_to_pool.short_description = f"Add selected topics to '{pool_name}' pool" + add_to_pool.__name__ = f"add_to_specific_pool_{pool_name}" + return add_to_pool + + +def create_remove_from_pool_action(pool_name): + def remove_from_pool(modeladmin, request, queryset): + try: + pool = TopicPool.objects.get(name=pool_name) + for topic in queryset: + topic.pools.remove(pool) + modeladmin.message_user(request, f"Removed {queryset.count()} topics from {pool.name}", messages.SUCCESS) + + except TopicPool.DoesNotExist: + modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) + + remove_from_pool.short_description = f"Remove selected topics from '{pool_name}' pool" + remove_from_pool.__name__ = f"remove_from_pool_{pool_name}" + return remove_from_pool + + +class PoolFilter(admin.SimpleListFilter): + title = 'Pool Filter' + parameter_name = 'pool' + + def lookups(self, request, model_admin): + return [ + (PoolType.GENERAL.value, 'General Pool'), + (PoolType.TORAH_TAB.value, 'TorahTab Pool'), + ] + + def queryset(self, request, queryset): + pool_name = self.value() + if pool_name: + pool = TopicPool.objects.get(name=pool_name) + return queryset.filter(pools=pool) + return queryset class TopicAdmin(admin.ModelAdmin): list_display = ('slug', 'en_title', 'he_title', 'is_in_pool_general', 'is_in_pool_torah_tab') + list_filter = (PoolFilter,) filter_horizontal = ('pools',) readonly_fields = ('slug', 'en_title', 'he_title') - actions = [create_add_to_specific_pool_action(pool_name) for pool_name in (PoolType.GENERAL.value, PoolType.TORAH_TAB.value)] + actions = [ + create_add_to_pool_action(PoolType.GENERAL.value), + create_add_to_pool_action(PoolType.TORAH_TAB.value), + create_remove_from_pool_action(PoolType.GENERAL.value), + create_remove_from_pool_action(PoolType.TORAH_TAB.value), + ] def get_queryset(self, request): queryset = super().get_queryset(request) - return queryset.annotate( - in_pool_general=Case( - When(pools__name=PoolType.GENERAL.value, then=True), - default=False, - output_field=BooleanField() - ), - in_pool_torah_tab=Case( - When(pools__name=PoolType.TORAH_TAB.value, then=True), - default=False, - output_field=BooleanField() - ) - ).filter(pools__name=PoolType.LIBRARY.value) + return queryset.filter(pools__name=PoolType.LIBRARY.value) def is_in_pool_general(self, obj): - return obj.in_pool_general + return obj.pools.filter(name=PoolType.GENERAL.value).exists() is_in_pool_general.boolean = True is_in_pool_general.short_description = "General?" - is_in_pool_general.admin_order_field = 'in_pool_general' def is_in_pool_torah_tab(self, obj): - return obj.in_pool_torah_tab + return obj.pools.filter(name=PoolType.TORAH_TAB.value).exists() is_in_pool_torah_tab.boolean = True is_in_pool_torah_tab.short_description = "TorahTab?" - is_in_pool_torah_tab.admin_order_field = 'in_pool_torah_tab' class TopicPoolAdmin(admin.ModelAdmin): From e85ecf1f117a4b30e277cb6f11998d58a94e5310 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:51:59 +0200 Subject: [PATCH 041/125] refactor(topics): refactor sefaria functions to use new django models --- reader/views.py | 2 +- sefaria/helper/topic.py | 4 ++-- sefaria/model/topic.py | 21 ++++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/reader/views.py b/reader/views.py index 1395cf1ac2..dff1551aa5 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,7 +4229,7 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ - from admin_tools.models.topic_pool_link import PoolType + from topics.models.topic_pool_link import PoolType cb = request.GET.get("callback", None) random_topic = get_random_topic(PoolType.PROMOTED.value) if random_topic is None: diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index d69e3aa0cd..e7af0f6836 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -285,8 +285,8 @@ def get_random_topic(pool=None) -> Optional[Topic]: :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool. """ - from admin_tools.models import TopicPoolLink - random_topic_slugs = TopicPoolLink.objects.get_random_topic_slugs(pool=pool, limit=1) + from topics.models import Topic as DjangoTopic + random_topic_slugs = DjangoTopic.objects.sample_topic_slugs('random', pool, limit=1) if len(random_topic_slugs) == 0: return None diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index c126c8b2d0..d28ca42e45 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -5,7 +5,8 @@ from .schema import AbstractTitledObject, TitleGroup from .text import Ref, IndexSet, AbstractTextRecord, Index, Term from .category import Category -from admin_tools.models.topic_pool_link import TopicPoolLink, PoolType +from topics.models import Topic as DjangoTopic +from topics.models import TopicPool, PoolType from sefaria.system.exceptions import InputError, DuplicateRecordError from sefaria.model.timeperiod import TimePeriod, LifePeriod from sefaria.system.validators import validate_url @@ -188,7 +189,7 @@ def load(self, query, proj=None): def _set_derived_attributes(self): self.set_titles(getattr(self, "titles", None)) - self.pools = TopicPoolLink.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) + self.pools = DjangoTopic.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) if self.__class__ != Topic and not getattr(self, "subclass", False): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) @@ -228,17 +229,15 @@ def get_pools(self) -> list[str]: def has_pool(self, pool: str) -> bool: return pool in self.get_pools() - def add_pool(self, pool: str) -> None: - try: - link = TopicPoolLink(pool=pool, topic_slug=self.slug) - link.save() - except IntegrityError: - raise DuplicateRecordError(f"'{pool}'<>'{self.slug}' link already exists in TopicPoolLink table.") + def add_pool(self, pool_name: str) -> None: + pool = TopicPool.objects.get(name=pool_name) + DjangoTopic.objects.get(slug=self.slug).pools.add(pool) self.pools = self.get_pools() - self.pools.append(pool) + self.pools.append(pool_name) def remove_pool(self, pool) -> None: - TopicPoolLink.objects.filter(pool=pool, topic_slug=self.slug).delete() + pool = TopicPool.objects.get(name=pool) + DjangoTopic.objects.get(slug=self.slug).pools.remove(pool) pools = self.get_pools() pools.remove(pool) @@ -959,7 +958,7 @@ def set_description(self, lang, title, prompt): return self def get_related_pool(self): - return PoolType.SHEETS.value if self.is_sheet else PoolType.TEXTUAL.value + return PoolType.SHEETS.value if self.is_sheet else PoolType.LIBRARY.value def get_topic(self): return Topic().load({'slug': self.toTopic}) From b4837142dcda2963d4f8ac6dfa9d781027879848 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:23 +0200 Subject: [PATCH 042/125] chore(topics): add topic migrations --- topics/migrations/0001_initial.py | 37 ++++++++++++++++++++ topics/migrations/0002_auto_20241113_0809.py | 20 +++++++++++ 2 files changed, 57 insertions(+) create mode 100644 topics/migrations/0001_initial.py create mode 100644 topics/migrations/0002_auto_20241113_0809.py diff --git a/topics/migrations/0001_initial.py b/topics/migrations/0001_initial.py new file mode 100644 index 0000000000..86d8cb24f2 --- /dev/null +++ b/topics/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-13 12:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Topic', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('slug', models.CharField(max_length=255, unique=True)), + ('en_title', models.CharField(blank=True, default='', max_length=255)), + ('he_title', models.CharField(blank=True, default='', max_length=255)), + ], + ), + migrations.CreateModel( + name='TopicPool', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ], + ), + migrations.AddField( + model_name='topic', + name='pools', + field=models.ManyToManyField(related_name='topics', to='topics.TopicPool'), + ), + ] diff --git a/topics/migrations/0002_auto_20241113_0809.py b/topics/migrations/0002_auto_20241113_0809.py new file mode 100644 index 0000000000..4fff2f2c79 --- /dev/null +++ b/topics/migrations/0002_auto_20241113_0809.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-13 12:09 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('topics', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='topic', + name='pools', + field=models.ManyToManyField(blank=True, related_name='topics', to='topics.TopicPool'), + ), + ] From fb18fcd1e5570a63584799c5c5e0b57ed7ab3ddc Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:35 +0200 Subject: [PATCH 043/125] chore(topics): add PoolType to model export --- topics/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/models/__init__.py b/topics/models/__init__.py index 3c756d8991..4c01d93533 100644 --- a/topics/models/__init__.py +++ b/topics/models/__init__.py @@ -1,2 +1,2 @@ from .topic import Topic -from .pool import TopicPool +from .pool import TopicPool, PoolType From f67db0858bfaae82afc0f325a22882ed2bfa9e65 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:54 +0200 Subject: [PATCH 044/125] refactor(topics): rename pools --- topics/models/pool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/topics/models/pool.py b/topics/models/pool.py index 3facbb6b90..b84df46fec 100644 --- a/topics/models/pool.py +++ b/topics/models/pool.py @@ -3,9 +3,10 @@ class PoolType(Enum): - TEXTUAL = "textual" + LIBRARY = "library" SHEETS = "sheets" - PROMOTED = "promoted" + TORAH_TAB = "torah_tab" + GENERAL = "general" class TopicPool(models.Model): From 67dec73208b81ce51ca30c7bd7f192b59dbc8c03 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:55:21 +0200 Subject: [PATCH 045/125] feat(topics): add utility funcs to topic model --- topics/models/topic.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/topics/models/topic.py b/topics/models/topic.py index 6bba6523d4..9613518ace 100644 --- a/topics/models/topic.py +++ b/topics/models/topic.py @@ -1,12 +1,32 @@ from django.db import models +import random from topics.models.pool import TopicPool +class TopicManager(models.Manager): + def sample_topic_slugs(self, order, pool: str = None, limit=10) -> list[str]: + if pool: + topics = self.get_topic_slugs_by_pool(pool) + else: + topics = self.all().values_list('slug', flat=True) + if order == 'random': + return random.sample(list(topics), min(limit, len(topics))) + else: + raise Exception("Invalid order: '{}'".format(order)) + + def get_pools_by_topic_slug(self, topic_slug: str) -> list[str]: + return self.filter(topic_slug=topic_slug).values_list("pools__name", flat=True) + + def get_topic_slugs_by_pool(self, pool: str) -> list[str]: + return self.filter(pools__name=pool).values_list("slug", flat=True) + + class Topic(models.Model): slug = models.CharField(max_length=255, unique=True) en_title = models.CharField(max_length=255, blank=True, default="") he_title = models.CharField(max_length=255, blank=True, default="") - pools = models.ManyToManyField(TopicPool, related_name="topics") + pools = models.ManyToManyField(TopicPool, related_name="topics", blank=True) + objects = TopicManager() def __str__(self): return f"Topic('{self.slug}')" From 86804eb2ca6130fbf8b268ec1020b490a98a33bc Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:51:48 +0200 Subject: [PATCH 046/125] fix(topics): remove pools from mongo topics model --- sefaria/model/topic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index d28ca42e45..c729d83a02 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -158,7 +158,6 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): "data_source", #any topic edited manually should display automatically in the TOC and this flag ensures this 'image', "portal_slug", # slug to relevant Portal object - 'pools', # list of strings, any of them represents a pool that this topic is member of ] attr_schemas = { From 17c6a31fc50417f14728d7f370df4eb9ab30bcd9 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:52:11 +0200 Subject: [PATCH 047/125] fix(topics): fix query --- topics/models/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/models/topic.py b/topics/models/topic.py index 9613518ace..b0211a5def 100644 --- a/topics/models/topic.py +++ b/topics/models/topic.py @@ -15,7 +15,7 @@ def sample_topic_slugs(self, order, pool: str = None, limit=10) -> list[str]: raise Exception("Invalid order: '{}'".format(order)) def get_pools_by_topic_slug(self, topic_slug: str) -> list[str]: - return self.filter(topic_slug=topic_slug).values_list("pools__name", flat=True) + return self.filter(slug=topic_slug).values_list("pools__name", flat=True) def get_topic_slugs_by_pool(self, pool: str) -> list[str]: return self.filter(pools__name=pool).values_list("slug", flat=True) From b2682468cbfb02e49e2c6697a8122dcd9b1d8767 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:53:06 +0200 Subject: [PATCH 048/125] refactor(topics): import and pool name --- reader/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reader/views.py b/reader/views.py index dff1551aa5..e4ca937670 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,9 +4229,9 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ - from topics.models.topic_pool_link import PoolType + from topics.models import PoolType cb = request.GET.get("callback", None) - random_topic = get_random_topic(PoolType.PROMOTED.value) + random_topic = get_random_topic(PoolType.TORAH_TAB.value) if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) From 30736ee4ee1e426920594de1df73427fdb421aea Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:05:02 +0200 Subject: [PATCH 049/125] chore(topics): update django topic model on mongo topic save --- sefaria/model/topic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index c729d83a02..1faaea7a06 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -193,6 +193,13 @@ def _set_derived_attributes(self): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) + def _pre_save(self): + super()._pre_save() + django_topic, created = DjangoTopic.objects.get_or_create(slug=self.slug) + django_topic.en_title = self.get_primary_title('en') + django_topic.he_title = self.get_primary_title('he') + django_topic.save() + def _validate(self): super(Topic, self)._validate() if getattr(self, 'subclass', False): From 53affe9e19e055853014f344b4d4f8dd510bff9c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:16:56 +0200 Subject: [PATCH 050/125] chore(topics): update django topic when mongo topic slug changes --- sefaria/model/topic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 1faaea7a06..edf4c8411c 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -389,6 +389,7 @@ def set_slug(self, new_slug) -> None: old_slug = getattr(self, slug_field) setattr(self, slug_field, new_slug) setattr(self, slug_field, self.normalize_slug_field(slug_field)) + DjangoTopic.objects.filter(slug=old_slug).update(slug=new_slug) self.save() # so that topic with this slug exists when saving links to it self.merge(old_slug) @@ -464,6 +465,7 @@ def merge(self, other: Union['Topic', str]) -> None: setattr(self, attr, getattr(other, attr)) self.save() other.delete() + DjangoTopic.objects.get(slug=other_slug).delete() def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): """ From f754481f342a528701df2347f4243966ab0fcf84 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:17:27 +0200 Subject: [PATCH 051/125] chore(topics): remove extra newline --- sefaria/model/topic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index edf4c8411c..e211379e37 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -393,7 +393,6 @@ def set_slug(self, new_slug) -> None: self.save() # so that topic with this slug exists when saving links to it self.merge(old_slug) - def merge(self, other: Union['Topic', str]) -> None: """ Merge `other` into `self`. This means that all data from `other` will be merged into self. From d787bf66b1037b54756a528eeae3fa3b772f6ce0 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:22:00 +0200 Subject: [PATCH 052/125] refactor(topics): move delete to Topic delete dependency --- sefaria/model/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index e211379e37..5d23144241 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -464,7 +464,6 @@ def merge(self, other: Union['Topic', str]) -> None: setattr(self, attr, getattr(other, attr)) self.save() other.delete() - DjangoTopic.objects.get(slug=other_slug).delete() def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): """ @@ -1170,6 +1169,7 @@ def process_topic_delete(topic): for sheet in db.sheets.find({"topics.slug": topic.slug}): sheet["topics"] = [t for t in sheet["topics"] if t["slug"] != topic.slug] db.sheets.save(sheet) + DjangoTopic.objects.get(slug=topic.slug).delete() def process_topic_description_change(topic, **kwargs): """ From af9f31d0b831676803d0712605526539fd8f396c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:44:58 +0200 Subject: [PATCH 053/125] test(topics): add tests to make sure django topic remains in sync with mongo topic --- sefaria/model/tests/topic_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sefaria/model/tests/topic_test.py b/sefaria/model/tests/topic_test.py index 56345624f1..4d61ccab5c 100644 --- a/sefaria/model/tests/topic_test.py +++ b/sefaria/model/tests/topic_test.py @@ -3,6 +3,7 @@ from sefaria.model.text import Ref from sefaria.system.database import db from sefaria.system.exceptions import SluggedMongoRecordMissingError +from topics.models import Topic as DjangoTopic from sefaria.helper.topic import update_topic @@ -155,6 +156,22 @@ def test_merge(self, topic_graph_to_merge): {"slug": '30', 'asTyped': 'thirty'} ] + t40 = Topic.init('40') + assert t40 is None + DjangoTopic.objects.get(slug='20') + with pytest.raises(DjangoTopic.DoesNotExist): + DjangoTopic.objects.get(slug='40') + + def test_change_title(self, topic_graph): + ts = topic_graph['topics'] + dt1 = DjangoTopic.objects.get(slug=ts['1'].slug) + assert dt1.en_title == ts['1'].get_primary_title('en') + ts['1'].title_group.add_title('new title', 'en', True, True) + ts['1'].save() + dt1 = DjangoTopic.objects.get(slug=ts['1'].slug) + assert dt1.en_title == ts['1'].get_primary_title('en') + + def test_sanitize(self): t = Topic() t.slug = "sdfsdg