From 720822b525431a30a38db2938521283acc80c407 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Mon, 13 Apr 2020 17:35:14 +0200 Subject: [PATCH 01/20] Issue-38/Issue-116/Issue-117/Issue-118/Issue-119: Working with directories differently (from environment.yml), added daemon, pid and privileges --- src/application/optionsloader.py | 38 ++++++++++++++++++- src/application/serve.py | 40 ++++++++++++++------ src/application/settingsloader.py | 52 +++++++++++++++++++++++--- src/data/environment/localhost.yml | 24 ++++++++++-- src/data/environment/production.sample | 24 ++++++++++-- 5 files changed, 153 insertions(+), 25 deletions(-) diff --git a/src/application/optionsloader.py b/src/application/optionsloader.py index a8362b0..f38f6a9 100644 --- a/src/application/optionsloader.py +++ b/src/application/optionsloader.py @@ -1,15 +1,49 @@ ### # -# Version: 1.0.0 -# Date: 2020-04-09 +# Full history: see below +# +# Version: 1.1.0 +# Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) # +# Features: +# - Extra properties for +# - Daemon +# - Extra directories +# - privileges (starting as root, lower to other user and group) +# ### from singleton import Singleton class OptionsLoader(metaclass=Singleton): + # Will be used for loading the correct file environment = '' + + # Directories set through the command line data_dir = '' + + # Directories set from the settings file (environment.yml) theme_dir = '' + log_dir = '' + run_dir = '' + + # Properties set from the settings file (environment.yml) + daemon = False + + # User privileges can be used to start as root and run on port 80 (privileged port) + # and then run with a user with less rights + privileges = False + uid = 0 + gid = 0 + +### +# +# Version: 1.0.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Original code +### + diff --git a/src/application/serve.py b/src/application/serve.py index 65bf308..a1b0bdf 100644 --- a/src/application/serve.py +++ b/src/application/serve.py @@ -2,13 +2,14 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-09 +# Version: 1.2.0 +# Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) # # Features: -# - Renaming categories to tags -# - Dynamic paths to themes and data +# - Daemon functionality +# - PID functionality +# - UID and GID functionality # ### @@ -17,6 +18,8 @@ import os import sys +from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges + from dataloader import DataLoader from optionsloader import OptionsLoader from settingsloader import SettingsLoader @@ -90,34 +93,47 @@ def print_post(self, post, **_): if __name__ == '__main__': environment = 'localhost' data_dir = "" - theme_dir = "" - opts, args = getopt.getopt(sys.argv[1:], 'd:e:t:', ['env=', 'data=', 'theme=']) + opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data=']) for opt, arg in opts: if opt in ['-d', '--data']: data_dir = arg if opt in ['-e', '--env']: environment = arg - if opt in ['-t', '--theme']: - theme_dir = arg if data_dir == '': data_dir = os.path.join(os.getcwd(), 'src', 'data') - if theme_dir == '': - theme_dir = os.path.join(os.getcwd(), 'src', 'theme', 'default') - OptionsLoader().environment = environment OptionsLoader().data_dir = data_dir - OptionsLoader().theme_dir = theme_dir settings = SettingsLoader(environment).parse() + if OptionsLoader().daemon: + daemon = Daemonizer(cherrypy.engine) + daemon.subscribe() + + pid = PIDFile(cherrypy.engine, os.path.join(OptionsLoader().run_dir, 'cherryblog.pid')) + pid.subscribe() + + if OptionsLoader().privileges: + privileges = DropPrivileges(cherrypy.engine, uid=OptionsLoader().uid, gid=OptionsLoader().gid) + privileges.subscribe() + cherrypy.quickstart(Application(), config=settings) + ### # +# Version: 1.1.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Renaming categories to tags +# - Dynamic paths to themes and data +# # Version: 1.0.1 # Date: 2020-04-08 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/settingsloader.py b/src/application/settingsloader.py index 3f4e631..225f266 100644 --- a/src/application/settingsloader.py +++ b/src/application/settingsloader.py @@ -2,13 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-09 +# Version: 1.2.0 +# Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) # -# Fixes: -# - Used os.path.join in parse. It has a slash in it and formats. -# - Dynamic paths to themes and data +# Features: +# - Split between the properties needed by the engine and needed by the application # ### @@ -26,12 +25,24 @@ def __init__(self, environment): self.__environment = environment def parse(self): + # data_dir and environment are set before the SettingsLoader is called + # and are read in serve.py where the command line is parsed environment_dir = os.path.join(OptionsLoader().data_dir, 'environment') environment_file = os.path.join(environment_dir, '{0}.yml'.format(self.__environment)) + # read the yaml file file = open(environment_file, 'r') settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader) + # set the settings needed elsewhere in the code + self.__option_settings(settings_yaml) + + # return the settings really needed in a format for CherryPy + # they will be used when starting the engine in serve.py + return self.__engine_settings(settings_yaml) + + @staticmethod + def __engine_settings(settings_yaml): global_settings = { 'server.socket_host': settings_yaml['server']['socket_host'], 'server.socket_port': settings_yaml['server']['socket_port'], @@ -61,8 +72,39 @@ def parse(self): return settings + @staticmethod + def __option_settings(settings_yaml): + if settings_yaml['directories']['theme']['absolute']: + OptionsLoader().theme_dir = settings_yaml['directories']['theme']['path'] + else: + OptionsLoader().theme_dir = os.path.join(os.getcwd(), settings_yaml['directories']['theme']['path']) + + if settings_yaml['directories']['log']['absolute']: + OptionsLoader().log_dir = settings_yaml['directories']['log']['path'] + else: + OptionsLoader().log_dir = os.path.join(os.getcwd(), settings_yaml['directories']['log']['path']) + + if settings_yaml['directories']['run']['absolute']: + OptionsLoader().run_dir = settings_yaml['directories']['run']['path'] + else: + OptionsLoader().run_dir = os.path.join(os.getcwd(), settings_yaml['directories']['run']['path']) + + OptionsLoader().daemon = settings_yaml['engine']['daemon'] + + OptionsLoader().privileges = settings_yaml['user']['privileges'] + OptionsLoader().uid = settings_yaml['user']['uid'] + OptionsLoader().gid = settings_yaml['user']['gid'] + ### # +# Version: 1.1.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Fixes: +# - Used os.path.join in parse. It has a slash in it and formats. +# - Dynamic paths to themes and data +# # Version: 1.0.1 # Date: 2020-04-08 # Author: Yves Vindevogel (vindevoy) diff --git a/src/data/environment/localhost.yml b/src/data/environment/localhost.yml index ae18056..e2ec4fb 100644 --- a/src/data/environment/localhost.yml +++ b/src/data/environment/localhost.yml @@ -1,7 +1,19 @@ --- +directories: + theme: + absolute: False + path: 'src/theme/default' + log: + absolute: False + path: 'log' + run: + absolute: False + path: 'log' + engine: autoreload: True + daemon: False server: socket_host: '127.0.0.1' @@ -10,9 +22,15 @@ server: tools: staticdirs: - - url: '/static' - absolute: False - path: 'src/theme/default/static' - url: '/images' absolute: False path: 'src/data/images' + - url: '/static' + absolute: False + path: 'src/theme/default/static' + +user: + privileges: False + uid: 502 + gid: 20 + diff --git a/src/data/environment/production.sample b/src/data/environment/production.sample index cdbdd5e..a6c2936 100644 --- a/src/data/environment/production.sample +++ b/src/data/environment/production.sample @@ -1,7 +1,19 @@ --- +directories: + theme: + absolute: False + path: 'src/theme/default' + log: + absolute: False + path: 'log' + run: + absolute: False + path: 'log' + engine: autoreload: True + daemon: False server: socket_host: '1.2.3.4' @@ -10,9 +22,15 @@ server: tools: staticdirs: - - url: '/static' - absolute: False - path: 'src/theme/default/static' - url: '/images' absolute: False path: 'src/data/images' + - url: '/static' + absolute: False + path: 'src/theme/default/static' + +user: + privileges: True + uid: 3112 + gid: 3112 + From 4d447a5d058d615b201f5bc24bfa0ace74d94d15 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Mon, 13 Apr 2020 19:09:42 +0200 Subject: [PATCH 02/20] Issue-23: Split code in MVC structure --- Makefile | 14 +-- src/application/{ => common}/singleton.py | 0 .../{serve.py => controller/application.py} | 102 ++++++------------ .../options.py} | 23 ++-- src/application/{ => controller}/settings.py | 6 +- .../{ => controller}/settingsloader.py | 32 +++--- src/application/main.py | 59 ++++++++++ src/application/{ => model}/dataloader.py | 36 +++---- src/application/{ => view}/templateloader.py | 6 +- 9 files changed, 148 insertions(+), 130 deletions(-) rename src/application/{ => common}/singleton.py (100%) rename src/application/{serve.py => controller/application.py} (55%) rename src/application/{optionsloader.py => controller/options.py} (79%) rename src/application/{ => controller}/settings.py (88%) rename src/application/{ => controller}/settingsloader.py (68%) create mode 100644 src/application/main.py rename src/application/{ => model}/dataloader.py (87%) rename src/application/{ => view}/templateloader.py (85%) diff --git a/Makefile b/Makefile index d9bf37a..2d7a159 100644 --- a/Makefile +++ b/Makefile @@ -14,16 +14,6 @@ clean: @echo '[OK] Cleaned' -setup: - @mkdir -p ./src/application - @mkdir -p ./src/theme/default - @mkdir -p ./src/data/pages - @mkdir -p ./src/data/blog - @mkdir -p ./src/data/site - @touch ./src/data/site/settings.yml - - @echo '[OK] Setup has created the /src directory and sub-directories' - download: clean @mkdir -p ./tmp @wget -O ./tmp/startbootstrap-blog-home.zip https://github.com/BlackrockDigital/startbootstrap-blog-home/archive/gh-pages.zip @@ -108,11 +98,11 @@ history: develop: @mkdir -p ./log - @python3 ./src/application/serve.py 2>&1 | tee ./log/develop.log + @python3 ./src/application/main.py 2>&1 | tee ./log/develop.log production: @mkdir -p /var/log/cherryblog - @python3 ./src/application/serve.py --env production 2>&1 | tee /var/log/cherryblog/production.log & + @python3 ./src/application/main.py --env production 2>&1 | tee /var/log/cherryblog/production.log & ### # diff --git a/src/application/singleton.py b/src/application/common/singleton.py similarity index 100% rename from src/application/singleton.py rename to src/application/common/singleton.py diff --git a/src/application/serve.py b/src/application/controller/application.py similarity index 55% rename from src/application/serve.py rename to src/application/controller/application.py index a1b0bdf..32aac1b 100644 --- a/src/application/serve.py +++ b/src/application/controller/application.py @@ -2,29 +2,19 @@ # # Full history: see below # -# Version: 1.2.0 +# Version: 2.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) # -# Features: -# - Daemon functionality -# - PID functionality -# - UID and GID functionality +# This is a split of the original application.py in the root directory, containing only the Application class # ### import cherrypy -import getopt -import os -import sys -from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges - -from dataloader import DataLoader -from optionsloader import OptionsLoader -from settingsloader import SettingsLoader -from singleton import Singleton -from templateloader import TemplateLoader +from model.dataloader import DataLoader +from common.singleton import Singleton +from view.templateloader import TemplateLoader class Application(metaclass=Singleton): @@ -69,63 +59,37 @@ def tags(self, tag, page_index=1, **_): return rendered - @cherrypy.expose - def print_page(self, page, **_): - data = DataLoader().get_page_data(page) - data['url'] = '/print_page/{0}'.format(page) - - template = TemplateLoader().get_template('print_page.html') - rendered = template.render(data=data) - - return rendered - - @cherrypy.expose - def print_post(self, post, **_): - data = DataLoader().get_post_data(post) - data['url'] = '/print_post/{0}'.format(post) - - template = TemplateLoader().get_template('print_post.html') - rendered = template.render(data=data) - - return rendered - - -if __name__ == '__main__': - environment = 'localhost' - data_dir = "" - - opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data=']) - - for opt, arg in opts: - if opt in ['-d', '--data']: - data_dir = arg - if opt in ['-e', '--env']: - environment = arg - - if data_dir == '': - data_dir = os.path.join(os.getcwd(), 'src', 'data') - - OptionsLoader().environment = environment - OptionsLoader().data_dir = data_dir - - settings = SettingsLoader(environment).parse() - - if OptionsLoader().daemon: - daemon = Daemonizer(cherrypy.engine) - daemon.subscribe() - - pid = PIDFile(cherrypy.engine, os.path.join(OptionsLoader().run_dir, 'cherryblog.pid')) - pid.subscribe() - - if OptionsLoader().privileges: - privileges = DropPrivileges(cherrypy.engine, uid=OptionsLoader().uid, gid=OptionsLoader().gid) - privileges.subscribe() - - cherrypy.quickstart(Application(), config=settings) - + # @cherrypy.expose + # def print_page(self, page, **_): + # data = DataLoader().get_page_data(page) + # data['url'] = '/print_page/{0}'.format(page) + # + # template = TemplateLoader().get_template('print_page.html') + # rendered = template.render(data=data) + # + # return rendered + # + # @cherrypy.expose + # def print_post(self, post, **_): + # data = DataLoader().get_post_data(post) + # data['url'] = '/print_post/{0}'.format(post) + # + # template = TemplateLoader().get_template('print_post.html') + # rendered = template.render(data=data) + # + # return rendered ### # +# Version: 1.2.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Daemon functionality +# - PID functionality +# - UID and GID functionality +# # Version: 1.1.0 # Date: 2020-04-09 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/optionsloader.py b/src/application/controller/options.py similarity index 79% rename from src/application/optionsloader.py rename to src/application/controller/options.py index f38f6a9..4cc26f0 100644 --- a/src/application/optionsloader.py +++ b/src/application/controller/options.py @@ -2,22 +2,18 @@ # # Full history: see below # -# Version: 1.1.0 +# Version: 2.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) # -# Features: -# - Extra properties for -# - Daemon -# - Extra directories -# - privileges (starting as root, lower to other user and group) +# Renamed class from OptionsLoader to Options (as it's not loading anything) # ### -from singleton import Singleton +from common.singleton import Singleton -class OptionsLoader(metaclass=Singleton): +class Options(metaclass=Singleton): # Will be used for loading the correct file environment = '' @@ -40,10 +36,19 @@ class OptionsLoader(metaclass=Singleton): ### # +# Version: 1.1.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Extra properties for +# - Daemon +# - Extra directories +# - privileges (starting as root, lower to other user and group) +# # Version: 1.0.0 # Date: 2020-04-09 # Author: Yves Vindevogel (vindevoy) # # Original code ### - diff --git a/src/application/settings.py b/src/application/controller/settings.py similarity index 88% rename from src/application/settings.py rename to src/application/controller/settings.py index c5af276..1f583f9 100644 --- a/src/application/settings.py +++ b/src/application/controller/settings.py @@ -16,15 +16,15 @@ import os import yaml -from optionsloader import OptionsLoader -from singleton import Singleton +from controller.options import Options +from common.singleton import Singleton class Settings(metaclass=Singleton): __index_settings = None def __init__(self): - settings_dir = os.path.join(OptionsLoader().data_dir, 'index') + settings_dir = os.path.join(Options().data_dir, 'index') file = open(os.path.join(settings_dir, 'settings.yml'), 'r') self.__index_settings = yaml.load(file, Loader=yaml.SafeLoader) diff --git a/src/application/settingsloader.py b/src/application/controller/settingsloader.py similarity index 68% rename from src/application/settingsloader.py rename to src/application/controller/settingsloader.py index 225f266..1e21c2e 100644 --- a/src/application/settingsloader.py +++ b/src/application/controller/settingsloader.py @@ -14,8 +14,8 @@ import os import yaml -from optionsloader import OptionsLoader -from singleton import Singleton +from controller.options import Options +from common.singleton import Singleton class SettingsLoader(metaclass=Singleton): @@ -26,8 +26,8 @@ def __init__(self, environment): def parse(self): # data_dir and environment are set before the SettingsLoader is called - # and are read in serve.py where the command line is parsed - environment_dir = os.path.join(OptionsLoader().data_dir, 'environment') + # and are read in application.py where the command line is parsed + environment_dir = os.path.join(Options().data_dir, 'environment') environment_file = os.path.join(environment_dir, '{0}.yml'.format(self.__environment)) # read the yaml file @@ -38,7 +38,7 @@ def parse(self): self.__option_settings(settings_yaml) # return the settings really needed in a format for CherryPy - # they will be used when starting the engine in serve.py + # they will be used when starting the engine in application.py return self.__engine_settings(settings_yaml) @staticmethod @@ -67,7 +67,7 @@ def __engine_settings(settings_yaml): settings['/favicon.ico'] = { 'tools.staticfile.on': True, - 'tools.staticfile.filename': os.path.join(OptionsLoader().data_dir, 'images', 'favicon.ico') + 'tools.staticfile.filename': os.path.join(Options().data_dir, 'images', 'favicon.ico') } return settings @@ -75,25 +75,25 @@ def __engine_settings(settings_yaml): @staticmethod def __option_settings(settings_yaml): if settings_yaml['directories']['theme']['absolute']: - OptionsLoader().theme_dir = settings_yaml['directories']['theme']['path'] + Options().theme_dir = settings_yaml['directories']['theme']['path'] else: - OptionsLoader().theme_dir = os.path.join(os.getcwd(), settings_yaml['directories']['theme']['path']) + Options().theme_dir = os.path.join(os.getcwd(), settings_yaml['directories']['theme']['path']) if settings_yaml['directories']['log']['absolute']: - OptionsLoader().log_dir = settings_yaml['directories']['log']['path'] + Options().log_dir = settings_yaml['directories']['log']['path'] else: - OptionsLoader().log_dir = os.path.join(os.getcwd(), settings_yaml['directories']['log']['path']) + Options().log_dir = os.path.join(os.getcwd(), settings_yaml['directories']['log']['path']) if settings_yaml['directories']['run']['absolute']: - OptionsLoader().run_dir = settings_yaml['directories']['run']['path'] + Options().run_dir = settings_yaml['directories']['run']['path'] else: - OptionsLoader().run_dir = os.path.join(os.getcwd(), settings_yaml['directories']['run']['path']) + Options().run_dir = os.path.join(os.getcwd(), settings_yaml['directories']['run']['path']) - OptionsLoader().daemon = settings_yaml['engine']['daemon'] + Options().daemon = settings_yaml['engine']['daemon'] - OptionsLoader().privileges = settings_yaml['user']['privileges'] - OptionsLoader().uid = settings_yaml['user']['uid'] - OptionsLoader().gid = settings_yaml['user']['gid'] + Options().privileges = settings_yaml['user']['privileges'] + Options().uid = settings_yaml['user']['uid'] + Options().gid = settings_yaml['user']['gid'] ### # diff --git a/src/application/main.py b/src/application/main.py new file mode 100644 index 0000000..067ec85 --- /dev/null +++ b/src/application/main.py @@ -0,0 +1,59 @@ +### +# +# Full history: see below +# +# Version: 2.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# main.py is the entry point of the application and is derived from the original application.py file +# It has been split into main.py containing only the main() function +# and application.py in controller that maps the URLs +# History of the file is kept in application.py +# +### + +import cherrypy +import getopt +import os +import sys + +from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges + +from controller.application import Application +from controller.options import Options +from controller.settingsloader import SettingsLoader + + +if __name__ == '__main__': + environment = 'localhost' + data_dir = "" + + opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data=']) + + for opt, arg in opts: + if opt in ['-d', '--data']: + data_dir = arg + if opt in ['-e', '--env']: + environment = arg + + if data_dir == '': + data_dir = os.path.join(os.getcwd(), 'src', 'data') + + Options().environment = environment + Options().data_dir = data_dir + + settings = SettingsLoader(environment).parse() + + if Options().daemon: + daemon = Daemonizer(cherrypy.engine) + daemon.subscribe() + + pid = PIDFile(cherrypy.engine, os.path.join(Options().run_dir, 'cherryblog.pid')) + pid.subscribe() + + if Options().privileges: + privileges = DropPrivileges(cherrypy.engine, uid=Options().uid, gid=Options().gid) + privileges.subscribe() + + cherrypy.quickstart(Application(), config=settings) diff --git a/src/application/dataloader.py b/src/application/model/dataloader.py similarity index 87% rename from src/application/dataloader.py rename to src/application/model/dataloader.py index 94b6316..d612293 100644 --- a/src/application/dataloader.py +++ b/src/application/model/dataloader.py @@ -25,15 +25,15 @@ from pathlib import Path from operator import itemgetter -from optionsloader import OptionsLoader -from settings import Settings -from singleton import Singleton +from controller.options import Options +from controller.settings import Settings +from common.singleton import Singleton class DataLoader(metaclass=Singleton): @staticmethod def __get_settings(): - settings_dir = os.path.join(OptionsLoader().data_dir, 'settings') + settings_dir = os.path.join(Options().data_dir, 'settings') file = open(os.path.join(settings_dir, 'global.yml'), 'r') settings = yaml.load(file, Loader=yaml.SafeLoader) @@ -41,12 +41,12 @@ def __get_settings(): return settings def __get_tags(self): - settings_dir = os.path.join(OptionsLoader().data_dir, 'tags_widget') + settings_dir = os.path.join(Options().data_dir, 'tags_widget') file = open(os.path.join(settings_dir, 'settings.yml'), 'r') settings = yaml.load(file, Loader=yaml.SafeLoader) - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') # Starting with a dictionary as this is the easiest to find existing tags tags = {} @@ -80,7 +80,7 @@ def __get_tags(self): @staticmethod def __get_main_menu(): - config_dir = os.path.join(OptionsLoader().data_dir, 'main_menu') + config_dir = os.path.join(Options().data_dir, 'main_menu') file = open(os.path.join(config_dir, 'settings.yml'), 'r') menu = yaml.load(file, Loader=yaml.SafeLoader) @@ -89,7 +89,7 @@ def __get_main_menu(): @staticmethod def __get_footer_menu(): - config_dir = os.path.join(OptionsLoader().data_dir, 'footer_menu') + config_dir = os.path.join(Options().data_dir, 'footer_menu') file = open(os.path.join(config_dir, 'settings.yml'), 'r') menu = yaml.load(file, Loader=yaml.SafeLoader) @@ -98,7 +98,7 @@ def __get_footer_menu(): @staticmethod def __get_important_news(): - config_dir = os.path.join(OptionsLoader().data_dir, 'important_news_widget') + config_dir = os.path.join(Options().data_dir, 'important_news_widget') file = open(os.path.join(config_dir, 'settings.yml'), 'r') news = yaml.load(file, Loader=yaml.SafeLoader) @@ -107,7 +107,7 @@ def __get_important_news(): @staticmethod def __get_version(): - config_dir = os.path.join(OptionsLoader().data_dir, 'version_widget') + config_dir = os.path.join(Options().data_dir, 'version_widget') file = open(os.path.join(config_dir, 'settings.yml'), 'r') versions = yaml.load(file, Loader=yaml.SafeLoader) @@ -125,18 +125,18 @@ def __get_common(self): @staticmethod def __count_pages(): - pages_dir = os.path.join(OptionsLoader().data_dir, 'pages') + pages_dir = os.path.join(Options().data_dir, 'pages') return len(os.listdir(pages_dir)) @staticmethod def __count_posts(): - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') return len(os.listdir(posts_dir)) def __count_tag_posts(self, tag): - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') count_entries = 0 @@ -156,7 +156,7 @@ def get_index_data(self, page_index): data_index = {} - intro_dir = os.path.join(OptionsLoader().data_dir, 'index') + intro_dir = os.path.join(Options().data_dir, 'index') intro_file = open(os.path.join(intro_dir, 'introduction.md'), 'r') intro_meta, data_index['introduction'] = self.__split_file(intro_file.read()) @@ -165,7 +165,7 @@ def get_index_data(self, page_index): data['index'] = data_index - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') data['posts'] = [] data['spotlight_posts'] = [] @@ -226,7 +226,7 @@ def get_index_data(self, page_index): def get_tag_data(self, tag, page_index): data = self.__get_common() - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') data['posts'] = [] @@ -273,7 +273,7 @@ def get_tag_data(self, tag, page_index): def get_page_data(self, page): data = self.__get_common() - pages_dir = os.path.join(OptionsLoader().data_dir, 'pages') + pages_dir = os.path.join(Options().data_dir, 'pages') file = open(os.path.join(pages_dir, '{0}.md'.format(page)), 'r') meta, content = self.__split_file(file.read()) @@ -286,7 +286,7 @@ def get_page_data(self, page): def get_post_data(self, post): data = self.__get_common() - posts_dir = os.path.join(OptionsLoader().data_dir, 'posts') + posts_dir = os.path.join(Options().data_dir, 'posts') file = open(os.path.join(posts_dir, '{0}.md'.format(post)), 'r') meta, content = self.__split_file(file.read()) diff --git a/src/application/templateloader.py b/src/application/view/templateloader.py similarity index 85% rename from src/application/templateloader.py rename to src/application/view/templateloader.py index 62776ae..4cc84ed 100644 --- a/src/application/templateloader.py +++ b/src/application/view/templateloader.py @@ -13,15 +13,15 @@ from jinja2 import Environment, FileSystemLoader -from optionsloader import OptionsLoader -from singleton import Singleton +from controller.options import Options +from common.singleton import Singleton class TemplateLoader(metaclass=Singleton): __environment = None def __init__(self): - self.__environment = Environment(loader=FileSystemLoader(OptionsLoader().theme_dir)) + self.__environment = Environment(loader=FileSystemLoader(Options().theme_dir)) def get_template(self, file): return self.__environment.get_template(file) From eebdcd90a1efb74b231c09816a7861f57a3a5d27 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Tue, 14 Apr 2020 00:14:49 +0200 Subject: [PATCH 03/20] Issue-120: Split DataLoader class --- src/application/common/content.py | 47 +++ .../{controller => common}/options.py | 0 src/application/controller/application.py | 10 +- src/application/controller/dataloader.py | 118 +++++++ src/application/controller/settings.py | 59 ---- src/application/controller/settingsloader.py | 8 +- src/application/main.py | 2 +- src/application/model/codeversion.py | 23 ++ src/application/model/commondata.py | 36 ++ src/application/model/dataloader.py | 326 ------------------ src/application/model/important_news.py | 23 ++ src/application/model/index.py | 126 +++++++ src/application/model/pages.py | 52 +++ src/application/model/posts.py | 52 +++ src/application/model/settings.py | 29 ++ src/application/model/tags.py | 140 ++++++++ src/application/view/templateloader.py | 2 +- .../settings.yml | 0 .../settings.yml | 0 src/data/settings/global.yml | 4 +- src/data/{tags_widget => tags}/settings.yml | 0 src/theme/default/elements/_widget/tags.html | 8 +- .../default/elements/_widget/version.html | 4 +- 23 files changed, 664 insertions(+), 405 deletions(-) create mode 100644 src/application/common/content.py rename src/application/{controller => common}/options.py (100%) create mode 100644 src/application/controller/dataloader.py delete mode 100644 src/application/controller/settings.py create mode 100644 src/application/model/codeversion.py create mode 100644 src/application/model/commondata.py delete mode 100644 src/application/model/dataloader.py create mode 100644 src/application/model/important_news.py create mode 100644 src/application/model/index.py create mode 100644 src/application/model/pages.py create mode 100644 src/application/model/posts.py create mode 100644 src/application/model/settings.py create mode 100644 src/application/model/tags.py rename src/data/{version_widget => codeversion}/settings.yml (100%) rename src/data/{important_news_widget => important_news}/settings.yml (100%) rename src/data/{tags_widget => tags}/settings.yml (100%) diff --git a/src/application/common/content.py b/src/application/common/content.py new file mode 100644 index 0000000..adbe3a5 --- /dev/null +++ b/src/application/common/content.py @@ -0,0 +1,47 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +import markdown +import os +import yaml + +from common.options import Options +from common.singleton import Singleton + + +class Content(metaclass=Singleton): + def load_data_settings_yaml(self, directory): + return self.load_settings_yaml(os.path.join(Options().data_dir, directory)) + + def load_settings_yaml(self, directory): + return self.load_yaml(directory, 'settings.yml') + + @staticmethod + def load_yaml(directory, file): + yaml_file = open(os.path.join(directory, file), 'r') + + return yaml.load(yaml_file, Loader=yaml.SafeLoader) + + @staticmethod + def split_file(data): + split = data.split('-' * 10) + + meta = split[0] + content = "" + + if len(split) == 2: + content = split[1] + + meta_data = yaml.load(meta, Loader=yaml.SafeLoader) + content_html = markdown.markdown(content) + + return meta_data, content_html diff --git a/src/application/controller/options.py b/src/application/common/options.py similarity index 100% rename from src/application/controller/options.py rename to src/application/common/options.py diff --git a/src/application/controller/application.py b/src/application/controller/application.py index 32aac1b..6a5a661 100644 --- a/src/application/controller/application.py +++ b/src/application/controller/application.py @@ -12,15 +12,15 @@ import cherrypy -from model.dataloader import DataLoader from common.singleton import Singleton +from controller.dataloader import DataLoader from view.templateloader import TemplateLoader class Application(metaclass=Singleton): @cherrypy.expose def index(self, page_index=1, **_): - data = DataLoader().get_index_data(page_index) + data = DataLoader().index_data(page_index) data['url'] = '/index/{0}'.format(page_index) template = TemplateLoader().get_template('screen_index.html') @@ -31,7 +31,7 @@ def index(self, page_index=1, **_): @cherrypy.expose def pages(self, page, **_): # page on the URL: http://www.yoursite.ext/pages/page - data = DataLoader().get_page_data(page) + data = DataLoader().pages_data(page) data['url'] = '/pages/{0}'.format(page) template = TemplateLoader().get_template('screen_page.html') @@ -41,7 +41,7 @@ def pages(self, page, **_): @cherrypy.expose def posts(self, post, **_): - data = DataLoader().get_post_data(post) + data = DataLoader().posts_data(post) data['url'] = '/posts/{0}'.format(post) template = TemplateLoader().get_template('screen_post.html') @@ -51,7 +51,7 @@ def posts(self, post, **_): @cherrypy.expose def tags(self, tag, page_index=1, **_): - data = DataLoader().get_tag_data(tag, page_index) + data = DataLoader().tags_data(tag, page_index) data['url'] = '/tags/{0}/{1}'.format(tag, page_index) template = TemplateLoader().get_template('screen_tag.html') diff --git a/src/application/controller/dataloader.py b/src/application/controller/dataloader.py new file mode 100644 index 0000000..605c2d2 --- /dev/null +++ b/src/application/controller/dataloader.py @@ -0,0 +1,118 @@ +### +# +# Full history: see below +# +# Version: 1.1.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Renaming categories to tags +# - Added introduction to the index page +# - Dynamic paths to themes and data +# - Updated the path of the main menu settings file +# - Data for important_news and version widget +# - Data for footer_menu +# +### + +from common.singleton import Singleton +from model.codeversion import CodeVersion +from model.commondata import CommonData +from model.important_news import ImportantNews +from model.index import Index +from model.pages import Pages +from model.posts import Posts +from model.settings import Settings +from model.tags import Tags + + +class DataLoader(metaclass=Singleton): + # code version + @property + def code_version_data(self): + return CodeVersion().data + + # content + @property + def common_data(self): + return CommonData(self).data() + + # important news + @property + def important_news_data(self): + return ImportantNews().data + + # index + @property + def index_main_menu(self): + return Index(self).main_menu + + @property + def index_footer_menu(self): + return Index(self).footer_menu + + def index_data(self, page_index): + return Index(self).data(page_index) + + @property + def index_max_posts(self): + return Index(self).max_posts + + @property + def index_spotlight_posts(self): + return Index(self).spotlight_posts + + @property + def index_highlight_posts(self): + return Index(self).highlight_posts + + # pages + @property + def pages_count(self): + return Pages(self).count + + def pages_data(self, page): + return Pages(self).data(page) + + @property + def pages_directory(self): + return Pages(self).directory + + # posts + @property + def posts_count(self): + return Posts(self).count + + def posts_data(self, post): + return Posts(self).data(post) + + @property + def posts_directory(self): + return Posts(self).directory + + # settings + @property + def global_settings(self): + return Settings().global_settings + + # tags + def tags_posts_count(self, tag): + return Tags(self).count_posts(tag) + + @property + def tags_list(self): + return Tags(self).list + + def tags_data(self, tag, page_index): + return Tags(self).data(tag, page_index) + +### +# +# Version: 1.0.0 +# Date: 2020-04-07 +# Author: Yves Vindevogel (vindevoy) +# +# Original code +# +### diff --git a/src/application/controller/settings.py b/src/application/controller/settings.py deleted file mode 100644 index 1f583f9..0000000 --- a/src/application/controller/settings.py +++ /dev/null @@ -1,59 +0,0 @@ -### -# -# Full history: see below -# -# Version: 1.1.0 -# Date: 2020-04-09 -# Author: Yves Vindevogel (vindevoy) -# -# Features: -# - Uses a dynamic root directory where to retrieve the settings -# - Dynamic paths to themes and data -# - Path for index.yml updated and renamed to settings.yml -# -### - -import os -import yaml - -from controller.options import Options -from common.singleton import Singleton - - -class Settings(metaclass=Singleton): - __index_settings = None - - def __init__(self): - settings_dir = os.path.join(Options().data_dir, 'index') - file = open(os.path.join(settings_dir, 'settings.yml'), 'r') - - self.__index_settings = yaml.load(file, Loader=yaml.SafeLoader) - - @property - def index_max_posts(self): - return int(self.__index_settings['max_posts']) - - @property - def index_spotlight_posts(self): - return int(self.__index_settings['spotlight_posts']) - - @property - def index_highlight_posts(self): - return int(self.__index_settings['highlight_posts']) - -### -# -# Version: 1.0.1 -# Date: 2020-04-08 -# Author: Yves Vindevogel (vindevoy) -# -# Fixes: -# - Hard-coded values -# -# Version: 1.0.0 -# Date: 2020-04-07 -# Author: Yves Vindevogel (vindevoy) -# -# Original code -# -### diff --git a/src/application/controller/settingsloader.py b/src/application/controller/settingsloader.py index 1e21c2e..be6050e 100644 --- a/src/application/controller/settingsloader.py +++ b/src/application/controller/settingsloader.py @@ -12,9 +12,9 @@ ### import os -import yaml -from controller.options import Options +from common.content import Content +from common.options import Options from common.singleton import Singleton @@ -28,11 +28,9 @@ def parse(self): # data_dir and environment are set before the SettingsLoader is called # and are read in application.py where the command line is parsed environment_dir = os.path.join(Options().data_dir, 'environment') - environment_file = os.path.join(environment_dir, '{0}.yml'.format(self.__environment)) # read the yaml file - file = open(environment_file, 'r') - settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader) + settings_yaml = Content().load_yaml(environment_dir, '{0}.yml'.format(self.__environment)) # set the settings needed elsewhere in the code self.__option_settings(settings_yaml) diff --git a/src/application/main.py b/src/application/main.py index 067ec85..f35695e 100644 --- a/src/application/main.py +++ b/src/application/main.py @@ -21,7 +21,7 @@ from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges from controller.application import Application -from controller.options import Options +from common.options import Options from controller.settingsloader import SettingsLoader diff --git a/src/application/model/codeversion.py b/src/application/model/codeversion.py new file mode 100644 index 0000000..730a792 --- /dev/null +++ b/src/application/model/codeversion.py @@ -0,0 +1,23 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +from common.content import Content +from common.singleton import Singleton + + +class CodeVersion(metaclass=Singleton): + __settings_dir = 'codeversion' + + data = None + + def __init__(self): + self.data = Content().load_data_settings_yaml(self.__settings_dir) diff --git a/src/application/model/commondata.py b/src/application/model/commondata.py new file mode 100644 index 0000000..2b72b41 --- /dev/null +++ b/src/application/model/commondata.py @@ -0,0 +1,36 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +from common.singleton import Singleton + +## +# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! +## + + +class CommonData(metaclass=Singleton): + __data_loader = None + + def __init__(self, data_loader): + self.__data_loader = data_loader + + def data(self): + dl = self.__data_loader + + return { + 'settings': dl.global_settings, + 'tags_list': dl.tags_list, + 'main_menu': dl.index_main_menu, + 'footer_menu': dl.index_footer_menu, + 'important_news': dl.important_news_data, + 'code_version': dl.code_version_data + } diff --git a/src/application/model/dataloader.py b/src/application/model/dataloader.py deleted file mode 100644 index d612293..0000000 --- a/src/application/model/dataloader.py +++ /dev/null @@ -1,326 +0,0 @@ -### -# -# Full history: see below -# -# Version: 1.1.0 -# Date: 2020-04-09 -# Author: Yves Vindevogel (vindevoy) -# -# Features: -# - Renaming categories to tags -# - Added introduction to the index page -# - Dynamic paths to themes and data -# - Updated the path of the main menu settings file -# - Data for important_news and version widget -# - Data for footer_menu -# -### - -import markdown -import math -import yaml -import os -import string - -from pathlib import Path -from operator import itemgetter - -from controller.options import Options -from controller.settings import Settings -from common.singleton import Singleton - - -class DataLoader(metaclass=Singleton): - @staticmethod - def __get_settings(): - settings_dir = os.path.join(Options().data_dir, 'settings') - file = open(os.path.join(settings_dir, 'global.yml'), 'r') - - settings = yaml.load(file, Loader=yaml.SafeLoader) - - return settings - - def __get_tags(self): - settings_dir = os.path.join(Options().data_dir, 'tags_widget') - file = open(os.path.join(settings_dir, 'settings.yml'), 'r') - - settings = yaml.load(file, Loader=yaml.SafeLoader) - - posts_dir = os.path.join(Options().data_dir, 'posts') - - # Starting with a dictionary as this is the easiest to find existing tags - tags = {} - - for file in os.listdir(posts_dir): - file = open(os.path.join(posts_dir, file), 'r') - - meta, _ = self.__split_file(file.read()) # No need to catch the content - - for tag in meta['tags']: - label = self.__tag_label(tag) - - if label in settings['skip_tags']: - continue - - if label in tags.keys(): - current_count = tags[label]['count'] - - tags[label]['count'] = current_count + 1 - else: - data = {'label': label, 'count': 1, 'text': string.capwords(tag)} - tags[label] = data - - # Pushing this into a simple array for Jinja2 - tags_array = [] - - for _, value in tags.items(): # Only need the value - tags_array.append(value) - - return sorted(tags_array, key=itemgetter('count'), reverse=True) - - @staticmethod - def __get_main_menu(): - config_dir = os.path.join(Options().data_dir, 'main_menu') - file = open(os.path.join(config_dir, 'settings.yml'), 'r') - - menu = yaml.load(file, Loader=yaml.SafeLoader) - - return menu - - @staticmethod - def __get_footer_menu(): - config_dir = os.path.join(Options().data_dir, 'footer_menu') - file = open(os.path.join(config_dir, 'settings.yml'), 'r') - - menu = yaml.load(file, Loader=yaml.SafeLoader) - - return menu - - @staticmethod - def __get_important_news(): - config_dir = os.path.join(Options().data_dir, 'important_news_widget') - file = open(os.path.join(config_dir, 'settings.yml'), 'r') - - news = yaml.load(file, Loader=yaml.SafeLoader) - - return news - - @staticmethod - def __get_version(): - config_dir = os.path.join(Options().data_dir, 'version_widget') - file = open(os.path.join(config_dir, 'settings.yml'), 'r') - - versions = yaml.load(file, Loader=yaml.SafeLoader) - - return versions - - def __get_common(self): - return {'settings': self.__get_settings(), - 'tags': self.__get_tags(), - 'main_menu': self.__get_main_menu(), - 'footer_menu': self.__get_footer_menu(), - 'important_news': self.__get_important_news(), - 'version': self.__get_version() - } - - @staticmethod - def __count_pages(): - pages_dir = os.path.join(Options().data_dir, 'pages') - - return len(os.listdir(pages_dir)) - - @staticmethod - def __count_posts(): - posts_dir = os.path.join(Options().data_dir, 'posts') - - return len(os.listdir(posts_dir)) - - def __count_tag_posts(self, tag): - posts_dir = os.path.join(Options().data_dir, 'posts') - - count_entries = 0 - - for file in os.listdir(posts_dir): - file = open(os.path.join(posts_dir, file), 'r') - - post, _ = self.__split_file(file.read()) - - for tag_raw in post['tags']: - if self.__tag_label(tag_raw) == tag: - count_entries += 1 - - return count_entries - - def get_index_data(self, page_index): - data = self.__get_common() - - data_index = {} - - intro_dir = os.path.join(Options().data_dir, 'index') - intro_file = open(os.path.join(intro_dir, 'introduction.md'), 'r') - - intro_meta, data_index['introduction'] = self.__split_file(intro_file.read()) - - data_index['image'] = intro_meta['image'] - - data['index'] = data_index - - posts_dir = os.path.join(Options().data_dir, 'posts') - - data['posts'] = [] - data['spotlight_posts'] = [] - data['highlight_posts'] = [] - - # For now we take the list of files in reversed sort order - # (newest should get a newer entry at the end of the list, like post1, ..., postx) - # I will sort this out later - - max_entries = Settings().index_max_posts - spotlight_entries = Settings().index_spotlight_posts - highlight_entries = Settings().index_highlight_posts - - count_entries = 0 - skip_entries = (int(page_index) - 1) * max_entries - - for file in sorted(os.listdir(posts_dir), reverse=True): - count_entries += 1 - - # We count the entries, but for pages 2 and more, you don't show them - if skip_entries >= count_entries: - continue - - file = open(os.path.join(posts_dir, file), 'r') - - post, post['content'] = self.__split_file(file.read()) - - stem = Path(file.name).stem - post['url'] = stem - - if page_index == 1: - if count_entries <= spotlight_entries: - data['spotlight_posts'].append(post) - - if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries): - data['highlight_posts'].append(post) - - if count_entries > (spotlight_entries + highlight_entries): - data['posts'].append(post) - - else: - data['posts'].append(post) - - if count_entries == (max_entries + skip_entries): - break - - total_posts = self.__count_posts() - total_index_pages = math.ceil(total_posts / max_entries) - - data['pagination'] = {'current_page': int(page_index), - 'total_pages': total_index_pages, - 'spotlight_posts': len(data['spotlight_posts']), - 'highlight_posts': len(data['highlight_posts']), - 'posts': len(data['posts'])} - - return data - - def get_tag_data(self, tag, page_index): - data = self.__get_common() - - posts_dir = os.path.join(Options().data_dir, 'posts') - - data['posts'] = [] - - max_entries = Settings().index_max_posts - count_entries = 0 - skip_entries = (int(page_index) - 1) * max_entries - - for file in sorted(os.listdir(posts_dir), reverse=True): - file = open(os.path.join(posts_dir, file), 'r') - - post, post['content'] = self.__split_file(file.read()) - - must_include = False - - for tag_raw in post['tags']: - if self.__tag_label(tag_raw) == tag: - must_include = True - break - - if must_include: - count_entries += 1 - - # We count the entries, but for pages 2 and more, you don't show them - if skip_entries >= count_entries: - continue - - stem = Path(file.name).stem - post['url'] = stem - - data['posts'].append(post) - - if count_entries == max_entries: - break - - data['tag'] = {'name': string.capwords(tag.replace('-', ' ')), 'path': tag} - - total_posts = self.__count_tag_posts(tag) - total_index_pages = math.ceil(total_posts / max_entries) - - data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages} - - return data - - def get_page_data(self, page): - data = self.__get_common() - - pages_dir = os.path.join(Options().data_dir, 'pages') - file = open(os.path.join(pages_dir, '{0}.md'.format(page)), 'r') - - meta, content = self.__split_file(file.read()) - - meta['content'] = content - data['page'] = meta - - return data - - def get_post_data(self, post): - data = self.__get_common() - - posts_dir = os.path.join(Options().data_dir, 'posts') - file = open(os.path.join(posts_dir, '{0}.md'.format(post)), 'r') - - meta, content = self.__split_file(file.read()) - - meta['content'] = content - data['post'] = meta - - return data - - @staticmethod - def __split_file(data): - split = data.split('-' * 10) - - meta = split[0] - content = "" - - if len(split) == 2: - content = split[1] - - meta_data = yaml.load(meta, Loader=yaml.SafeLoader) - content_html = markdown.markdown(content) - - return meta_data, content_html - - @staticmethod - def __tag_label(tag): - return tag.lower().replace(' ', '-') - -### -# -# Version: 1.0.0 -# Date: 2020-04-07 -# Author: Yves Vindevogel (vindevoy) -# -# Original code -# -### diff --git a/src/application/model/important_news.py b/src/application/model/important_news.py new file mode 100644 index 0000000..90b4497 --- /dev/null +++ b/src/application/model/important_news.py @@ -0,0 +1,23 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +from common.content import Content +from common.singleton import Singleton + + +class ImportantNews(metaclass=Singleton): + __settings_dir = 'important_news' + + data = None + + def __init__(self): + self.data = Content().load_data_settings_yaml(self.__settings_dir) diff --git a/src/application/model/index.py b/src/application/model/index.py new file mode 100644 index 0000000..210be18 --- /dev/null +++ b/src/application/model/index.py @@ -0,0 +1,126 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +import os +import math + +from pathlib import Path + +from common.content import Content +from common.options import Options +from common.singleton import Singleton + +## +# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! +## + + +class Index(metaclass=Singleton): + __settings_dir = 'index' + __main_menu_settings_dir = 'main_menu' + __footer_menu_settings_dir = 'footer_menu' + __data_loader = None + + main_menu = None + footer_menu = None + + max_posts = 0 + spotlight_posts = 0 + highlight_posts = 0 + + def __init__(self, data_loader): + self.__data_loader = data_loader + + self.main_menu = Content().load_data_settings_yaml(self.__main_menu_settings_dir) + self.footer_menu = Content().load_data_settings_yaml(self.__footer_menu_settings_dir) + + settings = Content().load_data_settings_yaml(self.__settings_dir) + + self.max_posts = settings['max_posts'] + self.spotlight_posts = settings['spotlight_posts'] + self.highlight_posts = settings['highlight_posts'] + + # TODO: data must be kept in memory + + def data(self, page_index): + data = self.__data_loader.common_data + + data_index = {} + + # TODO: this must be a method in content + + intro_dir = os.path.join(Options().data_dir, 'index') + intro_file = open(os.path.join(intro_dir, 'introduction.md'), 'r') + + intro_meta, data_index['introduction'] = Content().split_file(intro_file.read()) + + data_index['image'] = intro_meta['image'] + + data['index'] = data_index + + posts_dir = os.path.join(Options().data_dir, 'posts') + + data['posts'] = [] + data['spotlight_posts'] = [] + data['highlight_posts'] = [] + + # For now we take the list of files in reversed sort order + # (newest should get a newer entry at the end of the list, like post1, ..., postx) + # I will sort this out later + + max_entries = self.max_posts + spotlight_entries = self.spotlight_posts + highlight_entries = self.highlight_posts + + count_entries = 0 + skip_entries = (int(page_index) - 1) * max_entries + + for file in sorted(os.listdir(posts_dir), reverse=True): + count_entries += 1 + + # We count the entries, but for pages 2 and more, you don't show them + if skip_entries >= count_entries: + continue + + file = open(os.path.join(posts_dir, file), 'r') + + post, post['content'] = Content().split_file(file.read()) + + stem = Path(file.name).stem + post['url'] = stem + + if page_index == 1: + if count_entries <= spotlight_entries: + data['spotlight_posts'].append(post) + + if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries): + data['highlight_posts'].append(post) + + if count_entries > (spotlight_entries + highlight_entries): + data['posts'].append(post) + + else: + data['posts'].append(post) + + if count_entries == (max_entries + skip_entries): + break + + total_posts = self.__data_loader.posts_count + total_index_pages = math.ceil(total_posts / max_entries) + + data['pagination'] = {'current_page': int(page_index), + 'total_pages': total_index_pages, + 'spotlight_posts': len(data['spotlight_posts']), + 'highlight_posts': len(data['highlight_posts']), + 'posts': len(data['posts'])} + + return data diff --git a/src/application/model/pages.py b/src/application/model/pages.py new file mode 100644 index 0000000..c635244 --- /dev/null +++ b/src/application/model/pages.py @@ -0,0 +1,52 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +import os + +from common.singleton import Singleton +from common.content import Content +from common.options import Options + +## +# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! +## + + +class Pages(metaclass=Singleton): + # TODO: rename to base dir + + __settings_dir = 'pages' + __data_loader = None + + directory = None + count = 0 + + def __init__(self, data_loader): + self.__data_loader = data_loader + + self.directory = os.path.join(Options().data_dir, self.__settings_dir) + self.count = len(os.listdir(self.directory)) + + # TODO: data must be kept in memory + + def data(self, page): + data = self.__data_loader.common_data + + # TODO: must be from content, reading content + file = open(os.path.join(self.directory, '{0}.md'.format(page)), 'r') + + meta, content = Content().split_file(file.read()) + + meta['content'] = content + data['page'] = meta + + return data diff --git a/src/application/model/posts.py b/src/application/model/posts.py new file mode 100644 index 0000000..5ae493d --- /dev/null +++ b/src/application/model/posts.py @@ -0,0 +1,52 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - This class was split of the DataLoader class +# - Data stored in memory +# +### + +import os + +from common.singleton import Singleton +from common.content import Content +from common.options import Options + +## +# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! +## + + +class Posts(metaclass=Singleton): + __settings_dir = 'posts' + __data_loader = None + + directory = None + count = 0 + + # TODO: data must be kept in memory + + def __init__(self, data_loader): + self.__data_loader = data_loader + + self.directory = os.path.join(Options().data_dir, self.__settings_dir) + self.count = len(os.listdir(self.directory)) + + def data(self, post): + data = self.__data_loader.common_data + + # TODO: must be from content, reading content + file = open(os.path.join(self.directory, '{0}.md'.format(post)), 'r') + + meta, content = Content().split_file(file.read()) + + meta['content'] = content + data['post'] = meta + + return data diff --git a/src/application/model/settings.py b/src/application/model/settings.py new file mode 100644 index 0000000..8f68c82 --- /dev/null +++ b/src/application/model/settings.py @@ -0,0 +1,29 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +import os +import yaml + +from common.content import Content +from common.options import Options +from common.singleton import Singleton + + +class Settings(metaclass=Singleton): + __settings_dir = 'settings' + + global_settings = None + + def __init__(self): + settings_dir = os.path.join(Options().data_dir, self.__settings_dir) + + self.global_settings = Content().load_yaml(settings_dir, 'global.yml') diff --git a/src/application/model/tags.py b/src/application/model/tags.py new file mode 100644 index 0000000..852986c --- /dev/null +++ b/src/application/model/tags.py @@ -0,0 +1,140 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + +import math +import os +import string + +from operator import itemgetter +from pathlib import Path + +from common.content import Content +from common.options import Options +from common.singleton import Singleton + + +class Tags(metaclass=Singleton): + __settings_dir = 'tags' + __data_loader = None + + list = None + + def __init__(self, data_loader): + self.__data_loader = data_loader + self.__build_list() + + def __build_list(self): + settings_dir = os.path.join(Options().data_dir, self.__settings_dir) + settings = Content().load_data_settings_yaml(settings_dir) + + posts_dir = self.__data_loader.posts_directory + + # Starting with a dictionary as this is the easiest to find existing tags + tags = {} + + for file in os.listdir(posts_dir): + file = open(os.path.join(posts_dir, file), 'r') + + meta, _ = Content().split_file(file.read()) # No need to catch the content + + for tag in meta['tags']: + label = self.__tag_label(tag) + + if label in settings['skip_tags']: + continue + + if label in tags.keys(): + current_count = tags[label]['count'] + + tags[label]['count'] = current_count + 1 + else: + data = {'label': label, 'count': 1, 'text': string.capwords(tag)} + tags[label] = data + + # Pushing this into a simple array for Jinja2 + tags_array = [] + + for _, value in tags.items(): # Only need the value + tags_array.append(value) + + self.list = sorted(tags_array, key=itemgetter('count'), reverse=True) + + def data(self, tag, page_index): + data = self.__data_loader.common_data + + posts_dir = self.__data_loader.posts_directory + + data['posts'] = [] + + max_entries = self.__data_loader.index_max_posts + count_entries = 0 + skip_entries = (int(page_index) - 1) * max_entries + + for file in sorted(os.listdir(posts_dir), reverse=True): + file = open(os.path.join(posts_dir, file), 'r') + + post, post['content'] = Content().split_file(file.read()) + + must_include = False + + for tag_raw in post['tags']: + if self.__tag_label(tag_raw) == tag: + must_include = True + break + + if must_include: + count_entries += 1 + + # We count the entries, but for pages 2 and more, you don't show them + if skip_entries >= count_entries: + continue + + stem = Path(file.name).stem + post['url'] = stem + + data['posts'].append(post) + + if count_entries == max_entries: + break + + data['tag'] = {'name': self.__tag_text(tag), 'path': tag} + + total_posts = self.count_posts(tag) + total_index_pages = math.ceil(total_posts / max_entries) + + data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages} + + return data + + def count_posts(self, tag): + posts_dir = self.__data_loader.posts_directory + + count_entries = 0 + + for file in os.listdir(posts_dir): + file = open(os.path.join(posts_dir, file), 'r') + + post, _ = Content().split_file(file.read()) + + for tag_raw in post['tags']: + if self.__tag_label(tag_raw) == tag: + count_entries += 1 + + return count_entries + + @staticmethod + def __tag_label(tag): + return tag.lower().replace(' ', '-') + + @staticmethod + def __tag_text(tag): + return string.capwords(tag.replace('-', ' ')) diff --git a/src/application/view/templateloader.py b/src/application/view/templateloader.py index 4cc84ed..c85d774 100644 --- a/src/application/view/templateloader.py +++ b/src/application/view/templateloader.py @@ -13,7 +13,7 @@ from jinja2 import Environment, FileSystemLoader -from controller.options import Options +from common.options import Options from common.singleton import Singleton diff --git a/src/data/version_widget/settings.yml b/src/data/codeversion/settings.yml similarity index 100% rename from src/data/version_widget/settings.yml rename to src/data/codeversion/settings.yml diff --git a/src/data/important_news_widget/settings.yml b/src/data/important_news/settings.yml similarity index 100% rename from src/data/important_news_widget/settings.yml rename to src/data/important_news/settings.yml diff --git a/src/data/settings/global.yml b/src/data/settings/global.yml index 9699b64..3d7d616 100644 --- a/src/data/settings/global.yml +++ b/src/data/settings/global.yml @@ -33,7 +33,7 @@ navigation: page_of_of: 'of' widget: - tag: + tags: title: 'Tags' important_news: title: 'IMPORTANT NEWS' @@ -42,5 +42,5 @@ widget: text_color: '#000000' text_weight: 'normal' - version: + code_version: title: 'Latest version' \ No newline at end of file diff --git a/src/data/tags_widget/settings.yml b/src/data/tags/settings.yml similarity index 100% rename from src/data/tags_widget/settings.yml rename to src/data/tags/settings.yml diff --git a/src/theme/default/elements/_widget/tags.html b/src/theme/default/elements/_widget/tags.html index 01aebd9..da5f480 100644 --- a/src/theme/default/elements/_widget/tags.html +++ b/src/theme/default/elements/_widget/tags.html @@ -1,11 +1,11 @@
-
{{ data.settings.widget.tag.title }}
+
{{ data.settings.widget.tags.title }}
    - {% for tag in data.tags %} + {% for tag in data.tags_list %}
  • {{ tag.text }} ({{ tag.count }})
    • @@ -18,7 +18,7 @@
    {{ data.settings.widget.tag.title }}
      {% set count = namespace(value=0) %} - {% for tag in data.tags %} + {% for tag in data.tags_list %} {% if count.value % 2 == 0 %}
    • {{ tag.text }} ({{ tag.count }}) @@ -35,7 +35,7 @@
      {{ data.settings.widget.tag.title }}
        {% set count = namespace(value=0) %} - {% for tag in data.tags %} + {% for tag in data.tags_list %} {% if count.value % 2 == 1 %}
      • {{ tag.text }} ({{ tag.count }}) diff --git a/src/theme/default/elements/_widget/version.html b/src/theme/default/elements/_widget/version.html index 5cf82e8..cdf9461 100644 --- a/src/theme/default/elements/_widget/version.html +++ b/src/theme/default/elements/_widget/version.html @@ -1,9 +1,9 @@
        -
        {{ data.settings.widget.version.title }}
        +
        {{ data.settings.widget.code_version.title }}
        - {% for v in data.version.text %} + {% for v in data.code_version.text %} {{ v }}
        {% endfor %}
        From 4f02bb2fa3737b76e05397732c934aeb069f7c82 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Tue, 14 Apr 2020 00:56:59 +0200 Subject: [PATCH 04/20] Issue-31: Store data in memory --- src/application/common/content.py | 10 ++++++++- src/application/model/codeversion.py | 4 ++-- src/application/model/important_news.py | 4 ++-- src/application/model/index.py | 28 ++++++++++++------------- src/application/model/pages.py | 20 +++++++++--------- src/application/model/posts.py | 18 +++++++++------- src/application/model/settings.py | 4 ++-- src/application/model/tags.py | 20 ++++++------------ 8 files changed, 55 insertions(+), 53 deletions(-) diff --git a/src/application/common/content.py b/src/application/common/content.py index adbe3a5..9dc1ba6 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -31,8 +31,16 @@ def load_yaml(directory, file): return yaml.load(yaml_file, Loader=yaml.SafeLoader) + def read_content(self, directory, file): + content_dir = os.path.join(Options().data_dir, directory) + content_file = open(os.path.join(content_dir, file), 'r') + + meta, html = self.__split_file(content_file.read()) + + return meta, html + @staticmethod - def split_file(data): + def __split_file(data): split = data.split('-' * 10) meta = split[0] diff --git a/src/application/model/codeversion.py b/src/application/model/codeversion.py index 730a792..df45ef4 100644 --- a/src/application/model/codeversion.py +++ b/src/application/model/codeversion.py @@ -15,9 +15,9 @@ class CodeVersion(metaclass=Singleton): - __settings_dir = 'codeversion' + __base_dir = 'codeversion' data = None def __init__(self): - self.data = Content().load_data_settings_yaml(self.__settings_dir) + self.data = Content().load_data_settings_yaml(self.__base_dir) diff --git a/src/application/model/important_news.py b/src/application/model/important_news.py index 90b4497..f4db8ad 100644 --- a/src/application/model/important_news.py +++ b/src/application/model/important_news.py @@ -15,9 +15,9 @@ class ImportantNews(metaclass=Singleton): - __settings_dir = 'important_news' + __base_dir = 'important_news' data = None def __init__(self): - self.data = Content().load_data_settings_yaml(self.__settings_dir) + self.data = Content().load_data_settings_yaml(self.__base_dir) diff --git a/src/application/model/index.py b/src/application/model/index.py index 210be18..40a7a2e 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -25,11 +25,13 @@ class Index(metaclass=Singleton): - __settings_dir = 'index' + __base_dir = 'index' __main_menu_settings_dir = 'main_menu' __footer_menu_settings_dir = 'footer_menu' __data_loader = None + __data = {} + main_menu = None footer_menu = None @@ -43,25 +45,23 @@ def __init__(self, data_loader): self.main_menu = Content().load_data_settings_yaml(self.__main_menu_settings_dir) self.footer_menu = Content().load_data_settings_yaml(self.__footer_menu_settings_dir) - settings = Content().load_data_settings_yaml(self.__settings_dir) + settings = Content().load_data_settings_yaml(self.__base_dir) self.max_posts = settings['max_posts'] self.spotlight_posts = settings['spotlight_posts'] self.highlight_posts = settings['highlight_posts'] - # TODO: data must be kept in memory - def data(self, page_index): - data = self.__data_loader.common_data + key_index = 'index-{0}'.format(page_index) - data_index = {} + if key_index in self.__data.keys(): + return self.__data[key_index] - # TODO: this must be a method in content + data = self.__data_loader.common_data - intro_dir = os.path.join(Options().data_dir, 'index') - intro_file = open(os.path.join(intro_dir, 'introduction.md'), 'r') + data_index = {} - intro_meta, data_index['introduction'] = Content().split_file(intro_file.read()) + intro_meta, data_index['introduction'] = Content().read_content('index', 'introduction.md') data_index['image'] = intro_meta['image'] @@ -91,11 +91,9 @@ def data(self, page_index): if skip_entries >= count_entries: continue - file = open(os.path.join(posts_dir, file), 'r') + post, post['content'] = Content().read_content(posts_dir, file) - post, post['content'] = Content().split_file(file.read()) - - stem = Path(file.name).stem + stem = Path(file).stem post['url'] = stem if page_index == 1: @@ -123,4 +121,6 @@ def data(self, page_index): 'highlight_posts': len(data['highlight_posts']), 'posts': len(data['posts'])} + self.__data[key_index] = data + return data diff --git a/src/application/model/pages.py b/src/application/model/pages.py index c635244..81c9654 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -22,31 +22,31 @@ class Pages(metaclass=Singleton): - # TODO: rename to base dir - - __settings_dir = 'pages' + __base_dir = 'pages' __data_loader = None + __pages = {} + directory = None count = 0 def __init__(self, data_loader): self.__data_loader = data_loader - self.directory = os.path.join(Options().data_dir, self.__settings_dir) + self.directory = os.path.join(Options().data_dir, self.__base_dir) self.count = len(os.listdir(self.directory)) - # TODO: data must be kept in memory - def data(self, page): - data = self.__data_loader.common_data + if page in self.__pages.keys(): + return self.__pages[page] - # TODO: must be from content, reading content - file = open(os.path.join(self.directory, '{0}.md'.format(page)), 'r') + data = self.__data_loader.common_data - meta, content = Content().split_file(file.read()) + meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(page)) meta['content'] = content data['page'] = meta + self.__pages[page] = data + return data diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 5ae493d..09b4112 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -24,29 +24,31 @@ class Posts(metaclass=Singleton): - __settings_dir = 'posts' + __base_dir = 'posts' __data_loader = None + __posts = {} + directory = None count = 0 - # TODO: data must be kept in memory - def __init__(self, data_loader): self.__data_loader = data_loader - self.directory = os.path.join(Options().data_dir, self.__settings_dir) + self.directory = os.path.join(Options().data_dir, self.__base_dir) self.count = len(os.listdir(self.directory)) def data(self, post): - data = self.__data_loader.common_data + if post in self.__posts.keys(): + return self.__posts[post] - # TODO: must be from content, reading content - file = open(os.path.join(self.directory, '{0}.md'.format(post)), 'r') + data = self.__data_loader.common_data - meta, content = Content().split_file(file.read()) + meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(post)) meta['content'] = content data['post'] = meta + self.__posts[post] = data + return data diff --git a/src/application/model/settings.py b/src/application/model/settings.py index 8f68c82..1a06a4e 100644 --- a/src/application/model/settings.py +++ b/src/application/model/settings.py @@ -19,11 +19,11 @@ class Settings(metaclass=Singleton): - __settings_dir = 'settings' + __base_dir = 'settings' global_settings = None def __init__(self): - settings_dir = os.path.join(Options().data_dir, self.__settings_dir) + settings_dir = os.path.join(Options().data_dir, self.__base_dir) self.global_settings = Content().load_yaml(settings_dir, 'global.yml') diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 852986c..9c9ac89 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -18,12 +18,11 @@ from pathlib import Path from common.content import Content -from common.options import Options from common.singleton import Singleton class Tags(metaclass=Singleton): - __settings_dir = 'tags' + __base_dir = 'tags' __data_loader = None list = None @@ -33,8 +32,7 @@ def __init__(self, data_loader): self.__build_list() def __build_list(self): - settings_dir = os.path.join(Options().data_dir, self.__settings_dir) - settings = Content().load_data_settings_yaml(settings_dir) + settings = Content().load_data_settings_yaml(self.__base_dir) posts_dir = self.__data_loader.posts_directory @@ -42,9 +40,7 @@ def __build_list(self): tags = {} for file in os.listdir(posts_dir): - file = open(os.path.join(posts_dir, file), 'r') - - meta, _ = Content().split_file(file.read()) # No need to catch the content + meta, _ = Content().read_content(posts_dir, file) # No need to catch the content for tag in meta['tags']: label = self.__tag_label(tag) @@ -80,9 +76,7 @@ def data(self, tag, page_index): skip_entries = (int(page_index) - 1) * max_entries for file in sorted(os.listdir(posts_dir), reverse=True): - file = open(os.path.join(posts_dir, file), 'r') - - post, post['content'] = Content().split_file(file.read()) + post, post['content'] = Content().read_content(posts_dir, file) must_include = False @@ -98,7 +92,7 @@ def data(self, tag, page_index): if skip_entries >= count_entries: continue - stem = Path(file.name).stem + stem = Path(file).stem post['url'] = stem data['posts'].append(post) @@ -121,9 +115,7 @@ def count_posts(self, tag): count_entries = 0 for file in os.listdir(posts_dir): - file = open(os.path.join(posts_dir, file), 'r') - - post, _ = Content().split_file(file.read()) + post, _ = Content().read_content(posts_dir, file) for tag_raw in post['tags']: if self.__tag_label(tag_raw) == tag: From 2cebb05cb4d65c99a5d670e4e53c0832e41e957c Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Tue, 14 Apr 2020 01:18:08 +0200 Subject: [PATCH 05/20] Issue-26: Separator comes from settings --- src/application/common/content.py | 10 +++++++--- src/application/common/options.py | 1 + src/application/controller/settingsloader.py | 2 ++ src/data/environment/localhost.yml | 3 +++ src/data/environment/production.sample | 3 +++ 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/application/common/content.py b/src/application/common/content.py index 9dc1ba6..bf5a0ff 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -19,6 +19,8 @@ class Content(metaclass=Singleton): + __meta_content_separator = None + def load_data_settings_yaml(self, directory): return self.load_settings_yaml(os.path.join(Options().data_dir, directory)) @@ -39,9 +41,11 @@ def read_content(self, directory, file): return meta, html - @staticmethod - def __split_file(data): - split = data.split('-' * 10) + def __split_file(self, data): + if self.__meta_content_separator is None: + self.__meta_content_separator = Options().meta_content_separator + + split = data.split(self.__meta_content_separator) meta = split[0] content = "" diff --git a/src/application/common/options.py b/src/application/common/options.py index 4cc26f0..f19777e 100644 --- a/src/application/common/options.py +++ b/src/application/common/options.py @@ -27,6 +27,7 @@ class Options(metaclass=Singleton): # Properties set from the settings file (environment.yml) daemon = False + meta_content_separator = '' # User privileges can be used to start as root and run on port 80 (privileged port) # and then run with a user with less rights diff --git a/src/application/controller/settingsloader.py b/src/application/controller/settingsloader.py index be6050e..52b90b3 100644 --- a/src/application/controller/settingsloader.py +++ b/src/application/controller/settingsloader.py @@ -87,6 +87,8 @@ def __option_settings(settings_yaml): else: Options().run_dir = os.path.join(os.getcwd(), settings_yaml['directories']['run']['path']) + Options().meta_content_separator = settings_yaml['content']['meta_content_separator'] + Options().daemon = settings_yaml['engine']['daemon'] Options().privileges = settings_yaml['user']['privileges'] diff --git a/src/data/environment/localhost.yml b/src/data/environment/localhost.yml index e2ec4fb..5375d33 100644 --- a/src/data/environment/localhost.yml +++ b/src/data/environment/localhost.yml @@ -1,5 +1,8 @@ --- +content: + meta_content_separator: "----------" + directories: theme: absolute: False diff --git a/src/data/environment/production.sample b/src/data/environment/production.sample index a6c2936..34dc258 100644 --- a/src/data/environment/production.sample +++ b/src/data/environment/production.sample @@ -1,5 +1,8 @@ --- +content: + meta_content_separator: "----------" + directories: theme: absolute: False From 26cee8c5703e38fb5ceaa5a808a2be5ec167faa1 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 15:50:46 +0200 Subject: [PATCH 06/20] Issue-39: Added logger --- Makefile | 1 + src/application/common/logging_loader.py | 15 +++++ src/application/main.py | 17 ++++++ src/data/logging/settings.yml | 70 ++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 src/application/common/logging_loader.py create mode 100644 src/data/logging/settings.yml diff --git a/Makefile b/Makefile index 2d7a159..7cb9101 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ clean: @find . -name '__pycache__' -type d -delete + @find . -name '*.log' -type f -delete @rm -rf ./tmp ./log ./download @echo '[OK] Cleaned' diff --git a/src/application/common/logging_loader.py b/src/application/common/logging_loader.py new file mode 100644 index 0000000..7f463e6 --- /dev/null +++ b/src/application/common/logging_loader.py @@ -0,0 +1,15 @@ +import logging.config +import cherrypy + +from common.content import Content + +# https://stackoverflow.com/questions/41879512/cherrypy-is-not-respecting-desired-logging-format + + +class LoggingLoader: + @staticmethod + def configure(): + cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files) + + logging_settings = Content().load_yaml('./src/data/logging', 'settings.yml') + logging.config.dictConfig(logging_settings) diff --git a/src/application/main.py b/src/application/main.py index f35695e..fb958f5 100644 --- a/src/application/main.py +++ b/src/application/main.py @@ -15,31 +15,44 @@ import cherrypy import getopt +import logging import os import sys from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges from controller.application import Application +from common.logging_loader import LoggingLoader from common.options import Options from controller.settingsloader import SettingsLoader if __name__ == '__main__': + LoggingLoader().configure() + + logger = logging.getLogger('MAIN') + environment = 'localhost' data_dir = "" opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data=']) + logger.debug('opts: {0}'.format(opts)) + logger.debug('args: {0}'.format(args)) for opt, arg in opts: if opt in ['-d', '--data']: + logger.debug('Overriding data_dir to {0}.'.format(arg)) data_dir = arg if opt in ['-e', '--env']: + logger.debug('Overriding environment to {0}.'.format(arg)) environment = arg if data_dir == '': data_dir = os.path.join(os.getcwd(), 'src', 'data') + logger.info('Environment set to {0}.'.format(environment)) + logger.info('Data directory set to {0}.'.format(data_dir)) + Options().environment = environment Options().data_dir = data_dir @@ -48,6 +61,8 @@ if Options().daemon: daemon = Daemonizer(cherrypy.engine) daemon.subscribe() + else: + logger.info('Not running as daemon.') pid = PIDFile(cherrypy.engine, os.path.join(Options().run_dir, 'cherryblog.pid')) pid.subscribe() @@ -55,5 +70,7 @@ if Options().privileges: privileges = DropPrivileges(cherrypy.engine, uid=Options().uid, gid=Options().gid) privileges.subscribe() + else: + logger.info('No user privileges specified.') cherrypy.quickstart(Application(), config=settings) diff --git a/src/data/logging/settings.yml b/src/data/logging/settings.yml new file mode 100644 index 0000000..4c4aeae --- /dev/null +++ b/src/data/logging/settings.yml @@ -0,0 +1,70 @@ +--- + +version: 1 + +formatters: + void: + format: '' + + cherrypy: + format: '%(asctime)s [%(levelname)s] %(message)s' + + cherryblog: + format: '%(asctime)s [%(levelname)s] %(name)s %(message)s' + +handlers: + cherryblog_console: + level: DEBUG + class: logging.StreamHandler + formatter: cherryblog + stream: 'ext://sys.stdout' + + cherryblog_log: + level: DEBUG + class: logging.handlers.RotatingFileHandler + formatter: cherryblog + filename: './log/application.log' + maxBytes: 10485760 + backupCount: 20 + encoding: utf8 + + cherrypy_console: + level: DEBUG + class: logging.StreamHandler + formatter: cherrypy + stream: 'ext://sys.stdout' + + cherrypy_access: + level: DEBUG + class: logging.handlers.RotatingFileHandler + formatter: void + filename: './log/access.log' + maxBytes: 10485760 + backupCount: 20 + encoding: utf8 + + cherrypy_log: + level: DEBUG + class: logging.handlers.RotatingFileHandler + formatter: cherrypy + filename: './log/application.log' + maxBytes: 10485760 + backupCount: 20 + encoding: utf8 + + +loggers: + cherrypy.access: + handlers: [cherrypy_console, cherrypy_access] + level: INFO + propagate: no + + cherrypy.error: + handlers: [cherrypy_console, cherrypy_log] + level: INFO + propagate: no + + '': + handlers: [cherryblog_console, cherryblog_log] + level: INFO + propagate: no From 02e2b1839b54488935c29d5d8030a80732a5b6a7 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:04:33 +0200 Subject: [PATCH 07/20] Issue-137: Added logging --- src/application/common/content.py | 49 ++++++++++++++-- src/application/common/logging_loader.py | 15 ----- src/application/common/options.py | 2 + src/application/controller/application.py | 56 ++++++++++++++++--- .../{dataloader.py => data_loader.py} | 0 src/application/controller/logging_loader.py | 36 ++++++++++++ .../{settingsloader.py => settings_loader.py} | 31 +++++++--- src/application/main.py | 40 ++++++++----- src/application/model/codeversion.py | 8 +++ src/application/model/commondata.py | 13 ++++- src/application/model/important_news.py | 8 +++ src/application/model/index.py | 34 ++++++++++- src/application/model/pages.py | 17 +++++- src/application/model/posts.py | 17 +++++- src/application/model/settings.py | 8 ++- src/application/model/tags.py | 43 +++++++++++++- src/application/view/templateloader.py | 9 +++ src/data/logging/settings.yml | 2 +- 18 files changed, 330 insertions(+), 58 deletions(-) delete mode 100644 src/application/common/logging_loader.py rename src/application/controller/{dataloader.py => data_loader.py} (100%) create mode 100644 src/application/controller/logging_loader.py rename src/application/controller/{settingsloader.py => settings_loader.py} (81%) diff --git a/src/application/common/content.py b/src/application/common/content.py index bf5a0ff..34fa710 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -2,14 +2,16 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Changes: +# - Added logging # ### +import logging import markdown import os import yaml @@ -19,25 +21,42 @@ class Content(metaclass=Singleton): + __logger = None __meta_content_separator = None + def __init__(self): + self.__logger = logging.getLogger('COMMON.CONTENT') + self.__logger.setLevel(Options().default_logging_level) + def load_data_settings_yaml(self, directory): + self.__logger.debug('load_data_settings_yaml - ' + 'Loading settings.yml from data sub-directory {0}.'.format(directory)) + return self.load_settings_yaml(os.path.join(Options().data_dir, directory)) def load_settings_yaml(self, directory): + self.__logger.debug('load_settings_yaml - Loading settings.yml from directory {0}.'.format(directory)) + return self.load_yaml(directory, 'settings.yml') - @staticmethod - def load_yaml(directory, file): + def load_yaml(self, directory, file): + self.__logger.debug('load_yaml - Loading {0} from directory {0}.'.format(file, directory)) + yaml_file = open(os.path.join(directory, file), 'r') - return yaml.load(yaml_file, Loader=yaml.SafeLoader) + content = yaml.load(yaml_file, Loader=yaml.SafeLoader) + self.__logger.debug('load_yaml - Content of yaml:\n{0}'.format(content)) + + return content def read_content(self, directory, file): content_dir = os.path.join(Options().data_dir, directory) + self.__logger.debug('read_content - Reading content file {0} from directory {0}.'.format(file, content_dir)) + content_file = open(os.path.join(content_dir, file), 'r') meta, html = self.__split_file(content_file.read()) + # No logging, already logged return meta, html @@ -45,6 +64,8 @@ def __split_file(self, data): if self.__meta_content_separator is None: self.__meta_content_separator = Options().meta_content_separator + self.__logger.debug('__split_file - Split file separator is {0}'.format(self.__meta_content_separator)) + split = data.split(self.__meta_content_separator) meta = split[0] @@ -52,8 +73,24 @@ def __split_file(self, data): if len(split) == 2: content = split[1] + else: + self.__logger.debug('__split_file - No content found.') meta_data = yaml.load(meta, Loader=yaml.SafeLoader) + self.__logger.debug('__split_file - Meta data:\n{0}'.format(meta_data)) + + self.__logger.debug('__split_file - Markdown data:\n{0}'.format(content)) content_html = markdown.markdown(content) + self.__logger.debug('__split_file - HTML:\n{0}'.format(content_html)) return meta_data, content_html + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/common/logging_loader.py b/src/application/common/logging_loader.py deleted file mode 100644 index 7f463e6..0000000 --- a/src/application/common/logging_loader.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging.config -import cherrypy - -from common.content import Content - -# https://stackoverflow.com/questions/41879512/cherrypy-is-not-respecting-desired-logging-format - - -class LoggingLoader: - @staticmethod - def configure(): - cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files) - - logging_settings = Content().load_yaml('./src/data/logging', 'settings.yml') - logging.config.dictConfig(logging_settings) diff --git a/src/application/common/options.py b/src/application/common/options.py index f19777e..8251de4 100644 --- a/src/application/common/options.py +++ b/src/application/common/options.py @@ -35,6 +35,8 @@ class Options(metaclass=Singleton): uid = 0 gid = 0 + default_logging_level = '' + ### # # Version: 1.1.0 diff --git a/src/application/controller/application.py b/src/application/controller/application.py index 6a5a661..3199ff4 100644 --- a/src/application/controller/application.py +++ b/src/application/controller/application.py @@ -2,61 +2,95 @@ # # Full history: see below # -# Version: 2.0.0 -# Date: 2020-04-13 +# Version: 2.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This is a split of the original application.py in the root directory, containing only the Application class +# Added logging of the requests for performance insights # ### import cherrypy +import logging +from datetime import datetime + +from common.options import Options from common.singleton import Singleton -from controller.dataloader import DataLoader +from controller.data_loader import DataLoader from view.templateloader import TemplateLoader class Application(metaclass=Singleton): + __logger = None + + def __init__(self): + self.__logger = logging.getLogger('APPLICATION') + self.__logger.setLevel(Options().default_logging_level) + @cherrypy.expose def index(self, page_index=1, **_): + request = '/index/{0}'.format(page_index) + start = datetime.now() + data = DataLoader().index_data(page_index) - data['url'] = '/index/{0}'.format(page_index) + data['url'] = request template = TemplateLoader().get_template('screen_index.html') rendered = template.render(data=data) + finished = datetime.now() + self.__logger.info('{0} {1}'.format(request, finished - start)) + return rendered @cherrypy.expose def pages(self, page, **_): + request = '/pages/{0}'.format(page) + start = datetime.now() + # page on the URL: http://www.yoursite.ext/pages/page data = DataLoader().pages_data(page) - data['url'] = '/pages/{0}'.format(page) + data['url'] = request template = TemplateLoader().get_template('screen_page.html') rendered = template.render(data=data) + finished = datetime.now() + self.__logger.info('{0} {1}'.format(request, finished - start)) + return rendered @cherrypy.expose def posts(self, post, **_): + request = '/posts/{0}'.format(post) + start = datetime.now() + data = DataLoader().posts_data(post) - data['url'] = '/posts/{0}'.format(post) + data['url'] = request template = TemplateLoader().get_template('screen_post.html') rendered = template.render(data=data) + finished = datetime.now() + self.__logger.info('{0} {1}'.format(request, finished - start)) + return rendered @cherrypy.expose def tags(self, tag, page_index=1, **_): + request = '/tags/{0}/{1}'.format(tag, page_index) + start = datetime.now() + data = DataLoader().tags_data(tag, page_index) - data['url'] = '/tags/{0}/{1}'.format(tag, page_index) + data['url'] = request template = TemplateLoader().get_template('screen_tag.html') rendered = template.render(data=data) + finished = datetime.now() + self.__logger.info('{0} {1}'.format(request, finished - start)) + return rendered # @cherrypy.expose @@ -81,6 +115,12 @@ def tags(self, tag, page_index=1, **_): ### # +# Version: 2.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This is a split of the original application.py in the root directory, containing only the Application class +# # Version: 1.2.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/controller/dataloader.py b/src/application/controller/data_loader.py similarity index 100% rename from src/application/controller/dataloader.py rename to src/application/controller/data_loader.py diff --git a/src/application/controller/logging_loader.py b/src/application/controller/logging_loader.py new file mode 100644 index 0000000..031b761 --- /dev/null +++ b/src/application/controller/logging_loader.py @@ -0,0 +1,36 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +### + +import cherrypy +import logging +import logging.config +import os +import yaml + +from common.options import Options + +# https://stackoverflow.com/questions/41879512/cherrypy-is-not-respecting-desired-logging-format + + +class LoggingLoader: + # No need to log anything, the loader is not yet loaded so you don't have a logger + + @staticmethod + def configure(): + cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files) + + # DO NOT USE Content() here, it's not ready + settings_file = os.path.join(Options().data_dir, 'logging', 'settings.yml') + file = open(settings_file, 'r') + settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader) + + logging.config.dictConfig(settings_yaml) + + Options().default_logging_level = settings_yaml['loggers']['']['level'] diff --git a/src/application/controller/settingsloader.py b/src/application/controller/settings_loader.py similarity index 81% rename from src/application/controller/settingsloader.py rename to src/application/controller/settings_loader.py index 52b90b3..80dcdd7 100644 --- a/src/application/controller/settingsloader.py +++ b/src/application/controller/settings_loader.py @@ -2,18 +2,20 @@ # # Full history: see below # -# Version: 1.2.0 -# Date: 2020-04-13 +# Version: 1.2.1 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# Features: -# - Split between the properties needed by the engine and needed by the application +# Changes: +# Because of logging, the Content() class cannot be used # ### import os +import yaml + +# NEVER IMPORT Content from common.content => you need settings from this class here -from common.content import Content from common.options import Options from common.singleton import Singleton @@ -30,13 +32,17 @@ def parse(self): environment_dir = os.path.join(Options().data_dir, 'environment') # read the yaml file - settings_yaml = Content().load_yaml(environment_dir, '{0}.yml'.format(self.__environment)) + # DO NOT USE the Content() class here, it needs the settings itself to set the logging level + settings_file = os.path.join(environment_dir, '{0}.yml'.format(self.__environment)) + file = open(settings_file, 'r') + settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader) # set the settings needed elsewhere in the code self.__option_settings(settings_yaml) - # return the settings really needed in a format for CherryPy - # they will be used when starting the engine in application.py + # Return the settings really needed in a format for CherryPy + # They will be used when starting the engine in application.py + # They are logged in main.py return self.__engine_settings(settings_yaml) @staticmethod @@ -72,6 +78,8 @@ def __engine_settings(settings_yaml): @staticmethod def __option_settings(settings_yaml): + # The individual properties are logged in main + # No need to log them here if settings_yaml['directories']['theme']['absolute']: Options().theme_dir = settings_yaml['directories']['theme']['path'] else: @@ -97,6 +105,13 @@ def __option_settings(settings_yaml): ### # +# Version: 1.2.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Split between the properties needed by the engine and needed by the application +# # Version: 1.1.0 # Date: 2020-04-09 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/main.py b/src/application/main.py index fb958f5..22e1092 100644 --- a/src/application/main.py +++ b/src/application/main.py @@ -22,52 +22,66 @@ from cherrypy.process.plugins import Daemonizer, PIDFile, DropPrivileges from controller.application import Application -from common.logging_loader import LoggingLoader +from controller.logging_loader import LoggingLoader from common.options import Options -from controller.settingsloader import SettingsLoader +from controller.settings_loader import SettingsLoader +__application = 'CherryBlog' +__version = '1.2.0' if __name__ == '__main__': - LoggingLoader().configure() - - logger = logging.getLogger('MAIN') - environment = 'localhost' data_dir = "" opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data=']) - logger.debug('opts: {0}'.format(opts)) - logger.debug('args: {0}'.format(args)) for opt, arg in opts: if opt in ['-d', '--data']: - logger.debug('Overriding data_dir to {0}.'.format(arg)) data_dir = arg if opt in ['-e', '--env']: - logger.debug('Overriding environment to {0}.'.format(arg)) environment = arg if data_dir == '': data_dir = os.path.join(os.getcwd(), 'src', 'data') - logger.info('Environment set to {0}.'.format(environment)) - logger.info('Data directory set to {0}.'.format(data_dir)) - + # Options is a singleton and can be loaded with what we know already Options().environment = environment Options().data_dir = data_dir + # Load the settings from the environment.yml file to fill out the rest of the settings + # This also sets most of the unknown properties in Options() settings = SettingsLoader(environment).parse() + # Do not load the logging before you have the data_dir + # This will fill out the logging level in the Options() + LoggingLoader().configure() + + logger = logging.getLogger('MAIN') + + logger.info('{0} v.{1}'.format(__application, __version)) + logger.info('Environment set to {0}.'.format(environment)) + logger.info('Data directory set to {0}.'.format(data_dir)) + logger.info('Theme directory set to {0}.'.format(Options().theme_dir)) + logger.info('Log directory set to {0}.'.format(Options().log_dir)) + logger.info('Run directory set to {0}.'.format(Options().run_dir)) + logger.info('Meta-content separator set to {0}.'.format(Options().meta_content_separator)) + logger.info('Default logging level set to {0}.'.format(Options().default_logging_level)) + + logger.debug('main - CherryPy settings:\n{0}\n'.format(settings)) + if Options().daemon: + # Daemon info is logged by CherryPy daemon = Daemonizer(cherrypy.engine) daemon.subscribe() else: logger.info('Not running as daemon.') + # PID is logged by CherryPy pid = PIDFile(cherrypy.engine, os.path.join(Options().run_dir, 'cherryblog.pid')) pid.subscribe() if Options().privileges: + # Privileges are logged by CherryPy privileges = DropPrivileges(cherrypy.engine, uid=Options().uid, gid=Options().gid) privileges.subscribe() else: diff --git a/src/application/model/codeversion.py b/src/application/model/codeversion.py index df45ef4..45b8b9e 100644 --- a/src/application/model/codeversion.py +++ b/src/application/model/codeversion.py @@ -10,14 +10,22 @@ # ### +import logging + +from common.options import Options from common.content import Content from common.singleton import Singleton class CodeVersion(metaclass=Singleton): __base_dir = 'codeversion' + __logger = None data = None def __init__(self): + self.__logger = logging.getLogger('MODEL.CODE_VERSION') + self.__logger.setLevel(Options().default_logging_level) + self.data = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('__init__ - {0}'.format(self.data)) diff --git a/src/application/model/commondata.py b/src/application/model/commondata.py index 2b72b41..ebee347 100644 --- a/src/application/model/commondata.py +++ b/src/application/model/commondata.py @@ -10,6 +10,9 @@ # ### +import logging + +from common.options import Options from common.singleton import Singleton ## @@ -18,15 +21,19 @@ class CommonData(metaclass=Singleton): + __logger = None __data_loader = None def __init__(self, data_loader): + self.__logger = logging.getLogger('MODEL.COMMON_DATA') + self.__logger.setLevel(Options().default_logging_level) + self.__data_loader = data_loader def data(self): dl = self.__data_loader - return { + cd = { 'settings': dl.global_settings, 'tags_list': dl.tags_list, 'main_menu': dl.index_main_menu, @@ -34,3 +41,7 @@ def data(self): 'important_news': dl.important_news_data, 'code_version': dl.code_version_data } + + self.__logger.debug('data - {0}'.format(cd)) + + return cd diff --git a/src/application/model/important_news.py b/src/application/model/important_news.py index f4db8ad..b74fdfb 100644 --- a/src/application/model/important_news.py +++ b/src/application/model/important_news.py @@ -10,14 +10,22 @@ # ### +import logging + from common.content import Content +from common.options import Options from common.singleton import Singleton class ImportantNews(metaclass=Singleton): __base_dir = 'important_news' + __logger = None data = None def __init__(self): + self.__logger = logging.getLogger('MODEL.IMPORTANT_NEWS') + self.__logger.setLevel(Options().default_logging_level) + self.data = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('__init__ - {0}'.format(self.data)) diff --git a/src/application/model/index.py b/src/application/model/index.py index 40a7a2e..a071a20 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -10,8 +10,9 @@ # ### -import os +import logging import math +import os from pathlib import Path @@ -28,6 +29,8 @@ class Index(metaclass=Singleton): __base_dir = 'index' __main_menu_settings_dir = 'main_menu' __footer_menu_settings_dir = 'footer_menu' + + __logger = None __data_loader = None __data = {} @@ -40,24 +43,37 @@ class Index(metaclass=Singleton): highlight_posts = 0 def __init__(self, data_loader): + self.__logger = logging.getLogger('MODEL.INDEX') + self.__logger.setLevel(Options().default_logging_level) + self.__data_loader = data_loader self.main_menu = Content().load_data_settings_yaml(self.__main_menu_settings_dir) + self.__logger.debug('__init__ - main_menu: {0}'.format(self.main_menu)) + self.footer_menu = Content().load_data_settings_yaml(self.__footer_menu_settings_dir) + self.__logger.debug('__init__ - footer_menu: {0}'.format(self.footer_menu)) settings = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('__init__ - settings: {0}'.format(settings)) self.max_posts = settings['max_posts'] self.spotlight_posts = settings['spotlight_posts'] self.highlight_posts = settings['highlight_posts'] def data(self, page_index): + self.__logger.debug('data - page_index: {0}'.format(page_index)) + key_index = 'index-{0}'.format(page_index) if key_index in self.__data.keys(): + self.__logger.debug('data - index page found: {0}'.format(page_index)) return self.__data[key_index] + else: + self.__logger.debug('data - index page not found: {0}'.format(page_index)) data = self.__data_loader.common_data + self.__logger.debug('data - common_data: {0}'.format(data)) data_index = {} @@ -66,8 +82,10 @@ def data(self, page_index): data_index['image'] = intro_meta['image'] data['index'] = data_index + self.__logger.debug('data - data[index]: {0}'.format(data_index)) posts_dir = os.path.join(Options().data_dir, 'posts') + self.__logger.debug('data - posts_dir: {0}'.format(posts_dir)) data['posts'] = [] data['spotlight_posts'] = [] @@ -80,9 +98,13 @@ def data(self, page_index): max_entries = self.max_posts spotlight_entries = self.spotlight_posts highlight_entries = self.highlight_posts + self.__logger.debug('data - max_entries: {0}'.format(max_entries)) + self.__logger.debug('data - spotlight_entries: {0}'.format(spotlight_entries)) + self.__logger.debug('data - highlight_entries: {0}'.format(highlight_entries)) count_entries = 0 skip_entries = (int(page_index) - 1) * max_entries + self.__logger.debug('data - skip_entries: {0}'.format(skip_entries)) for file in sorted(os.listdir(posts_dir), reverse=True): count_entries += 1 @@ -96,24 +118,33 @@ def data(self, page_index): stem = Path(file).stem post['url'] = stem + self.__logger.debug('data - post: {0}'.format(post)) + if page_index == 1: if count_entries <= spotlight_entries: + self.__logger.debug('data - post added to spotlight_posts.') data['spotlight_posts'].append(post) if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries): + self.__logger.debug('data - post added to highlight_posts.') data['highlight_posts'].append(post) if count_entries > (spotlight_entries + highlight_entries): + self.__logger.debug('data - post added to (standard) posts.') data['posts'].append(post) else: + self.__logger.debug('data - post added to (standard) posts.') data['posts'].append(post) if count_entries == (max_entries + skip_entries): + self.__logger.debug('data - enough posts for this index page.') break total_posts = self.__data_loader.posts_count total_index_pages = math.ceil(total_posts / max_entries) + self.__logger.debug('data - total_posts: {0}'.format(total_posts)) + self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages)) data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages, @@ -122,5 +153,6 @@ def data(self, page_index): 'posts': len(data['posts'])} self.__data[key_index] = data + self.__logger.debug('data - {0}'.format(data)) return data diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 81c9654..4ea8cd8 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -10,11 +10,12 @@ # ### +import logging import os -from common.singleton import Singleton from common.content import Content from common.options import Options +from common.singleton import Singleton ## # Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! @@ -23,6 +24,8 @@ class Pages(metaclass=Singleton): __base_dir = 'pages' + + __logger = None __data_loader = None __pages = {} @@ -31,16 +34,27 @@ class Pages(metaclass=Singleton): count = 0 def __init__(self, data_loader): + self.__logger = logging.getLogger('MODEL.PAGES') + self.__logger.setLevel(Options().default_logging_level) + self.__data_loader = data_loader self.directory = os.path.join(Options().data_dir, self.__base_dir) self.count = len(os.listdir(self.directory)) + self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) + self.__logger.debug('__init__ - count: {0}'.format(self.count)) def data(self, page): + self.__logger.debug('data - page: {0}'.format(page)) + if page in self.__pages.keys(): + self.__logger.debug('data - page found: {0}'.format(page)) return self.__pages[page] + else: + self.__logger.debug('data - page not found: {0}'.format(page)) data = self.__data_loader.common_data + self.__logger.debug('data - common_data: {0}'.format(data)) meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(page)) @@ -48,5 +62,6 @@ def data(self, page): data['page'] = meta self.__pages[page] = data + self.__logger.debug('data - pages[{0}]: {1}'.format(page, data)) return data diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 09b4112..df01138 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -12,11 +12,12 @@ # ### +import logging import os -from common.singleton import Singleton from common.content import Content from common.options import Options +from common.singleton import Singleton ## # Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! @@ -25,6 +26,8 @@ class Posts(metaclass=Singleton): __base_dir = 'posts' + + __logger = None __data_loader = None __posts = {} @@ -33,16 +36,27 @@ class Posts(metaclass=Singleton): count = 0 def __init__(self, data_loader): + self.__logger = logging.getLogger('MODEL.POSTS') + self.__logger.setLevel(Options().default_logging_level) + self.__data_loader = data_loader self.directory = os.path.join(Options().data_dir, self.__base_dir) self.count = len(os.listdir(self.directory)) + self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) + self.__logger.debug('__init__ - count: {0}'.format(self.count)) def data(self, post): + self.__logger.debug('data - post: {0}'.format(post)) + if post in self.__posts.keys(): + self.__logger.debug('data - post found: {0}'.format(post)) return self.__posts[post] + else: + self.__logger.debug('data - post not found: {0}'.format(post)) data = self.__data_loader.common_data + self.__logger.debug('data - common_data: {0}'.format(data)) meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(post)) @@ -50,5 +64,6 @@ def data(self, post): data['post'] = meta self.__posts[post] = data + self.__logger.debug('data - posts[{0}]: {1}'.format(post, data)) return data diff --git a/src/application/model/settings.py b/src/application/model/settings.py index 1a06a4e..a67f37d 100644 --- a/src/application/model/settings.py +++ b/src/application/model/settings.py @@ -10,8 +10,8 @@ # ### +import logging import os -import yaml from common.content import Content from common.options import Options @@ -20,10 +20,16 @@ class Settings(metaclass=Singleton): __base_dir = 'settings' + __logger = None global_settings = None def __init__(self): + self.__logger = logging.getLogger('MODEL.SETTINGS') + self.__logger.setLevel(Options().default_logging_level) + settings_dir = os.path.join(Options().data_dir, self.__base_dir) + self.__logger.debug('__init__ - settings_dir: {0}'.format(settings_dir)) self.global_settings = Content().load_yaml(settings_dir, 'global.yml') + self.__logger.debug('__init__ - global_settings: {0}'.format(self.global_settings)) diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 9c9ac89..22673c1 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -10,6 +10,7 @@ # ### +import logging import math import os import string @@ -18,23 +19,31 @@ from pathlib import Path from common.content import Content +from common.options import Options from common.singleton import Singleton class Tags(metaclass=Singleton): __base_dir = 'tags' + + __logger = None __data_loader = None list = None def __init__(self, data_loader): + self.__logger = logging.getLogger('MODEL.TAGS') + self.__logger.setLevel(Options().default_logging_level) + self.__data_loader = data_loader self.__build_list() def __build_list(self): settings = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('__build_list - settings: {0}'.format(settings)) posts_dir = self.__data_loader.posts_directory + self.__logger.debug('__build_list - posts_dir: {0}'.format(posts_dir)) # Starting with a dictionary as this is the easiest to find existing tags tags = {} @@ -46,16 +55,20 @@ def __build_list(self): label = self.__tag_label(tag) if label in settings['skip_tags']: + self.__logger.debug('__build_list - tag {0} found in skip_tags'.format(tag)) continue if label in tags.keys(): + self.__logger.debug('__build_list - tag {0} already exists, +1'.format(tag)) current_count = tags[label]['count'] - tags[label]['count'] = current_count + 1 else: + self.__logger.debug('__build_list - tag {0} does not already exist'.format(tag)) data = {'label': label, 'count': 1, 'text': string.capwords(tag)} tags[label] = data + self.__logger.debug('__build_list - tags: '.format(tags)) + # Pushing this into a simple array for Jinja2 tags_array = [] @@ -63,20 +76,29 @@ def __build_list(self): tags_array.append(value) self.list = sorted(tags_array, key=itemgetter('count'), reverse=True) + self.__logger.debug('__build_list - sorted tags: '.format(self.list)) def data(self, tag, page_index): + self.__logger.debug('data - tag: {0}'.format(tag)) + self.__logger.debug('data - page_index tags: {0}'.format(page_index)) + data = self.__data_loader.common_data + self.__logger.debug('data - common_data: {0}'.format(data)) posts_dir = self.__data_loader.posts_directory + self.__logger.debug('data - posts_dir: {0}'.format(posts_dir)) data['posts'] = [] - max_entries = self.__data_loader.index_max_posts count_entries = 0 + max_entries = self.__data_loader.index_max_posts skip_entries = (int(page_index) - 1) * max_entries + self.__logger.debug('data - max_entries: {0}'.format(max_entries)) + self.__logger.debug('data - skip_entries: {0}'.format(skip_entries)) for file in sorted(os.listdir(posts_dir), reverse=True): post, post['content'] = Content().read_content(posts_dir, file) + self.__logger.debug('data - post: {0}'.format(post)) must_include = False @@ -85,12 +107,17 @@ def data(self, tag, page_index): must_include = True break + self.__logger.debug('data - must_include: {0}'.format(must_include)) + if must_include: count_entries += 1 # We count the entries, but for pages 2 and more, you don't show them if skip_entries >= count_entries: + self.__logger.debug('data - post skipped}') continue + else: + self.__logger.debug('data - post added') stem = Path(file).stem post['url'] = stem @@ -98,19 +125,27 @@ def data(self, tag, page_index): data['posts'].append(post) if count_entries == max_entries: + self.__logger.debug('data - enough posts') break data['tag'] = {'name': self.__tag_text(tag), 'path': tag} total_posts = self.count_posts(tag) total_index_pages = math.ceil(total_posts / max_entries) + self.__logger.debug('data - total_posts: {0}'.format(total_posts)) + self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages)) data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages} + self.__logger.debug('data - {0}'.format(data)) + return data def count_posts(self, tag): + self.__logger.debug('count_posts - tag: {0}'.format(tag)) + posts_dir = self.__data_loader.posts_directory + self.__logger.debug('count_posts - posts_dir: {0}'.format(posts_dir)) count_entries = 0 @@ -120,6 +155,10 @@ def count_posts(self, tag): for tag_raw in post['tags']: if self.__tag_label(tag_raw) == tag: count_entries += 1 + self.__logger.debug('count_posts - file {0} includes tag '.format(file)) + break + + self.__logger.debug('count_posts - tag {0} has {1} posts'.format(tag, count_entries)) return count_entries diff --git a/src/application/view/templateloader.py b/src/application/view/templateloader.py index c85d774..c080249 100644 --- a/src/application/view/templateloader.py +++ b/src/application/view/templateloader.py @@ -11,6 +11,8 @@ # ### +import logging + from jinja2 import Environment, FileSystemLoader from common.options import Options @@ -18,12 +20,19 @@ class TemplateLoader(metaclass=Singleton): + __logger = None __environment = None def __init__(self): + self.__logger = logging.getLogger('VIEW.TEMPLATE_LOADER') + self.__logger.setLevel(Options().default_logging_level) + + # Theme dir already logged in main self.__environment = Environment(loader=FileSystemLoader(Options().theme_dir)) def get_template(self, file): + self.__logger.debug('get_template - Template file {0}'.format(file)) + return self.__environment.get_template(file) ### diff --git a/src/data/logging/settings.yml b/src/data/logging/settings.yml index 4c4aeae..a4440aa 100644 --- a/src/data/logging/settings.yml +++ b/src/data/logging/settings.yml @@ -55,7 +55,7 @@ handlers: loggers: cherrypy.access: - handlers: [cherrypy_console, cherrypy_access] + handlers: [cherrypy_access] level: INFO propagate: no From 9873192f1e164fe3759d32520d8216f8a09c1bfd Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:26:15 +0200 Subject: [PATCH 08/20] Updated comment --- src/application/main.py | 22 ++++++++++++++----- src/application/model/codeversion.py | 16 +++++++++++--- src/application/model/commondata.py | 17 +++++++++++--- src/application/model/important_news.py | 17 +++++++++++--- src/application/model/index.py | 17 +++++++++++--- src/application/model/pages.py | 17 +++++++++++--- src/application/model/posts.py | 21 +++++++++++++----- src/application/model/settings.py | 17 +++++++++++--- src/application/model/tags.py | 17 +++++++++++--- src/application/view/templateloader.py | 14 ++++++++---- .../settings.yml | 0 11 files changed, 139 insertions(+), 36 deletions(-) rename src/data/{important_news => important_newsss}/settings.yml (100%) diff --git a/src/application/main.py b/src/application/main.py index 22e1092..4b00e0a 100644 --- a/src/application/main.py +++ b/src/application/main.py @@ -2,14 +2,11 @@ # # Full history: see below # -# Version: 2.0.0 -# Date: 2020-04-13 +# Version: 2.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# main.py is the entry point of the application and is derived from the original application.py file -# It has been split into main.py containing only the main() function -# and application.py in controller that maps the URLs -# History of the file is kept in application.py +# Added logging # ### @@ -88,3 +85,16 @@ logger.info('No user privileges specified.') cherrypy.quickstart(Application(), config=settings) + +### +# +# Version: 2.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# main.py is the entry point of the application and is derived from the original application.py file +# It has been split into main.py containing only the main() function +# and application.py in controller that maps the URLs +# History of the file is kept in application.py +# +### diff --git a/src/application/model/codeversion.py b/src/application/model/codeversion.py index 45b8b9e..b37601f 100644 --- a/src/application/model/codeversion.py +++ b/src/application/model/codeversion.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -29,3 +29,13 @@ def __init__(self): self.data = Content().load_data_settings_yaml(self.__base_dir) self.__logger.debug('__init__ - {0}'.format(self.data)) + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/model/commondata.py b/src/application/model/commondata.py index ebee347..9c8e6ca 100644 --- a/src/application/model/commondata.py +++ b/src/application/model/commondata.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -45,3 +45,14 @@ def data(self): self.__logger.debug('data - {0}'.format(cd)) return cd + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/model/important_news.py b/src/application/model/important_news.py index b74fdfb..64ed5f2 100644 --- a/src/application/model/important_news.py +++ b/src/application/model/important_news.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -29,3 +29,14 @@ def __init__(self): self.data = Content().load_data_settings_yaml(self.__base_dir) self.__logger.debug('__init__ - {0}'.format(self.data)) + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/model/index.py b/src/application/model/index.py index a071a20..d381ef2 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -156,3 +156,14 @@ def data(self, page_index): self.__logger.debug('data - {0}'.format(data)) return data + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 4ea8cd8..77f8029 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -65,3 +65,14 @@ def data(self, page): self.__logger.debug('data - pages[{0}]: {1}'.format(page, data)) return data + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/model/posts.py b/src/application/model/posts.py index df01138..4c35cb4 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -2,13 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# Features: -# - This class was split of the DataLoader class -# - Data stored in memory +# Added logging # ### @@ -67,3 +65,16 @@ def data(self, post): self.__logger.debug('data - posts[{0}]: {1}'.format(post, data)) return data + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - This class was split of the DataLoader class +# - Data stored in memory +# +### diff --git a/src/application/model/settings.py b/src/application/model/settings.py index a67f37d..fc3dd1c 100644 --- a/src/application/model/settings.py +++ b/src/application/model/settings.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -33,3 +33,14 @@ def __init__(self): self.global_settings = Content().load_yaml(settings_dir, 'global.yml') self.__logger.debug('__init__ - global_settings: {0}'.format(self.global_settings)) + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### + diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 22673c1..5eb19d9 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -2,11 +2,11 @@ # # Full history: see below # -# Version: 1.0.0 -# Date: 2020-04-13 +# Version: 1.1.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# This class was split of the DataLoader class +# Added logging # ### @@ -169,3 +169,14 @@ def __tag_label(tag): @staticmethod def __tag_text(tag): return string.capwords(tag.replace('-', ' ')) + + +### +# +# Version: 1.0.0 +# Date: 2020-04-13 +# Author: Yves Vindevogel (vindevoy) +# +# This class was split of the DataLoader class +# +### diff --git a/src/application/view/templateloader.py b/src/application/view/templateloader.py index c080249..4eedd21 100644 --- a/src/application/view/templateloader.py +++ b/src/application/view/templateloader.py @@ -2,12 +2,11 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-09 +# Version: 1.2.0 +# Date: 2020-04-15 # Author: Yves Vindevogel (vindevoy) # -# Features: -# - Dynamic paths for data and theme +# Added logging # ### @@ -37,6 +36,13 @@ def get_template(self, file): ### # +# Version: 1.1.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Dynamic paths for data and theme +# # Version: 1.0.0 # Date: 2020-04-07 # Author: Yves Vindevogel (vindevoy) diff --git a/src/data/important_news/settings.yml b/src/data/important_newsss/settings.yml similarity index 100% rename from src/data/important_news/settings.yml rename to src/data/important_newsss/settings.yml From f4f2ca4850966f0f0800e103161bf5bb02a1f9c4 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:27:50 +0200 Subject: [PATCH 09/20] Issue-120: Problem with missing content types solved --- src/application/common/content.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/application/common/content.py b/src/application/common/content.py index 34fa710..606b5e0 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -8,6 +8,7 @@ # # Changes: # - Added logging +# - Added try except on read of the files in case they don't exist # ### @@ -42,7 +43,12 @@ def load_settings_yaml(self, directory): def load_yaml(self, directory, file): self.__logger.debug('load_yaml - Loading {0} from directory {0}.'.format(file, directory)) - yaml_file = open(os.path.join(directory, file), 'r') + # If the file cannot be read (for instance when the user deleted the directory in data) + # don't care, return blanks + try: + yaml_file = open(os.path.join(directory, file), 'r') + except FileNotFoundError: + return {} content = yaml.load(yaml_file, Loader=yaml.SafeLoader) self.__logger.debug('load_yaml - Content of yaml:\n{0}'.format(content)) @@ -53,7 +59,10 @@ def read_content(self, directory, file): content_dir = os.path.join(Options().data_dir, directory) self.__logger.debug('read_content - Reading content file {0} from directory {0}.'.format(file, content_dir)) - content_file = open(os.path.join(content_dir, file), 'r') + try: + content_file = open(os.path.join(content_dir, file), 'r') + except FileNotFoundError: + return {}, '' meta, html = self.__split_file(content_file.read()) # No logging, already logged From f70eb0bb0ba4039c320dbb3c59a9286a4399b95c Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:29:14 +0200 Subject: [PATCH 10/20] Issue-120: forgot rename --- src/data/{important_newsss => important_news}/settings.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/data/{important_newsss => important_news}/settings.yml (100%) diff --git a/src/data/important_newsss/settings.yml b/src/data/important_news/settings.yml similarity index 100% rename from src/data/important_newsss/settings.yml rename to src/data/important_news/settings.yml From e494faa9e0b46a39786c6cba203317872b4ed993 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:35:17 +0200 Subject: [PATCH 11/20] Issue-124: Fixed bug on NoneType when yaml is empty --- src/application/common/content.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/application/common/content.py b/src/application/common/content.py index 606b5e0..7d91e54 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -51,6 +51,11 @@ def load_yaml(self, directory, file): return {} content = yaml.load(yaml_file, Loader=yaml.SafeLoader) + + # In case there's an empty file, yaml returns a None instead of a empty structure + if content is None: + content = {} + self.__logger.debug('load_yaml - Content of yaml:\n{0}'.format(content)) return content From 0aefc2cca9e19ca6d1e2cad78e05d48b5e26d364 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:45:40 +0200 Subject: [PATCH 12/20] Issue-125: Solved problem if posts is not found --- src/application/model/index.py | 51 ++++++++-------- src/application/model/pages.py | 7 ++- src/application/model/posts.py | 7 ++- src/application/model/tags.py | 107 ++++++++++++++++++--------------- 4 files changed, 97 insertions(+), 75 deletions(-) diff --git a/src/application/model/index.py b/src/application/model/index.py index d381ef2..22d487f 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -106,40 +106,43 @@ def data(self, page_index): skip_entries = (int(page_index) - 1) * max_entries self.__logger.debug('data - skip_entries: {0}'.format(skip_entries)) - for file in sorted(os.listdir(posts_dir), reverse=True): - count_entries += 1 + try: + for file in sorted(os.listdir(posts_dir), reverse=True): + count_entries += 1 - # We count the entries, but for pages 2 and more, you don't show them - if skip_entries >= count_entries: - continue + # We count the entries, but for pages 2 and more, you don't show them + if skip_entries >= count_entries: + continue - post, post['content'] = Content().read_content(posts_dir, file) + post, post['content'] = Content().read_content(posts_dir, file) - stem = Path(file).stem - post['url'] = stem + stem = Path(file).stem + post['url'] = stem - self.__logger.debug('data - post: {0}'.format(post)) + self.__logger.debug('data - post: {0}'.format(post)) - if page_index == 1: - if count_entries <= spotlight_entries: - self.__logger.debug('data - post added to spotlight_posts.') - data['spotlight_posts'].append(post) + if page_index == 1: + if count_entries <= spotlight_entries: + self.__logger.debug('data - post added to spotlight_posts.') + data['spotlight_posts'].append(post) - if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries): - self.__logger.debug('data - post added to highlight_posts.') - data['highlight_posts'].append(post) + if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries): + self.__logger.debug('data - post added to highlight_posts.') + data['highlight_posts'].append(post) - if count_entries > (spotlight_entries + highlight_entries): + if count_entries > (spotlight_entries + highlight_entries): + self.__logger.debug('data - post added to (standard) posts.') + data['posts'].append(post) + + else: self.__logger.debug('data - post added to (standard) posts.') data['posts'].append(post) - else: - self.__logger.debug('data - post added to (standard) posts.') - data['posts'].append(post) - - if count_entries == (max_entries + skip_entries): - self.__logger.debug('data - enough posts for this index page.') - break + if count_entries == (max_entries + skip_entries): + self.__logger.debug('data - enough posts for this index page.') + break + except FileNotFoundError: + pass total_posts = self.__data_loader.posts_count total_index_pages = math.ceil(total_posts / max_entries) diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 77f8029..35b9df3 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -40,7 +40,12 @@ def __init__(self, data_loader): self.__data_loader = data_loader self.directory = os.path.join(Options().data_dir, self.__base_dir) - self.count = len(os.listdir(self.directory)) + + try: + self.count = len(os.listdir(self.directory)) + except FileNotFoundError: + self.count = 0 + self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) self.__logger.debug('__init__ - count: {0}'.format(self.count)) diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 4c35cb4..840f3fe 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -40,7 +40,12 @@ def __init__(self, data_loader): self.__data_loader = data_loader self.directory = os.path.join(Options().data_dir, self.__base_dir) - self.count = len(os.listdir(self.directory)) + + try: + self.count = len(os.listdir(self.directory)) + except FileNotFoundError: + self.count = 0 + self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) self.__logger.debug('__init__ - count: {0}'.format(self.count)) diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 5eb19d9..2e69d86 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -48,24 +48,27 @@ def __build_list(self): # Starting with a dictionary as this is the easiest to find existing tags tags = {} - for file in os.listdir(posts_dir): - meta, _ = Content().read_content(posts_dir, file) # No need to catch the content - - for tag in meta['tags']: - label = self.__tag_label(tag) - - if label in settings['skip_tags']: - self.__logger.debug('__build_list - tag {0} found in skip_tags'.format(tag)) - continue - - if label in tags.keys(): - self.__logger.debug('__build_list - tag {0} already exists, +1'.format(tag)) - current_count = tags[label]['count'] - tags[label]['count'] = current_count + 1 - else: - self.__logger.debug('__build_list - tag {0} does not already exist'.format(tag)) - data = {'label': label, 'count': 1, 'text': string.capwords(tag)} - tags[label] = data + try: + for file in os.listdir(posts_dir): + meta, _ = Content().read_content(posts_dir, file) # No need to catch the content + + for tag in meta['tags']: + label = self.__tag_label(tag) + + if label in settings['skip_tags']: + self.__logger.debug('__build_list - tag {0} found in skip_tags'.format(tag)) + continue + + if label in tags.keys(): + self.__logger.debug('__build_list - tag {0} already exists, +1'.format(tag)) + current_count = tags[label]['count'] + tags[label]['count'] = current_count + 1 + else: + self.__logger.debug('__build_list - tag {0} does not already exist'.format(tag)) + data = {'label': label, 'count': 1, 'text': string.capwords(tag)} + tags[label] = data + except FileNotFoundError: + pass self.__logger.debug('__build_list - tags: '.format(tags)) @@ -96,37 +99,40 @@ def data(self, tag, page_index): self.__logger.debug('data - max_entries: {0}'.format(max_entries)) self.__logger.debug('data - skip_entries: {0}'.format(skip_entries)) - for file in sorted(os.listdir(posts_dir), reverse=True): - post, post['content'] = Content().read_content(posts_dir, file) - self.__logger.debug('data - post: {0}'.format(post)) + try: + for file in sorted(os.listdir(posts_dir), reverse=True): + post, post['content'] = Content().read_content(posts_dir, file) + self.__logger.debug('data - post: {0}'.format(post)) - must_include = False + must_include = False - for tag_raw in post['tags']: - if self.__tag_label(tag_raw) == tag: - must_include = True - break + for tag_raw in post['tags']: + if self.__tag_label(tag_raw) == tag: + must_include = True + break - self.__logger.debug('data - must_include: {0}'.format(must_include)) + self.__logger.debug('data - must_include: {0}'.format(must_include)) - if must_include: - count_entries += 1 + if must_include: + count_entries += 1 - # We count the entries, but for pages 2 and more, you don't show them - if skip_entries >= count_entries: - self.__logger.debug('data - post skipped}') - continue - else: - self.__logger.debug('data - post added') + # We count the entries, but for pages 2 and more, you don't show them + if skip_entries >= count_entries: + self.__logger.debug('data - post skipped}') + continue + else: + self.__logger.debug('data - post added') - stem = Path(file).stem - post['url'] = stem + stem = Path(file).stem + post['url'] = stem - data['posts'].append(post) + data['posts'].append(post) - if count_entries == max_entries: - self.__logger.debug('data - enough posts') - break + if count_entries == max_entries: + self.__logger.debug('data - enough posts') + break + except FileNotFoundError: + pass data['tag'] = {'name': self.__tag_text(tag), 'path': tag} @@ -149,14 +155,17 @@ def count_posts(self, tag): count_entries = 0 - for file in os.listdir(posts_dir): - post, _ = Content().read_content(posts_dir, file) - - for tag_raw in post['tags']: - if self.__tag_label(tag_raw) == tag: - count_entries += 1 - self.__logger.debug('count_posts - file {0} includes tag '.format(file)) - break + try: + for file in os.listdir(posts_dir): + post, _ = Content().read_content(posts_dir, file) + + for tag_raw in post['tags']: + if self.__tag_label(tag_raw) == tag: + count_entries += 1 + self.__logger.debug('count_posts - file {0} includes tag '.format(file)) + break + except FileNotFoundError: + pass self.__logger.debug('count_posts - tag {0} has {1} posts'.format(tag, count_entries)) From 036e7bd77d5ecd04ea18a71c8253ed2cc10dd0a3 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:54:42 +0200 Subject: [PATCH 13/20] Issue-123: No posts on index when there are no posts, similar with tags --- src/application/model/commondata.py | 1 + src/theme/default/elements/_widget/tags.html | 82 ++++++++++--------- .../default/elements/index/posts_title.html | 4 +- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/application/model/commondata.py b/src/application/model/commondata.py index 9c8e6ca..c33a8a3 100644 --- a/src/application/model/commondata.py +++ b/src/application/model/commondata.py @@ -36,6 +36,7 @@ def data(self): cd = { 'settings': dl.global_settings, 'tags_list': dl.tags_list, + 'tags_list_count': len(dl.tags_list), 'main_menu': dl.index_main_menu, 'footer_menu': dl.index_footer_menu, 'important_news': dl.important_news_data, diff --git a/src/theme/default/elements/_widget/tags.html b/src/theme/default/elements/_widget/tags.html index da5f480..5038c6b 100644 --- a/src/theme/default/elements/_widget/tags.html +++ b/src/theme/default/elements/_widget/tags.html @@ -1,51 +1,53 @@ -
        -
        {{ data.settings.widget.tags.title }}
        -
        -
        -
        -
          - {% for tag in data.tags_list %} -
        • - {{ tag.text }} ({{ tag.count }}) -
          • - {% endfor %} -
        -
        - - -
        -
          - {% set count = namespace(value=0) %} - - {% for tag in data.tags_list %} - {% if count.value % 2 == 0 %} +{% if data.tags_list_count > 0 %} +
          +
          {{ data.settings.widget.tags.title }}
          +
          +
          +
          +
            + {% for tag in data.tags_list %}
          • {{ tag.text }} ({{ tag.count }})
            • - {% endif %} + {% endfor %} +
          +
          - {% set count.value = count.value + 1 %} - {% endfor %} -
        -
        + +
        +
          + {% set count = namespace(value=0) %} - -
          -
            - {% set count = namespace(value=0) %} + {% for tag in data.tags_list %} + {% if count.value % 2 == 0 %} +
          • + {{ tag.text }} ({{ tag.count }}) +
            • + {% endif %} - {% for tag in data.tags_list %} - {% if count.value % 2 == 1 %} -
          • - {{ tag.text }} ({{ tag.count }}) -
            • - {% endif %} + {% set count.value = count.value + 1 %} + {% endfor %} +
          +
          + + +
          +
            + {% set count = namespace(value=0) %} + + {% for tag in data.tags_list %} + {% if count.value % 2 == 1 %} +
          • + {{ tag.text }} ({{ tag.count }}) +
            • + {% endif %} - {% set count.value = count.value + 1 %} - {% endfor %} -
          + {% set count.value = count.value + 1 %} + {% endfor %} +
        +
        -
        +{% endif %} \ No newline at end of file diff --git a/src/theme/default/elements/index/posts_title.html b/src/theme/default/elements/index/posts_title.html index 68944e1..d1dd82f 100644 --- a/src/theme/default/elements/index/posts_title.html +++ b/src/theme/default/elements/index/posts_title.html @@ -3,7 +3,9 @@ {% with primary_text = data.settings.index.primary_title_posts, secondary_text = data.settings.index.secondary_title_posts %} - {% include 'elements/_html/title.html' %} + {% if data.pagination.spotlight_posts > 0 %} + {% include 'elements/_html/title.html' %} + {% endif %} {% endwith %} {% endif %} From 9e60e1c299677ebcafb5ad238a8e7eb209348010 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 22:57:54 +0200 Subject: [PATCH 14/20] Issue-128: staticfiles --- src/application/controller/settings_loader.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/application/controller/settings_loader.py b/src/application/controller/settings_loader.py index 80dcdd7..92b439d 100644 --- a/src/application/controller/settings_loader.py +++ b/src/application/controller/settings_loader.py @@ -69,6 +69,17 @@ def __engine_settings(settings_yaml): settings[url] = {'tools.staticdir.on': True, 'tools.staticdir.dir': path} + for staticfile in settings_yaml['tools']['staticfiles']: + url = staticfile['url'] + absolute = staticfile['absolute'] + path = staticfile['path'] + + if not absolute: + path = os.path.join(os.getcwd(), path) + + settings[url] = {'tools.staticfile.on': True, + 'tools.staticfile.filename': path} + settings['/favicon.ico'] = { 'tools.staticfile.on': True, 'tools.staticfile.filename': os.path.join(Options().data_dir, 'images', 'favicon.ico') From feea52eceaa1e2912a7e78e284ce5479fecce5fb Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Wed, 15 Apr 2020 23:02:14 +0200 Subject: [PATCH 15/20] Issue-121: Favicon with relative path --- src/application/controller/settings_loader.py | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/application/controller/settings_loader.py b/src/application/controller/settings_loader.py index 92b439d..66eefbd 100644 --- a/src/application/controller/settings_loader.py +++ b/src/application/controller/settings_loader.py @@ -58,31 +58,37 @@ def __engine_settings(settings_yaml): 'global': global_settings } - for staticdir in settings_yaml['tools']['staticdirs']: - url = staticdir['url'] - absolute = staticdir['absolute'] - path = staticdir['path'] - - if not absolute: - path = os.path.join(os.getcwd(), path) - - settings[url] = {'tools.staticdir.on': True, - 'tools.staticdir.dir': path} - - for staticfile in settings_yaml['tools']['staticfiles']: - url = staticfile['url'] - absolute = staticfile['absolute'] - path = staticfile['path'] - - if not absolute: - path = os.path.join(os.getcwd(), path) - - settings[url] = {'tools.staticfile.on': True, - 'tools.staticfile.filename': path} + try: + for staticdir in settings_yaml['tools']['staticdirs']: + url = staticdir['url'] + absolute = staticdir['absolute'] + path = staticdir['path'] + + if not absolute: + path = os.path.abspath(os.path.join(os.getcwd(), path)) + + settings[url] = {'tools.staticdir.on': True, + 'tools.staticdir.dir': path} + except KeyError: + pass + + try: + for staticfile in settings_yaml['tools']['staticfiles']: + url = staticfile['url'] + absolute = staticfile['absolute'] + path = staticfile['path'] + + if not absolute: + path = os.path.abspath(os.path.join(os.getcwd(), path)) + + settings[url] = {'tools.staticfile.on': True, + 'tools.staticfile.filename': path} + except KeyError: + pass settings['/favicon.ico'] = { 'tools.staticfile.on': True, - 'tools.staticfile.filename': os.path.join(Options().data_dir, 'images', 'favicon.ico') + 'tools.staticfile.filename': os.path.abspath(os.path.join(Options().data_dir, 'images', 'favicon.ico')) } return settings From ba0ce9d7d959d78167209620ebcd8b92ad86bfd5 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Fri, 17 Apr 2020 09:45:15 +0200 Subject: [PATCH 16/20] Issue-138: Caching implemented on DataLoader instead of in the model classes --- src/application/controller/data_loader.py | 193 ++++++++++++++++++---- src/application/model/codeversion.py | 23 ++- src/application/model/commondata.py | 59 ------- src/application/model/important_news.py | 24 ++- src/application/model/index.py | 89 +++++----- src/application/model/pages.py | 54 +++--- src/application/model/posts.py | 54 +++--- src/application/model/settings.py | 27 ++- src/application/model/tags.py | 66 ++++---- 9 files changed, 339 insertions(+), 250 deletions(-) delete mode 100644 src/application/model/commondata.py diff --git a/src/application/controller/data_loader.py b/src/application/controller/data_loader.py index 605c2d2..c5d3bfa 100644 --- a/src/application/controller/data_loader.py +++ b/src/application/controller/data_loader.py @@ -2,23 +2,20 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-09 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # # Features: -# - Renaming categories to tags -# - Added introduction to the index page -# - Dynamic paths to themes and data -# - Updated the path of the main menu settings file -# - Data for important_news and version widget -# - Data for footer_menu +# - Caching implemented in this class instead of model classes # ### +import logging + +from common.options import Options from common.singleton import Singleton from model.codeversion import CodeVersion -from model.commondata import CommonData from model.important_news import ImportantNews from model.index import Index from model.pages import Pages @@ -28,87 +25,217 @@ class DataLoader(metaclass=Singleton): + __logger = None + + __cached_data = {} + + def __init__(self): + self.__logger = logging.getLogger('DATA_LOADER') + self.__logger.setLevel(Options().default_logging_level) + + def __cached_already(self, key): + return key in self.__cached_data.keys() + + def __get_cached(self, key): + return self.__cached_data[key] + + def __cache(self, key, data): + self.__logger.info('Caching {0}'.format(key)) + self.__cached_data[key] = data + + def __get_data(self, key, cls, method): + if self.__cached_already(key): + return self.__get_cached(key) + + data = getattr(cls, method) + self.__cache(key, data) + + return data + + @staticmethod + def __combine(*parts): + combined = {} + + for part in parts: + for item in part.items(): + combined[item[0]] = item[1] + + return combined + # code version @property def code_version_data(self): - return CodeVersion().data + return self.__get_data('code_version_data', CodeVersion(), 'data') - # content @property def common_data(self): - return CommonData(self).data() + key = 'common_data' + + if self.__cached_already(key): + return self.__get_cached(key) + + data = { + 'settings': self.global_settings, + 'tags_list': self.tags_list, + 'tags_list_count': self.tags_list_count, + 'main_menu': self.index_main_menu, + 'footer_menu': self.index_footer_menu, + 'important_news': self.important_news_data, + 'code_version': self.code_version_data + } + + self.__cache(key, data) + return data # important news @property def important_news_data(self): - return ImportantNews().data + return self.__get_data('important_news_data', ImportantNews(), 'data') # index @property def index_main_menu(self): - return Index(self).main_menu + return self.__get_data('index_main_menu', Index(), 'main_menu') @property def index_footer_menu(self): - return Index(self).footer_menu + return self.__get_data('index_footer_menu', Index(), 'footer_menu') def index_data(self, page_index): - return Index(self).data(page_index) + key = '/index/{0}'.format(page_index) + + if self.__cached_already(key): + return self.__get_cached(key) + + common = self.common_data + data = Index().data(page_index, self.posts_directory, self.posts_count) + combined = self.__combine(common, data) + + self.__cache(key, combined) + + return combined @property def index_max_posts(self): - return Index(self).max_posts + return self.__get_data('index_max_posts', Index(), 'max_posts') @property def index_spotlight_posts(self): - return Index(self).spotlight_posts + return self.__get_data('index_spotlight_posts', Index(), 'spotlight_posts') @property def index_highlight_posts(self): - return Index(self).highlight_posts + return self.__get_data('index_highlight_posts', Index(), 'highlight_posts') # pages + @property + def pages_directory(self): + return self.__get_data('pages_directory', Pages(), 'directory') + @property def pages_count(self): - return Pages(self).count + return self.__get_data('pages_count', Pages(), 'count') def pages_data(self, page): - return Pages(self).data(page) + key = '/pages/{0}'.format(page) - @property - def pages_directory(self): - return Pages(self).directory + if self.__cached_already(key): + return self.__get_cached(key) + + common = self.common_data + data = Pages().data(page) + combined = self.__combine(common, data) + + self.__cache(key, combined) + + return combined # posts + @property + def posts_directory(self): + return self.__get_data('posts_directory', Posts(), 'directory') + @property def posts_count(self): - return Posts(self).count + return self.__get_data('posts_count', Posts(), 'count') def posts_data(self, post): - return Posts(self).data(post) + key = '/posts/{0}'.format(post) - @property - def posts_directory(self): - return Posts(self).directory + if self.__cached_already(key): + return self.__get_cached(key) + + common = self.common_data + data = Posts().data(post) + combined = self.__combine(common, data) + + self.__cache(key, combined) + + return combined # settings @property def global_settings(self): - return Settings().global_settings + return self.__get_data('global_settings', Settings(), 'data') # tags def tags_posts_count(self, tag): - return Tags(self).count_posts(tag) + key = 'tags_posts_count/{0}'.format(tag) + + if self.__cached_already(key): + return self.__get_cached(key) + + data = Tags().count_posts(self.posts_directory, tag) + + self.__cache(key, data) + + return data @property def tags_list(self): - return Tags(self).list + key = 'tags_list' + + if self.__cached_already(key): + return self.__get_cached(key) + + tags_list = Tags().list(self.posts_directory) + + self.__cache(key, tags_list) + + return tags_list + + @property + def tags_list_count(self): + return len(self.tags_list) def tags_data(self, tag, page_index): - return Tags(self).data(tag, page_index) + key = 'tags_data/{0}/{1}'.format(tag, page_index) + + if self.__cached_already(key): + return self.__get_cached(key) + + common = self.common_data + data = Tags().data(self.posts_directory, tag, page_index, self.index_max_posts, self.tags_posts_count(tag)) + combined = self.__combine(common, data) + + self.__cache(key, combined) + + return combined ### # +# Version: 1.1.0 +# Date: 2020-04-09 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Renaming categories to tags +# - Added introduction to the index page +# - Dynamic paths to themes and data +# - Updated the path of the main menu settings file +# - Data for important_news and version widget +# - Data for footer_menu +# # Version: 1.0.0 # Date: 2020-04-07 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/codeversion.py b/src/application/model/codeversion.py index b37601f..4505f43 100644 --- a/src/application/model/codeversion.py +++ b/src/application/model/codeversion.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -21,17 +22,25 @@ class CodeVersion(metaclass=Singleton): __base_dir = 'codeversion' __logger = None - data = None - def __init__(self): self.__logger = logging.getLogger('MODEL.CODE_VERSION') self.__logger.setLevel(Options().default_logging_level) - self.data = Content().load_data_settings_yaml(self.__base_dir) - self.__logger.debug('__init__ - {0}'.format(self.data)) + @property + def data(self): + content = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('data - content: {0}'.format(content)) + + return content ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/commondata.py b/src/application/model/commondata.py deleted file mode 100644 index c33a8a3..0000000 --- a/src/application/model/commondata.py +++ /dev/null @@ -1,59 +0,0 @@ -### -# -# Full history: see below -# -# Version: 1.1.0 -# Date: 2020-04-15 -# Author: Yves Vindevogel (vindevoy) -# -# Added logging -# -### - -import logging - -from common.options import Options -from common.singleton import Singleton - -## -# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! -## - - -class CommonData(metaclass=Singleton): - __logger = None - __data_loader = None - - def __init__(self, data_loader): - self.__logger = logging.getLogger('MODEL.COMMON_DATA') - self.__logger.setLevel(Options().default_logging_level) - - self.__data_loader = data_loader - - def data(self): - dl = self.__data_loader - - cd = { - 'settings': dl.global_settings, - 'tags_list': dl.tags_list, - 'tags_list_count': len(dl.tags_list), - 'main_menu': dl.index_main_menu, - 'footer_menu': dl.index_footer_menu, - 'important_news': dl.important_news_data, - 'code_version': dl.code_version_data - } - - self.__logger.debug('data - {0}'.format(cd)) - - return cd - - -### -# -# Version: 1.0.0 -# Date: 2020-04-13 -# Author: Yves Vindevogel (vindevoy) -# -# This class was split of the DataLoader class -# -### diff --git a/src/application/model/important_news.py b/src/application/model/important_news.py index 64ed5f2..28d0903 100644 --- a/src/application/model/important_news.py +++ b/src/application/model/important_news.py @@ -2,14 +2,16 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### + import logging from common.content import Content @@ -21,18 +23,26 @@ class ImportantNews(metaclass=Singleton): __base_dir = 'important_news' __logger = None - data = None - def __init__(self): self.__logger = logging.getLogger('MODEL.IMPORTANT_NEWS') self.__logger.setLevel(Options().default_logging_level) - self.data = Content().load_data_settings_yaml(self.__base_dir) - self.__logger.debug('__init__ - {0}'.format(self.data)) + @property + def data(self): + content = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('data - content: {0}'.format(content)) + return content ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/index.py b/src/application/model/index.py index 22d487f..da10ff9 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -20,10 +21,6 @@ from common.options import Options from common.singleton import Singleton -## -# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! -## - class Index(metaclass=Singleton): __base_dir = 'index' @@ -31,62 +28,65 @@ class Index(metaclass=Singleton): __footer_menu_settings_dir = 'footer_menu' __logger = None - __data_loader = None + __settings = None - __data = {} + def __init__(self): + self.__logger = logging.getLogger('MODEL.INDEX') + self.__logger.setLevel(Options().default_logging_level) - main_menu = None - footer_menu = None + def __get_settings(self): + if self.__settings is None: + content = Content().load_data_settings_yaml(self.__base_dir) + self.__logger.debug('__get_settings - content: {0}'.format(content)) - max_posts = 0 - spotlight_posts = 0 - highlight_posts = 0 + self.__settings = content - def __init__(self, data_loader): - self.__logger = logging.getLogger('MODEL.INDEX') - self.__logger.setLevel(Options().default_logging_level) + return self.__settings - self.__data_loader = data_loader + @property + def max_posts(self): + settings = self.__get_settings() + return settings['max_posts'] - self.main_menu = Content().load_data_settings_yaml(self.__main_menu_settings_dir) - self.__logger.debug('__init__ - main_menu: {0}'.format(self.main_menu)) + @property + def spotlight_posts(self): + settings = self.__get_settings() + return settings['spotlight_posts'] - self.footer_menu = Content().load_data_settings_yaml(self.__footer_menu_settings_dir) - self.__logger.debug('__init__ - footer_menu: {0}'.format(self.footer_menu)) + @property + def highlight_posts(self): + settings = self.__get_settings() + return settings['highlight_posts'] - settings = Content().load_data_settings_yaml(self.__base_dir) - self.__logger.debug('__init__ - settings: {0}'.format(settings)) + @property + def main_menu(self): + content = Content().load_data_settings_yaml(self.__main_menu_settings_dir) + self.__logger.debug('main_menu: {0}'.format(content)) - self.max_posts = settings['max_posts'] - self.spotlight_posts = settings['spotlight_posts'] - self.highlight_posts = settings['highlight_posts'] + return content - def data(self, page_index): - self.__logger.debug('data - page_index: {0}'.format(page_index)) + @property + def footer_menu(self): + content = Content().load_data_settings_yaml(self.__footer_menu_settings_dir) + self.__logger.debug('footer_menu: {0}'.format(content)) - key_index = 'index-{0}'.format(page_index) + return content - if key_index in self.__data.keys(): - self.__logger.debug('data - index page found: {0}'.format(page_index)) - return self.__data[key_index] - else: - self.__logger.debug('data - index page not found: {0}'.format(page_index)) + def data(self, page_index, posts_dir, total_posts): + self.__logger.debug('data - page_index: {0}'.format(page_index)) - data = self.__data_loader.common_data + data = {} self.__logger.debug('data - common_data: {0}'.format(data)) data_index = {} - intro_meta, data_index['introduction'] = Content().read_content('index', 'introduction.md') + intro_meta, data_index['introduction'] = Content().read_content(self.__base_dir, 'introduction.md') data_index['image'] = intro_meta['image'] data['index'] = data_index self.__logger.debug('data - data[index]: {0}'.format(data_index)) - posts_dir = os.path.join(Options().data_dir, 'posts') - self.__logger.debug('data - posts_dir: {0}'.format(posts_dir)) - data['posts'] = [] data['spotlight_posts'] = [] data['highlight_posts'] = [] @@ -144,7 +144,6 @@ def data(self, page_index): except FileNotFoundError: pass - total_posts = self.__data_loader.posts_count total_index_pages = math.ceil(total_posts / max_entries) self.__logger.debug('data - total_posts: {0}'.format(total_posts)) self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages)) @@ -155,7 +154,6 @@ def data(self, page_index): 'highlight_posts': len(data['highlight_posts']), 'posts': len(data['posts'])} - self.__data[key_index] = data self.__logger.debug('data - {0}'.format(data)) return data @@ -163,6 +161,13 @@ def data(self, page_index): ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 35b9df3..4882cd8 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -17,56 +18,44 @@ from common.options import Options from common.singleton import Singleton -## -# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! -## - class Pages(metaclass=Singleton): __base_dir = 'pages' __logger = None - __data_loader = None - __pages = {} - - directory = None - count = 0 - - def __init__(self, data_loader): + def __init__(self): self.__logger = logging.getLogger('MODEL.PAGES') self.__logger.setLevel(Options().default_logging_level) - self.__data_loader = data_loader + @property + def directory(self): + directory = os.path.join(Options().data_dir, self.__base_dir) + self.__logger.debug('directory - dir: {0}'.format(directory)) - self.directory = os.path.join(Options().data_dir, self.__base_dir) + return directory + @property + def count(self): try: - self.count = len(os.listdir(self.directory)) + count = len(os.listdir(self.directory)) except FileNotFoundError: - self.count = 0 + count = 0 + + self.__logger.debug('count - count: {0}'.format(count)) - self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) - self.__logger.debug('__init__ - count: {0}'.format(self.count)) + return count def data(self, page): self.__logger.debug('data - page: {0}'.format(page)) - if page in self.__pages.keys(): - self.__logger.debug('data - page found: {0}'.format(page)) - return self.__pages[page] - else: - self.__logger.debug('data - page not found: {0}'.format(page)) - - data = self.__data_loader.common_data - self.__logger.debug('data - common_data: {0}'.format(data)) + data = {} meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(page)) meta['content'] = content data['page'] = meta - self.__pages[page] = data self.__logger.debug('data - pages[{0}]: {1}'.format(page, data)) return data @@ -74,6 +63,13 @@ def data(self, page): ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 840f3fe..5324abd 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -17,56 +18,44 @@ from common.options import Options from common.singleton import Singleton -## -# Do NOT import the DataLoader class, or you will have a circular reference, pass it through __init__ !! -## - class Posts(metaclass=Singleton): __base_dir = 'posts' __logger = None - __data_loader = None - __posts = {} - - directory = None - count = 0 - - def __init__(self, data_loader): + def __init__(self): self.__logger = logging.getLogger('MODEL.POSTS') self.__logger.setLevel(Options().default_logging_level) - self.__data_loader = data_loader + @property + def directory(self): + directory = os.path.join(Options().data_dir, self.__base_dir) + self.__logger.debug('directory - directory: {0}'.format(directory)) - self.directory = os.path.join(Options().data_dir, self.__base_dir) + return directory + @property + def count(self): try: - self.count = len(os.listdir(self.directory)) + count = len(os.listdir(self.directory)) except FileNotFoundError: - self.count = 0 + count = 0 + + self.__logger.debug('count - count: {0}'.format(count)) - self.__logger.debug('__init__ - directory: {0}'.format(self.directory)) - self.__logger.debug('__init__ - count: {0}'.format(self.count)) + return count def data(self, post): self.__logger.debug('data - post: {0}'.format(post)) - if post in self.__posts.keys(): - self.__logger.debug('data - post found: {0}'.format(post)) - return self.__posts[post] - else: - self.__logger.debug('data - post not found: {0}'.format(post)) - - data = self.__data_loader.common_data - self.__logger.debug('data - common_data: {0}'.format(data)) + data = {} meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(post)) meta['content'] = content data['post'] = meta - self.__posts[post] = data self.__logger.debug('data - posts[{0}]: {1}'.format(post, data)) return data @@ -74,6 +63,13 @@ def data(self, post): ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) diff --git a/src/application/model/settings.py b/src/application/model/settings.py index fc3dd1c..4c2d5df 100644 --- a/src/application/model/settings.py +++ b/src/application/model/settings.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -22,20 +23,29 @@ class Settings(metaclass=Singleton): __base_dir = 'settings' __logger = None - global_settings = None - def __init__(self): self.__logger = logging.getLogger('MODEL.SETTINGS') self.__logger.setLevel(Options().default_logging_level) + @property + def data(self): settings_dir = os.path.join(Options().data_dir, self.__base_dir) - self.__logger.debug('__init__ - settings_dir: {0}'.format(settings_dir)) + self.__logger.debug('data - settings_dir: {0}'.format(settings_dir)) - self.global_settings = Content().load_yaml(settings_dir, 'global.yml') - self.__logger.debug('__init__ - global_settings: {0}'.format(self.global_settings)) + content = Content().load_yaml(settings_dir, 'global.yml') + self.__logger.debug('data - content: {0}'.format(content)) + + return content ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) @@ -43,4 +53,3 @@ def __init__(self): # This class was split of the DataLoader class # ### - diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 2e69d86..622b7d9 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -2,11 +2,12 @@ # # Full history: see below # -# Version: 1.1.0 -# Date: 2020-04-15 +# Version: 1.2.0 +# Date: 2020-04-17 # Author: Yves Vindevogel (vindevoy) # -# Added logging +# Features: +# - Caching done outside this class # ### @@ -27,23 +28,16 @@ class Tags(metaclass=Singleton): __base_dir = 'tags' __logger = None - __data_loader = None - list = None - - def __init__(self, data_loader): + def __init__(self): self.__logger = logging.getLogger('MODEL.TAGS') self.__logger.setLevel(Options().default_logging_level) - self.__data_loader = data_loader - self.__build_list() + def list(self, posts_dir): + self.__logger.debug('list - posts_dir: {0}'.format(posts_dir)) - def __build_list(self): settings = Content().load_data_settings_yaml(self.__base_dir) - self.__logger.debug('__build_list - settings: {0}'.format(settings)) - - posts_dir = self.__data_loader.posts_directory - self.__logger.debug('__build_list - posts_dir: {0}'.format(posts_dir)) + self.__logger.debug('list - settings: {0}'.format(settings)) # Starting with a dictionary as this is the easiest to find existing tags tags = {} @@ -56,21 +50,21 @@ def __build_list(self): label = self.__tag_label(tag) if label in settings['skip_tags']: - self.__logger.debug('__build_list - tag {0} found in skip_tags'.format(tag)) + self.__logger.debug('list - tag {0} found in skip_tags'.format(tag)) continue if label in tags.keys(): - self.__logger.debug('__build_list - tag {0} already exists, +1'.format(tag)) + self.__logger.debug('list - tag {0} already exists, +1'.format(tag)) current_count = tags[label]['count'] tags[label]['count'] = current_count + 1 else: - self.__logger.debug('__build_list - tag {0} does not already exist'.format(tag)) + self.__logger.debug('list - tag {0} does not already exist'.format(tag)) data = {'label': label, 'count': 1, 'text': string.capwords(tag)} tags[label] = data except FileNotFoundError: pass - self.__logger.debug('__build_list - tags: '.format(tags)) + self.__logger.debug('list - tags: '.format(tags)) # Pushing this into a simple array for Jinja2 tags_array = [] @@ -78,23 +72,22 @@ def __build_list(self): for _, value in tags.items(): # Only need the value tags_array.append(value) - self.list = sorted(tags_array, key=itemgetter('count'), reverse=True) - self.__logger.debug('__build_list - sorted tags: '.format(self.list)) + tags_list = sorted(tags_array, key=itemgetter('count'), reverse=True) + self.__logger.debug('list - sorted tags: '.format(tags_list)) - def data(self, tag, page_index): - self.__logger.debug('data - tag: {0}'.format(tag)) - self.__logger.debug('data - page_index tags: {0}'.format(page_index)) + return tags_list - data = self.__data_loader.common_data - self.__logger.debug('data - common_data: {0}'.format(data)) - - posts_dir = self.__data_loader.posts_directory + def data(self, posts_dir, tag, page_index, index_max_posts, count_tag_posts): self.__logger.debug('data - posts_dir: {0}'.format(posts_dir)) + self.__logger.debug('data - tag: {0}'.format(tag)) + self.__logger.debug('data - page_index tags: {0}'.format(page_index)) + self.__logger.debug('data - index_max_posts tags: {0}'.format(index_max_posts)) + self.__logger.debug('data - count_tag_posts tags: {0}'.format(count_tag_posts)) - data['posts'] = [] + data = {'posts': []} count_entries = 0 - max_entries = self.__data_loader.index_max_posts + max_entries = index_max_posts skip_entries = (int(page_index) - 1) * max_entries self.__logger.debug('data - max_entries: {0}'.format(max_entries)) self.__logger.debug('data - skip_entries: {0}'.format(skip_entries)) @@ -136,9 +129,7 @@ def data(self, tag, page_index): data['tag'] = {'name': self.__tag_text(tag), 'path': tag} - total_posts = self.count_posts(tag) - total_index_pages = math.ceil(total_posts / max_entries) - self.__logger.debug('data - total_posts: {0}'.format(total_posts)) + total_index_pages = math.ceil(count_tag_posts / max_entries) self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages)) data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages} @@ -147,10 +138,8 @@ def data(self, tag, page_index): return data - def count_posts(self, tag): + def count_posts(self, posts_dir, tag): self.__logger.debug('count_posts - tag: {0}'.format(tag)) - - posts_dir = self.__data_loader.posts_directory self.__logger.debug('count_posts - posts_dir: {0}'.format(posts_dir)) count_entries = 0 @@ -182,6 +171,13 @@ def __tag_text(tag): ### # +# Version: 1.1.0 +# Date: 2020-04-15 +# Author: Yves Vindevogel (vindevoy) +# +# Features: +# - Added logging +# # Version: 1.0.0 # Date: 2020-04-13 # Author: Yves Vindevogel (vindevoy) From 73928c8385f9f73b1080d4bf26087397fa6a3b5e Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Fri, 17 Apr 2020 19:57:40 +0200 Subject: [PATCH 17/20] All the "content" pages are now returning the page, but also the meta data and the raw data. For future caching --- src/application/common/content.py | 14 +++++++------- src/application/controller/data_loader.py | 7 ++++--- src/application/model/index.py | 6 +++--- src/application/model/pages.py | 6 +++--- src/application/model/posts.py | 6 +++--- src/application/model/tags.py | 6 +++--- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/application/common/content.py b/src/application/common/content.py index 7d91e54..dda2384 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -69,10 +69,10 @@ def read_content(self, directory, file): except FileNotFoundError: return {}, '' - meta, html = self.__split_file(content_file.read()) + meta, raw, html = self.__split_file(content_file.read()) # No logging, already logged - return meta, html + return meta, raw, html def __split_file(self, data): if self.__meta_content_separator is None: @@ -83,21 +83,21 @@ def __split_file(self, data): split = data.split(self.__meta_content_separator) meta = split[0] - content = "" + content_raw = "" if len(split) == 2: - content = split[1] + content_raw = split[1] else: self.__logger.debug('__split_file - No content found.') meta_data = yaml.load(meta, Loader=yaml.SafeLoader) self.__logger.debug('__split_file - Meta data:\n{0}'.format(meta_data)) - self.__logger.debug('__split_file - Markdown data:\n{0}'.format(content)) - content_html = markdown.markdown(content) + self.__logger.debug('__split_file - Markdown data:\n{0}'.format(content_raw)) + content_html = markdown.markdown(content_raw) self.__logger.debug('__split_file - HTML:\n{0}'.format(content_html)) - return meta_data, content_html + return meta_data, content_raw, content_html ### # diff --git a/src/application/controller/data_loader.py b/src/application/controller/data_loader.py index c5d3bfa..de7a5b9 100644 --- a/src/application/controller/data_loader.py +++ b/src/application/controller/data_loader.py @@ -108,7 +108,8 @@ def index_data(self, page_index): return self.__get_cached(key) common = self.common_data - data = Index().data(page_index, self.posts_directory, self.posts_count) + data, _ = Index().data(page_index, self.posts_directory, self.posts_count) + # We don't care yet about the raw intro combined = self.__combine(common, data) self.__cache(key, combined) @@ -143,7 +144,7 @@ def pages_data(self, page): return self.__get_cached(key) common = self.common_data - data = Pages().data(page) + data, _, _ = Pages().data(page) # No catching the meta and raw data yet combined = self.__combine(common, data) self.__cache(key, combined) @@ -166,7 +167,7 @@ def posts_data(self, post): return self.__get_cached(key) common = self.common_data - data = Posts().data(post) + data, _, _ = Posts().data(post) # We don't do anything with the meta and raw data yet combined = self.__combine(common, data) self.__cache(key, combined) diff --git a/src/application/model/index.py b/src/application/model/index.py index da10ff9..77dde1a 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -80,7 +80,7 @@ def data(self, page_index, posts_dir, total_posts): data_index = {} - intro_meta, data_index['introduction'] = Content().read_content(self.__base_dir, 'introduction.md') + intro_meta, raw_intro, data_index['introduction'] = Content().read_content(self.__base_dir, 'introduction.md') data_index['image'] = intro_meta['image'] @@ -114,7 +114,7 @@ def data(self, page_index, posts_dir, total_posts): if skip_entries >= count_entries: continue - post, post['content'] = Content().read_content(posts_dir, file) + post, _, post['content'] = Content().read_content(posts_dir, file) stem = Path(file).stem post['url'] = stem @@ -156,7 +156,7 @@ def data(self, page_index, posts_dir, total_posts): self.__logger.debug('data - {0}'.format(data)) - return data + return data, raw_intro ### diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 4882cd8..70ba25c 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -51,14 +51,14 @@ def data(self, page): data = {} - meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(page)) + meta, content, html = Content().read_content(self.__base_dir, '{0}.md'.format(page)) - meta['content'] = content + meta['content'] = html data['page'] = meta self.__logger.debug('data - pages[{0}]: {1}'.format(page, data)) - return data + return data, meta, content ### diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 5324abd..4d365e5 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -51,14 +51,14 @@ def data(self, post): data = {} - meta, content = Content().read_content(self.__base_dir, '{0}.md'.format(post)) + meta, content, html = Content().read_content(self.__base_dir, '{0}.md'.format(post)) - meta['content'] = content + meta['content'] = html data['post'] = meta self.__logger.debug('data - posts[{0}]: {1}'.format(post, data)) - return data + return data, meta, content ### diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 622b7d9..59615e5 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -44,7 +44,7 @@ def list(self, posts_dir): try: for file in os.listdir(posts_dir): - meta, _ = Content().read_content(posts_dir, file) # No need to catch the content + meta, _, _ = Content().read_content(posts_dir, file) # No need to catch the content for tag in meta['tags']: label = self.__tag_label(tag) @@ -94,7 +94,7 @@ def data(self, posts_dir, tag, page_index, index_max_posts, count_tag_posts): try: for file in sorted(os.listdir(posts_dir), reverse=True): - post, post['content'] = Content().read_content(posts_dir, file) + post, _, post['content'] = Content().read_content(posts_dir, file) self.__logger.debug('data - post: {0}'.format(post)) must_include = False @@ -146,7 +146,7 @@ def count_posts(self, posts_dir, tag): try: for file in os.listdir(posts_dir): - post, _ = Content().read_content(posts_dir, file) + post, _, _ = Content().read_content(posts_dir, file) for tag_raw in post['tags']: if self.__tag_label(tag_raw) == tag: From e5df4665b6234116cbc114844369a5dee357e852 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Fri, 17 Apr 2020 20:23:51 +0200 Subject: [PATCH 18/20] Issue-139: Warnings in Try Excepts --- src/application/common/content.py | 2 ++ src/application/model/index.py | 1 + src/application/model/pages.py | 2 ++ src/application/model/posts.py | 2 ++ src/application/model/tags.py | 1 + 5 files changed, 8 insertions(+) diff --git a/src/application/common/content.py b/src/application/common/content.py index dda2384..03f1cd6 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -48,6 +48,7 @@ def load_yaml(self, directory, file): try: yaml_file = open(os.path.join(directory, file), 'r') except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE YAML FILE {0}/{1}'.format(directory, file)) return {} content = yaml.load(yaml_file, Loader=yaml.SafeLoader) @@ -67,6 +68,7 @@ def read_content(self, directory, file): try: content_file = open(os.path.join(content_dir, file), 'r') except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE YAML FILE {0}/{1}'.format(directory, file)) return {}, '' meta, raw, html = self.__split_file(content_file.read()) diff --git a/src/application/model/index.py b/src/application/model/index.py index 77dde1a..a98b2a3 100644 --- a/src/application/model/index.py +++ b/src/application/model/index.py @@ -142,6 +142,7 @@ def data(self, page_index, posts_dir, total_posts): self.__logger.debug('data - enough posts for this index page.') break except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(posts_dir)) pass total_index_pages = math.ceil(total_posts / max_entries) diff --git a/src/application/model/pages.py b/src/application/model/pages.py index 70ba25c..b74b438 100644 --- a/src/application/model/pages.py +++ b/src/application/model/pages.py @@ -8,6 +8,7 @@ # # Features: # - Caching done outside this class +# - Added warning in Try Except # ### @@ -40,6 +41,7 @@ def count(self): try: count = len(os.listdir(self.directory)) except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE PAGES DIRECTORY {0}'.format(self.directory)) count = 0 self.__logger.debug('count - count: {0}'.format(count)) diff --git a/src/application/model/posts.py b/src/application/model/posts.py index 4d365e5..b61f2c8 100644 --- a/src/application/model/posts.py +++ b/src/application/model/posts.py @@ -8,6 +8,7 @@ # # Features: # - Caching done outside this class +# - Added warning on Try Except # ### @@ -40,6 +41,7 @@ def count(self): try: count = len(os.listdir(self.directory)) except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(self.directory)) count = 0 self.__logger.debug('count - count: {0}'.format(count)) diff --git a/src/application/model/tags.py b/src/application/model/tags.py index 59615e5..845bcd9 100644 --- a/src/application/model/tags.py +++ b/src/application/model/tags.py @@ -62,6 +62,7 @@ def list(self, posts_dir): data = {'label': label, 'count': 1, 'text': string.capwords(tag)} tags[label] = data except FileNotFoundError: + self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(posts_dir)) pass self.__logger.debug('list - tags: '.format(tags)) From 5c086e4f94d813e579de856fe5956008bd55b565 Mon Sep 17 00:00:00 2001 From: Yves Vindevogel Date: Fri, 17 Apr 2020 23:56:33 +0200 Subject: [PATCH 19/20] Issue-134: Split of the global settings into I8N yaml --- src/application/common/content.py | 5 +++ src/application/controller/data_loader.py | 10 ++++- src/application/model/i8n.py | 44 ++++++++++++++++++ src/data/i8n/blog.yml | 4 ++ src/data/i8n/code_version.yml | 3 ++ src/data/i8n/important_news.yml | 7 +++ src/data/i8n/index.yml | 8 ++++ src/data/i8n/meta.yml | 7 +++ src/data/i8n/navigation.yml | 8 ++++ src/data/i8n/tags.yml | 4 ++ src/data/settings/global.yml | 45 +------------------ src/theme/default/elements/_html/footer.html | 4 +- src/theme/default/elements/_html/head.html | 4 +- .../elements/_html/highlight_post.html | 8 ++-- .../default/elements/_html/navigation.html | 6 +-- .../default/elements/_html/print_head.html | 4 +- .../elements/_html/spotlight_post.html | 8 ++-- .../default/elements/_html/standard_post.html | 8 ++-- .../elements/_widget/important_news.html | 10 ++--- src/theme/default/elements/_widget/tags.html | 2 +- .../default/elements/_widget/version.html | 2 +- .../elements/index/highlight_post.html | 2 +- .../elements/index/introduction_title.html | 4 +- .../default/elements/index/pagination.html | 4 +- .../default/elements/index/posts_title.html | 10 ++--- .../elements/index/spotlight_post.html | 2 +- .../default/elements/index/standard_post.html | 2 +- src/theme/default/elements/page/meta.html | 8 ++-- src/theme/default/elements/post/meta.html | 8 ++-- .../default/elements/tag/pagination.html | 4 +- src/theme/default/elements/tag/post.html | 2 +- src/theme/default/elements/tag/title.html | 2 +- src/theme/default/screen_index.html | 4 +- 33 files changed, 154 insertions(+), 99 deletions(-) create mode 100644 src/application/model/i8n.py create mode 100644 src/data/i8n/blog.yml create mode 100644 src/data/i8n/code_version.yml create mode 100644 src/data/i8n/important_news.yml create mode 100644 src/data/i8n/index.yml create mode 100644 src/data/i8n/meta.yml create mode 100644 src/data/i8n/navigation.yml create mode 100644 src/data/i8n/tags.yml diff --git a/src/application/common/content.py b/src/application/common/content.py index 03f1cd6..f0378ea 100644 --- a/src/application/common/content.py +++ b/src/application/common/content.py @@ -40,6 +40,11 @@ def load_settings_yaml(self, directory): return self.load_yaml(directory, 'settings.yml') + def load_data_yaml(self, directory, file): + self.__logger.debug('load_data_yaml - Loading {0} from directory {1}.'.format(file, directory)) + + return self.load_yaml(os.path.join(Options().data_dir, directory), file) + def load_yaml(self, directory, file): self.__logger.debug('load_yaml - Loading {0} from directory {0}.'.format(file, directory)) diff --git a/src/application/controller/data_loader.py b/src/application/controller/data_loader.py index de7a5b9..975eca4 100644 --- a/src/application/controller/data_loader.py +++ b/src/application/controller/data_loader.py @@ -16,6 +16,7 @@ from common.options import Options from common.singleton import Singleton from model.codeversion import CodeVersion +from model.i8n import I8N from model.important_news import ImportantNews from model.index import Index from model.pages import Pages @@ -75,6 +76,7 @@ def common_data(self): return self.__get_cached(key) data = { + 'i8n': self.i8n, 'settings': self.global_settings, 'tags_list': self.tags_list, 'tags_list_count': self.tags_list_count, @@ -92,6 +94,11 @@ def common_data(self): def important_news_data(self): return self.__get_data('important_news_data', ImportantNews(), 'data') + # i8n + @property + def i8n(self): + return I8N().data + # index @property def index_main_menu(self): @@ -101,7 +108,7 @@ def index_main_menu(self): def index_footer_menu(self): return self.__get_data('index_footer_menu', Index(), 'footer_menu') - def index_data(self, page_index): + def index_data(self, page_index, language=None): key = '/index/{0}'.format(page_index) if self.__cached_already(key): @@ -109,6 +116,7 @@ def index_data(self, page_index): common = self.common_data data, _ = Index().data(page_index, self.posts_directory, self.posts_count) + # We don't care yet about the raw intro combined = self.__combine(common, data) diff --git a/src/application/model/i8n.py b/src/application/model/i8n.py new file mode 100644 index 0000000..762cae1 --- /dev/null +++ b/src/application/model/i8n.py @@ -0,0 +1,44 @@ +### +# +# Full history: see below +# +# Version: 1.0.0 +# Date: 2020-04-17 +# Author: Yves Vindevogel (vindevoy) +# +# +### + +import logging +import os + +from pathlib import Path + +from common.content import Content +from common.singleton import Singleton +from common.options import Options + + +class I8N(metaclass=Singleton): + __base_dir = 'i8n' + __logger = None + + def __init__(self): + self.__logger = logging.getLogger('MODEL.I8N') + self.__logger.setLevel(Options().default_logging_level) + + @property + def data(self): + data = {} + + i8n_dir = os.path.join(Options().data_dir, self.__base_dir) + + for file in os.scandir(i8n_dir): + stem = Path(file).stem + self.__logger.debug('data - reading file {0}.yml'.format(stem)) + + data[stem] = Content().load_data_yaml(self.__base_dir, file) + + self.__logger.debug('data - {0}'.format(data)) + + return data diff --git a/src/data/i8n/blog.yml b/src/data/i8n/blog.yml new file mode 100644 index 0000000..60bed65 --- /dev/null +++ b/src/data/i8n/blog.yml @@ -0,0 +1,4 @@ +--- + +title: "CherryBlog" +copyright: "CherryBlog is licensed under the MIT license" diff --git a/src/data/i8n/code_version.yml b/src/data/i8n/code_version.yml new file mode 100644 index 0000000..c40e26c --- /dev/null +++ b/src/data/i8n/code_version.yml @@ -0,0 +1,3 @@ +--- + +title: 'Latest version' \ No newline at end of file diff --git a/src/data/i8n/important_news.yml b/src/data/i8n/important_news.yml new file mode 100644 index 0000000..6b81b2a --- /dev/null +++ b/src/data/i8n/important_news.yml @@ -0,0 +1,7 @@ +--- + +title: 'IMPORTANT NEWS' +title_color: '#FF0000' +title_weight: 'normal' +text_color: '#000000' +text_weight: 'normal' diff --git a/src/data/i8n/index.yml b/src/data/i8n/index.yml new file mode 100644 index 0000000..6a2eae4 --- /dev/null +++ b/src/data/i8n/index.yml @@ -0,0 +1,8 @@ +--- + +primary_title_introduction: "CherryBlog.ORG" +secondary_title_introduction: "open source blogging software" +primary_title_posts: "Latest news" +secondary_title_posts: "about CherryBlog" +primary_title_archive: 'News archive' + diff --git a/src/data/i8n/meta.yml b/src/data/i8n/meta.yml new file mode 100644 index 0000000..6dedf6e --- /dev/null +++ b/src/data/i8n/meta.yml @@ -0,0 +1,7 @@ +--- + +author_prefix: 'Written by ' +author_postfix: '' +date_prefix: ' on ' +date_postfix: '' + diff --git a/src/data/i8n/navigation.yml b/src/data/i8n/navigation.yml new file mode 100644 index 0000000..a573789 --- /dev/null +++ b/src/data/i8n/navigation.yml @@ -0,0 +1,8 @@ +--- + +home_label: "Home" +read_more_label: "Read More →" +previous_page_label: '← Newer' +next_page_label: 'Older →' +page_of_page: 'page' +page_of_of: 'of' diff --git a/src/data/i8n/tags.yml b/src/data/i8n/tags.yml new file mode 100644 index 0000000..b068e6e --- /dev/null +++ b/src/data/i8n/tags.yml @@ -0,0 +1,4 @@ +--- + +widget_title: 'Tags' +secondary_title: 'posts' diff --git a/src/data/settings/global.yml b/src/data/settings/global.yml index 3d7d616..ad6adfd 100644 --- a/src/data/settings/global.yml +++ b/src/data/settings/global.yml @@ -1,46 +1,3 @@ --- -blog: - title: "CherryBlog" - copyright: "CherryBlog is licensed under the MIT license" - -index: - primary_title_introduction: "CherryBlog.ORG" - secondary_title_introduction: "open source blogging software" - primary_title_posts: "Latest news" - secondary_title_posts: "about CherryBlog" - primary_title_archive: 'News archive' - -post: - -page: - -tag: - secondary_title: 'posts' - -meta: - author_prefix: 'Written by ' - author_postfix: '' - date_prefix: ' on ' - date_postfix: '' - -navigation: - home_label: "Home" - read_more_label: "Read More →" - previous_page_label: '← Newer' - next_page_label: 'Older →' - page_of_page: 'page' - page_of_of: 'of' - -widget: - tags: - title: 'Tags' - important_news: - title: 'IMPORTANT NEWS' - title_color: '#FF0000' - title_weight: 'normal' - text_color: '#000000' - text_weight: 'normal' - - code_version: - title: 'Latest version' \ No newline at end of file +language: 'en' \ No newline at end of file diff --git a/src/theme/default/elements/_html/footer.html b/src/theme/default/elements/_html/footer.html index a1c5ec1..dc5d5e9 100644 --- a/src/theme/default/elements/_html/footer.html +++ b/src/theme/default/elements/_html/footer.html @@ -14,7 +14,7 @@
        -

        {{ data.settings.blog.copyright }}

        +

        {{ data.i8n.blog.copyright }}

@@ -33,7 +33,7 @@
-

{{ data.settings.blog.copyright }}

+

{{ data.i8n.blog.copyright }}

diff --git a/src/theme/default/elements/_html/head.html b/src/theme/default/elements/_html/head.html index 516fe4c..457c864 100644 --- a/src/theme/default/elements/_html/head.html +++ b/src/theme/default/elements/_html/head.html @@ -1,9 +1,9 @@ - + -{{ data.settings.blog.title }} +{{ data.i8n.blog.title }} diff --git a/src/theme/default/elements/_html/highlight_post.html b/src/theme/default/elements/_html/highlight_post.html index 0761223..9d01945 100644 --- a/src/theme/default/elements/_html/highlight_post.html +++ b/src/theme/default/elements/_html/highlight_post.html @@ -10,10 +10,10 @@

{{ title }}

{% with author = author, date = date, - author_prefix = data.settings.meta.author_prefix, - author_postfix = data.settings.meta.author_postfix, - date_prefix = data.settings.meta.date_prefix, - date_postfix = data.settings.meta.date_postfix + author_prefix = data.i8n.meta.author_prefix, + author_postfix = data.i8n.meta.author_postfix, + date_prefix = data.i8n.meta.date_prefix, + date_postfix = data.i8n.meta.date_postfix %} {% include 'elements/_html/card_footer.html' %} {% endwith %} diff --git a/src/theme/default/elements/_html/navigation.html b/src/theme/default/elements/_html/navigation.html index 031a349..82681d2 100644 --- a/src/theme/default/elements/_html/navigation.html +++ b/src/theme/default/elements/_html/navigation.html @@ -1,7 +1,7 @@