From f64a2ab94a502b2eedb065f8ef072b13b47756e5 Mon Sep 17 00:00:00 2001 From: Situphen Date: Sun, 3 Jul 2022 17:53:20 +0200 Subject: [PATCH] =?UTF-8?q?Optimisation=20des=20requ=C3=AAtes=20pour=20les?= =?UTF-8?q?=20statistiques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/misc/graph.part.html | 6 +- templates/tutorialv2/stats/index.html | 42 ++-- zds/tutorialv2/views/statistics.py | 325 ++++++++++++-------------- 3 files changed, 179 insertions(+), 194 deletions(-) diff --git a/templates/misc/graph.part.html b/templates/misc/graph.part.html index f7960ecebe..1cc053c2f8 100644 --- a/templates/misc/graph.part.html +++ b/templates/misc/graph.part.html @@ -9,10 +9,10 @@

{% trans graph_title %}

diff --git a/templates/tutorialv2/stats/index.html b/templates/tutorialv2/stats/index.html index eca63b1a25..5121443904 100644 --- a/templates/tutorialv2/stats/index.html +++ b/templates/tutorialv2/stats/index.html @@ -41,21 +41,21 @@

{% endif %}

- {% if reports %} + {% if graphs_data %}
{% trans "Pages vues" %} {% trans "Temps moyen de lecture" %} - {% trans "Nombre de visiteurs uniques" %} + {% trans "Nombre de visites" %}
- {% include "misc/graph.part.html" with tab_name="tab-view-graph-content" graph_title="Évolution des pages vues sur le contenu" canvas_id="view-graph" report_key="nb_hits" y_label="Nombre de pages" %} - {% include "misc/graph.part.html" with tab_name="tab-visit-time-graph-content" graph_title="Évolution du temps moyen lecture (en secondes)" canvas_id="visit-time-graph" report_key="avg_time_on_page" y_label="Secondes" %} - {% include "misc/graph.part.html" with tab_name="tab-users-graph-content" graph_title="Évolution du nombre de visiteurs uniques" canvas_id="users-graph" report_key="nb_uniq_visitors" y_label="Nombre de visiteurs" %} + {% include "misc/graph.part.html" with tab_name="tab-view-graph-content" graph_title="Évolution des pages vues sur le contenu" canvas_id="view-graph" graph_data_key="nb_hits" y_label="Nombre de pages" %} + {% include "misc/graph.part.html" with tab_name="tab-visit-time-graph-content" graph_title="Évolution du temps moyen lecture (en secondes)" canvas_id="visit-time-graph" graph_data_key="avg_time_on_page" y_label="Secondes" %} + {% include "misc/graph.part.html" with tab_name="tab-users-graph-content" graph_title="Évolution du nombre de visites" canvas_id="users-graph" graph_data_key="nb_visits" y_label="Nombre de visites" %} {% endif %} - {% if cumulative_stats %} - {% if display == 'global' and cumulative_stats|length > 1 %} + {% if huge_table_data %} + {% if display == 'global' and huge_table_data|length > 1 %} {% if form.non_field_errors %}

{{ form.non_field_errors.as_text }} @@ -69,22 +69,22 @@

{% trans "Partie" %} {% trans "Vues" %} {% trans "Temps moyen sur la page" %} - {% trans "Visiteurs uniques" %} - {% if display == 'global' and cumulative_stats|length > 1 %}{% endif %} + {% trans "Visites" %} + {% if display == 'global' and huge_table_data|length > 1 %}{% endif %} - {% for url, view in cumulative_stats.items %} + {% for url, view in huge_table_data.items %} {{ url.name }} - {% if display != 'details' and cumulative_stats|length > 1 %} + {% if display != 'details' and huge_table_data|length > 1 %} - {% trans "(détails)" %} {% endif %} {{ view.nb_hits }} {{ view.avg_time_on_page|seconds_to_duration }} - {{ view.nb_uniq_visitors }} - {% if display == 'global' and cumulative_stats|length > 1 %} + {{ view.nb_visits }} + {% if display == 'global' and huge_table_data|length > 1 %} {% endif %} @@ -92,7 +92,7 @@

{% if display == 'global' %} - {% if cumulative_stats|length > 1 %} + {% if huge_table_data|length > 1 %} {% endif %} @@ -113,9 +113,9 @@

{% trans "Types de reférent vers ces pages" %}

{% trans "Visites" %} - {% for category, visits in type_referrers.items %} + {% for type, visits in referrer_types.items %} - {{ category }} + {{ type }} {{ visits }} {% endfor %} @@ -126,13 +126,13 @@

{% trans "Types de reférent vers ces pages" %}

{% trans "Sites entrants vers ces pages" %}

- + - {% for referrer, visits in referrers.items %} + {% for website, visits in referrer_websites.items %} - + {% endfor %} @@ -147,7 +147,7 @@

{% trans "Mots-clés vers ces pages" %}

- {% for keyword, visits in keywords.items %} + {% for keyword, visits in referrer_keywords.items %} @@ -247,4 +247,4 @@

Type de graphe

-{% endblock sidebar %} +{% endblock %} diff --git a/zds/tutorialv2/views/statistics.py b/zds/tutorialv2/views/statistics.py index 90ecd4058e..efbc93ad08 100644 --- a/zds/tutorialv2/views/statistics.py +++ b/zds/tutorialv2/views/statistics.py @@ -41,21 +41,14 @@ def post(self, request, *args, **kwargs): def get_form_kwargs(self): kwargs = super().get_form_kwargs() - kwargs["urls"] = [(named_url.url, named_url.name) for named_url in self.get_urls_to_render()] + self.get_urls_to_render() + kwargs["urls"] = [(named_url.url, named_url.name) for named_url in self.named_urls] return kwargs def form_valid(self, form): self.urls = form.cleaned_data["urls"] return super().get(self.request) - def get_urls_to_render(self): - all_named_urls = self.get_content_urls() - base_list = self.request.GET.getlist("urls", None) or self.urls - if base_list: - return [named_url for named_url in all_named_urls if named_url.url in base_list] - else: - return all_named_urls - def get_content_urls(self): content = self.versioned_object urls = [NamedUrl(content.title, content.get_absolute_url_online(), 0)] @@ -68,68 +61,24 @@ def get_content_urls(self): urls.append(NamedUrl(subchild.title, subchild.get_absolute_url_online(), 2)) return urls - def get_all_statistics(self, urls, start, end, methods): - date_ranges = "{},{}".format(start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")) - data_request = {"module": "API", "method": "API.getBulkRequest", "format": "json", "filter_limit": -1} - data_structured = {} - - for method in methods: - data_structured[method] = [] - - for index, method_url in enumerate(itertools.product(methods, urls)): - method = method_url[0] - url = method_url[1] - absolute_url = f"{self.request.scheme}://{self.request.get_host()}{url.url}" - param_url = f"pageUrl=={urllib.parse.quote_plus(absolute_url)}" - - request_params = {"method": method, "idSite": self.matomo_site_id, "date": date_ranges, "period": "day"} - if method.startswith("Referrers"): # referrers requests use segment for define url - request_params["segment"] = ",".join([param_url]) - elif method == "Actions.getPageUrl": - request_params["pageUrl"] = absolute_url - - data_request.update({f"urls[{index}]": urllib.parse.urlencode(request_params)}) - - response_matomo = requests.post(url=self.matomo_api_url, data=data_request) - data = response_matomo.json() - if isinstance(data, dict) and data.get("result", "") == "error": - raise StatisticsException(self.logger.error, data.get("message", _("Pas de message d'erreur"))) - else: - for index, method_url in enumerate(itertools.product(methods, urls)): - if isinstance(data[index], dict) and data[index].get("result", "") == "error": - raise StatisticsException( - self.logger.error, data[index].get("message", _("Pas de message d'erreur")) - ) - - method = method_url[0] - data_structured[method].append(data[index]) - - return data_structured - - @staticmethod - def get_stat_metrics(data, metric_name): - x = [] - y = [] - for key, val in data.items(): - x.append(key) - if len(val) == 0: - y.append(0) + def get_urls_to_render(self): + all_named_urls = self.get_content_urls() + urls = self.request.GET.getlist("urls", None) or self.urls + if urls: + named_urls = [named_url for named_url in all_named_urls if named_url.url in urls] + requested_named_urls = named_urls + if len(urls) == 1: + display_mode = "details" else: - y.append(val[0].get(metric_name, 0)) - - return (x, y) - - @staticmethod - def get_ref_metrics(data): - refs = {} - for key, val in data.items(): - for item in val: - if item["label"] in refs: - refs[item["label"]] += item["nb_visits"] - else: - refs[item["label"]] = item["nb_visits"] + display_mode = "comparison" + else: + requested_named_urls = [all_named_urls[0]] + named_urls = all_named_urls + display_mode = "global" - return refs + self.requested_named_urls = requested_named_urls + self.named_urls = named_urls + self.display_mode = display_mode def get_start_and_end_dates(self): end_date = self.request.GET.get("end_date", None) @@ -155,123 +104,159 @@ def get_start_and_end_dates(self): return start_date, end_date - def get_display_mode(self, urls): - # TODO make display_mode an enum ? - # Good idea, but not straightforward for the template integration - if len(urls) == len(self.get_content_urls()): - return "global" - if len(urls) == 1: - return "details" - return "comparison" + def get_matomo_request(self, request_data): + request_data.update( + { + "module": "API", + "idSite": self.matomo_site_id, + "language": "fr", + "format": "JSON", + } + ) - @staticmethod - def get_cumulative(stats): - cumul = {"total": 0} - for info_date, infos_stat in stats.items(): - cumul["total"] += len(infos_stat) - for info_stat in infos_stat: - for key, val in info_stat.items(): - if type(val) == str: - continue - if key in cumul: - cumul[key] += int(val) - else: - cumul[key] = int(val) - return cumul + # self.logger.info("Matomo request data") + # self.logger.info(request_data) + response = requests.post(url=self.matomo_api_url, data=request_data) + response_data = response.json() + # self.logger.info("Matomo response data") + # self.logger.info(response_data) + return response_data @staticmethod - def merge_ref_to_data(metrics, refs): - for key, item in refs.items(): - if key in metrics: - metrics[key] += item + def get_graph_metrics(data, metric_name): + x = [] + y = [] + for key, val in data.items(): + x.append(key) + if len(val) == 0: + y.append(0) else: - metrics[key] = item - return metrics - - @staticmethod - def merge_report_to_global(reports, fields): - metrics = {} - for key, item in reports.items(): - for field, is_avg in fields: - if field in metrics: - metrics[field] = ( - metrics[field][0], - [i + j for (i, j) in zip(metrics[field][1], item.get(field)[1])], - ) - else: - metrics[field] = item.get(field) - return metrics + y.append(val[0].get(metric_name, 0)) + return (x, y) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) if not (self.is_author or self.is_staff): raise PermissionDenied - urls = self.get_urls_to_render() - start_date, end_date = self.get_start_and_end_dates() - display_mode = self.get_display_mode(urls) - reports = {} - cumulative_stats = {} - referrers = {} - type_referrers = {} - keywords = {} - report_field = [("nb_uniq_visitors", False), ("nb_hits", False), ("avg_time_on_page", True)] + named_urls = self.named_urls + requested_named_urls = self.requested_named_urls + display_mode = self.display_mode - try: - # Each function sends only one bulk request for all the urls - # Each variable is a list of dictionnaries (one for each url) - all_statistics = self.get_all_statistics( - urls, - start_date, - end_date, - ["Referrers.getReferrerType", "Referrers.getWebsites", "Referrers.getKeywords", "Actions.getPageUrl"], + start_date, end_date = self.get_start_and_end_dates() + date_range = "{},{}".format(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d")) + + def get_request_label(named_url): + # Generate the "label" parameter corresponding to named_url for a request + # From: /alice-and-bob/foo-bar/ + # To: alice-and-bob > foo-bar + return named_url.url[1:-1].replace("/", " > ") + + def get_response_label(named_url): + # Generate the "label" corresponding to named_url in the response data + # From: /alice-and-bob/foo-bar/ + # To: foo-bar + return named_url.url.split("/")[-2] + + # GRAPHS + graphs_data = {} + for named_url in requested_named_urls: + graphs_data[named_url] = {} + response_data = self.get_matomo_request( + { + "method": "Actions.getPageUrls", + "date": date_range, + "period": "day", + "label": get_request_label(named_url), + } ) - except StatisticsException as e: - all_statistics = {} - logger_method, msg = e.args - logger_method(f"Something failed with Matomo reporting system: {msg}") - messages.error(self.request, _("Impossible de récupérer les statistiques du site ({}).").format(msg)) - except Exception as e: - all_statistics = {} - self.logger.error(f"Something failed with Matomo reporting system: {e}") - messages.error(self.request, _("Impossible de récupérer les statistiques du site ({}).").format(e)) - - if all_statistics != {}: - all_stats = all_statistics["Actions.getPageUrl"] - all_ref_websites = all_statistics["Referrers.getWebsites"] - all_ref_types = all_statistics["Referrers.getReferrerType"] - all_ref_keyword = all_statistics["Referrers.getKeywords"] - - for index, url in enumerate(urls): - cumul_stats = self.get_cumulative(all_stats[index]) - reports[url] = {} - cumulative_stats[url] = {} - - for item, is_avg in report_field: - reports[url][item] = self.get_stat_metrics(all_stats[index], item) - if is_avg: - cumulative_stats[url][item] = 0 - if cumul_stats.get("total") > 0: - cumulative_stats[url][item] = cumul_stats.get(item, 0) / cumul_stats.get("total") - else: - cumulative_stats[url][item] = cumul_stats.get(item, 0) - - referrers = self.merge_ref_to_data(referrers, self.get_ref_metrics(all_ref_websites[index])) - type_referrers = self.merge_ref_to_data(type_referrers, self.get_ref_metrics(all_ref_types[index])) - keywords = self.merge_ref_to_data(keywords, self.get_ref_metrics(all_ref_keyword[index])) - - if display_mode.lower() == "global": - reports = {NamedUrl(display_mode, "", 0): self.merge_report_to_global(reports, report_field)} + for metric_name in ("avg_time_on_page", "nb_hits", "nb_visits"): + graphs_data[named_url][metric_name] = self.get_graph_metrics(response_data, metric_name) + # self.logger.info("Graphs data") + # self.logger.info(graphs_data) + + # HUGE TABLE + bulk_request_data = {"method": "API.getBulkRequest"} + for index, named_url in enumerate(named_urls): + individual_request_data = { + "method": "Actions.getPageUrls", + "idSite": self.matomo_site_id, + "date": date_range, + "period": "range", + "label": get_request_label(named_url), + } + bulk_request_data.update({f"urls[{index}]": urllib.parse.urlencode(individual_request_data)}) + response_data = self.get_matomo_request(bulk_request_data) + + temp_data = {} + for elem in response_data: + try: + temp_data[elem[0]["label"]] = { + "avg_time_on_page": elem[0]["avg_time_on_page"], + "nb_hits": elem[0]["nb_hits"], + "nb_visits": elem[0]["nb_visits"], + } + except IndexError: + continue + huge_table_data = {} + for named_url in named_urls: + try: + huge_table_data[named_url] = temp_data[get_response_label(named_url)] + except KeyError: + huge_table_data[named_url] = { + "avg_time_on_page": 0, + "nb_hits": 0, + "nb_visits": 0, + } + # self.logger.info("Huge table data") + # self.logger.info(huge_table_data) + + # SMALL TABLES + segments = [] + for named_url in requested_named_urls: + # TODO: Replace https://zestedesavoir.com by the adequate settings variable + absolute_url = f"https://zestedesavoir.com{named_url.url}" + segments.append(f"pageUrl=^{urllib.parse.quote_plus(absolute_url)}") + + bulk_request_data = {"method": "API.getBulkRequest"} + for index, method in enumerate(["Referrers.getReferrerType", "Referrers.getWebsites", "Referrers.getKeywords"]): + individual_request_data = { + "method": method, + "idSite": self.matomo_site_id, + "date": date_range, + "period": "range", + "segment": ",".join(segments), + } + bulk_request_data.update({f"urls[{index}]": urllib.parse.urlencode(individual_request_data)}) + response_data = self.get_matomo_request(bulk_request_data) + + referrer_types = {} + for elem in response_data[0]: + referrer_types[elem["label"]] = elem["nb_visits"] + # self.logger.info("Referrer types") + # self.logger.info(referrer_types) + + referrer_websites = {} + for elem in response_data[1]: + referrer_websites[elem["label"]] = elem["nb_visits"] + # self.logger.info("Referrer websites") + # self.logger.info(referrer_websites) + + referrer_keywords = {} + for elem in response_data[2]: + referrer_keywords[elem["label"]] = elem["nb_visits"] + # self.logger.info("Referrer keywords") + # self.logger.info(referrer_keywords) context.update( { + "urls": named_urls, "display": display_mode, - "urls": urls, - "reports": reports, - "cumulative_stats": cumulative_stats, - "referrers": referrers, - "type_referrers": type_referrers, - "keywords": keywords, + "graphs_data": graphs_data, + "huge_table_data": huge_table_data, + "referrer_types": referrer_types, + "referrer_websites": referrer_websites, + "referrer_keywords": referrer_keywords, } ) return context
{% trans "Site" %}{% trans "Site web" %} {% trans "Visites" %}
{{ referrer }}{{ website }} {{ visits }}
{% trans "Visites" %}
{{ keyword }} {{ visits }}