Merge pull request #1033 from linea-it/develop

V0.18

.gitignore

@@ -11,6 +11,9 @@
 # Diretorio de projeto pycharm
 /.idea/
 
+# Diretorio de projeto vscode
+/.vscode/
+
 # Others
 *.pyc
 __pycache__

api/common/authentication.py

@@ -110,12 +110,8 @@ def getUserInfo(self, username):
 
         des_user_table = self.get_des_user_tablename()
 
-        sql = "SELECT email, firstname, lastname FROM %s WHERE username = '%s'" % (des_user_table.upper(), username)
-        return db.fetchone_dict(sql)
-
-        des_user_table = self.get_des_user_tablename()
-
-        sql = "SELECT email, firstname, lastname FROM %s WHERE username = '%s'" % (des_user_table.upper(), username)
+        sql = "SELECT email, firstname, lastname FROM %s WHERE username = '%s'" % (
+        des_user_table.upper(), username.lower())
 
         self.logger.debug("SQL: %s" % sql)
 

api/common/tasks.py

@@ -1,15 +1,21 @@
-# from __future__ import absolute_import, unicode_literals
-#
-# from celery import task
-#
-# @task(name="test")
-# def test():
-#     """
-#
-#     """
-#     print("----------- TESTE ----------------")
-#     # from statistics.general_statistics import GeneralStatistics
-#     #from statistics.models import Statistics
-#     from common.models import *
-#
-#     print(Filter.objects.count())
+from __future__ import absolute_import, unicode_literals
+
+from celery import task
+from celery.decorators import periodic_task
+from celery.task.schedules import crontab
+
+from django.conf import settings
+# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Garbage Colector %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #
+@periodic_task(
+    run_every=(crontab(minute='*/30')),
+    #run_every=10.0,
+    name="garbage_colector",
+    ignore_result=True
+)
+def garbage_colector():
+    """
+    Executa rotinas de limpesa
+    """
+    # Limpar os produtos
+    from product.garbagecolector import GarbageColectorProduct
+    GarbageColectorProduct().purge_products_expiration_time()

api/dri/settings/local_vars.py.template

@@ -95,6 +95,8 @@ PRODUCT_EXPIRATION_TIME = 12
 ## USER_QUERY
 # Tempo de execucao de um job no User query.
 USER_QUERY_EXECUTION_TIMEOUT = 60
+# Limite de linhas de uma query
+USER_QUERY_MAX_ROWS = 300
 
 ## TARGET VIEWER
 # Habilita ou desabilita a interface de registro de produtos pela opcao Database,

api/dri/urls.py

@@ -105,6 +105,7 @@
 router.register(r'userquery_preview', userquery_views.QueryPreview, base_name='preview_query')
 router.register(r'userquery_property', userquery_views.TableProperties, base_name='table')
 router.register(r'userquery_target', userquery_views.TargetViewerRegister, base_name='target_viewer_register')
+router.register(r'userquery_download', userquery_views.TableDownload, base_name='table_download')
 
 # Aladin API
 router.register(r'aladin/image', aladin_views.ImageViewSet)

api/lib/CatalogDB.py

@@ -219,6 +219,69 @@ def parse_url_filters(self, url_filters):
 
         return conditions
 
+    def get_condition_square(self, lowerleft, upperright, property_ra, property_dec):
+
+        # Tratar RA > 360
+        llra = float(lowerleft[0])
+        lldec = float(lowerleft[1])
+
+        urra = float(upperright[0])
+        urdec = float(upperright[1])
+
+        if llra > 360:
+            llra = llra - 360
+
+        if urra > 360:
+            urra = urra - 360
+
+        # Verificar se o RA 0 esta entre llra e urra
+        if (llra < 0 and urra < 0) or (llra > 0 and urra > 0):
+            # RA 0 nao esta na area da consulta pode se usar o between simples
+
+            # BETWEEN llra and urra
+            raCondition = between(
+                Column(str(property_ra)),
+                literal_column(str(llra)),
+                literal_column(str(urra))
+            )
+
+        else:
+            # Area de interesse passa pelo RA 0 usar 2 between separando ate 0 e depois de 0
+
+            llralt0 = 360 - (llra * -1)
+
+            # Solucao para catalogos com RA 0 - 360
+            raLTZero = between(
+                Column(str(property_ra)),
+                literal_column(str(llralt0)),
+                literal_column("360")
+            )
+
+            raGTZero = between(
+                Column(str(property_ra)),
+                literal_column("0"),
+                literal_column(str(urra))
+            )
+
+            raCondition360 = or_(raLTZero, raGTZero).self_group()
+
+            # Solucao para catalogos com RA -180 a 180
+            raCondition180 = between(
+                Column(str(property_ra)),
+                literal_column(str(llra)),
+                literal_column(str(urra))
+            )
+
+            raCondition = or_(raCondition360, raCondition180).self_group()
+
+        decCondition = between(
+            Column(str(property_dec)),
+            literal_column(str(lldec)),
+            literal_column(str(urdec))
+        )
+
+        return and_(raCondition, decCondition).self_group()
+
 
 class CatalogObjectsDBHelper(CatalogTable):
     def create_stm(self, columns=list(), filters=None, ordering=None, limit=None, start=None, url_filters=None,
@@ -239,28 +302,11 @@ def create_stm(self, columns=list(), filters=None, ordering=None, limit=None, st
         for condition in self.filters:
             if condition.get("op") == "coordinates":
 
-                ur = condition.get("upperright")
-                ll = condition.get("lowerleft")
-
-                # Tratar RA > 360
-                llra = float(ll[0])
-                urra = float(ur[0])
-
-                if llra > 360:
-                    llra = llra - 360
-
-                if urra > 360:
-                    urra = urra - 360
-
-                coordinates_filter = and_(between(
-                    Column(str(condition.get("property_ra"))),
-                    literal_column(str(llra)),
-                    literal_column(str(urra))
-                ), between(
-                    Column(str(condition.get("property_dec"))),
-                    literal_column(str(ll[1])),
-                    literal_column(str(ur[1]))
-                ))
+                coordinates_filter = self.get_condition_square(
+                    condition.get("lowerleft"),
+                    condition.get("upperright"),
+                    condition.get("property_ra"),
+                    condition.get("property_dec"))
 
             else:
                 filters.append(condition)
@@ -422,33 +468,20 @@ def create_stm(self, columns=list(), filters=None, ordering=None, limit=None, st
 
                     elif condition.get("column") == 'coordinates':
                         value = json.loads(condition.get("value"))
+
                         # Upper Right
-                        ur = value[0]
+                        upperright = value[0]
                         # Lower Left
-                        ll = value[1]
-                        # property_ra
+                        lowerleft = value[1]
+
                         property_ra = self.associations.get("pos.eq.ra;meta.main")
                         property_dec = self.associations.get("pos.eq.dec;meta.main")
 
-                        # Tratar RA > 360
-                        llra = float(ll[0])
-                        urra = float(ur[0])
-
-                        if llra > 360:
-                            llra = llra - 360
-
-                        if urra > 360:
-                            urra = urra - 360
-
-                        coordinate_filters = and_(between(
-                            Column(str(property_ra)),
-                            literal_column(str(ll[0])),
-                            literal_column(str(ur[0]))
-                        ), between(
-                            Column(str(property_dec)),
-                            literal_column(str(ll[1])),
-                            literal_column(str(ur[1]))
-                        ))
+                        coordinate_filters = self.get_condition_square(
+                            lowerleft,
+                            upperright,
+                            property_ra,
+                            property_dec)
 
                 else:
                     filters.append(condition)

api/product/export.py

@@ -13,11 +13,14 @@
 from django.contrib.auth.models import User
 from django.core.mail import EmailMessage
 from django.template.loader import render_to_string
-from lib.CatalogDB import TargetObjectsDBHelper
+from lib.CatalogDB import TargetObjectsDBHelper, CatalogTable
 from product.association import Association
 from common.notify import Notify
 from .models import Product
 
+from userquery.models import Table
+
+
 class Export:
     def __init__(self):
         # Get an instance of a logger
@@ -142,6 +145,58 @@ def table_to_csv(self, product_id, table, export_dir, user_id, schema=None, data
         else:
             self.logger.error("Query returned no results")
 
+    def table_to_csv(self, table, schema, export_dir, columns=None):
+        """
+        Le uma tabela criada pelo user_query e cria um csv com o resultado.
+        OBS: NAO recomendada para tabelas grandes. por que neste metodo todos as linhas
+        sao recuperadas ao mesmo tempo e e feito um for para inserir as linhas no csv.
+        :param export_dir: diretorio onde o arquivo csv vai ser gerado.
+        """
+
+        self.logger.info("Export table \"%s\" to csv" % table)
+
+        name = ("%s.csv" % table)
+
+        filename = os.path.join(export_dir, name)
+        self.logger.debug("Filename: %s" % filename)
+
+        catalogTable = CatalogTable(table, schema=schema, database='catalog')
+
+        # review columns selection
+        if not columns:
+            columns = catalogTable.column_names
+
+        rows, count = catalogTable.query(columns)
+
+        self.logger.debug("Row Count: %s" % count)
+
+        if count > 0:
+            columns = self.get_columns(rows[0])
+
+            lines = list()
+            for row in rows:
+                lines.append(row)
+
+            self.logger.info("Creating csv file")
+            with open(filename, 'w') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=columns)
+
+                self.logger.info("Writing the headers")
+                writer.writeheader()
+
+                self.logger.info("Writing the rows")
+                writer.writerows(lines)
+
+            csvfile.close()
+            self.logger.info("Successfully created")
+
+            file_size = humanize.naturalsize(os.path.getsize(filename))
+            self.logger.debug("File Size %s" % file_size)
+
+            return filename
+        else:
+            self.logger.error("Query returned no results")
+
     def csv_to_fits(self, csv, fits):
         self.logger.info("Export csv \"%s\" to fits" % csv)
 
@@ -311,26 +366,27 @@ def create_zip(self, export_dir):
 
         return url
 
-    def notify_user_export_start(self, user, product):
+    def notify_user_export_start(self, user, product=None, display_name=None):
         """
         Envia um email para o usuario informando que os arquivos estao sendo criados.
         """
         if user.email:
             self.logger.info("Sending mail notification START.")
 
+            if not display_name:
+                display_name = product.prd_display_name
+
             subject = "Download in Progress"
             body = render_to_string("export_notification_start.html", {
                 "username": user.username,
-                "target_display_name": product.prd_display_name
+                "target_display_name": display_name
             })
 
             Notify().send_email(subject, body, user.email)
 
         else:
             self.logger.info("It was not possible to notify the user, for not having the email registered.")
 
-
-
     def notify_user_export_success(self, user_id, product_name, url):
         """
         Envia um email para o usuario informando que o export terminou.
@@ -358,7 +414,6 @@ def notify_user_export_success(self, user_id, product_name, url):
         else:
             self.logger.info("It was not possible to notify the user, for not having the email registered.")
 
-
     def notify_user_export_failure(self, user, product):
         """
         Envia um email para o usuario informando que houve um erro na geracao dos arquivos.