Skip to content

Commit

Permalink
Merge pull request #1047 from linea-it/develop
Browse files Browse the repository at this point in the history 
v0.19
  • Loading branch information
linea-relmanager authored Jan 17, 2018
2 parents 4ee346a + ae861ac commit d1b26f0
Show file tree
Hide file tree
Showing 33 changed files with 1,073 additions and 167 deletions.
69 changes: 55 additions & 14 deletions api/catalog/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def list(self, request):
# colunas associadas ao produto
associations = Association().get_associations_by_product_id(catalog.pk)


# Recuperar no Settigs em qual schema do database estao as tabelas de rating e reject
schema_rating_reject = settings.SCHEMA_RATING_REJECT

Expand All @@ -100,7 +101,6 @@ def list(self, request):
)

rows, count = catalog_db.query(
# columns=list(["coadd_objects_id", "niter_model_g"]),
ordering=request.query_params.get('ordering', None),
limit=request.query_params.get('limit', None),
start=request.query_params.get('offset', None),
Expand All @@ -121,22 +121,21 @@ def list(self, request):
"_meta_reject": None,
})


row.update({
"_meta_id": row.get(associations.get("meta.id;meta.main")),
"_meta_property_id": associations.get("meta.id;meta.main")
})
row.update({
"_meta_ra": row.get(associations.get("pos.eq.ra;meta.main")),
"_meta_property_ra": associations.get("pos.eq.ra;meta.main")
})
row.update({
"_meta_dec": row.get(associations.get("pos.eq.dec;meta.main")),
"_meta_property_id": associations.get("meta.id;meta.main"),
"_meta_property_ra": associations.get("pos.eq.ra;meta.main"),
"_meta_property_dec": associations.get("pos.eq.dec;meta.main")
})
row.update({
"_meta_radius": row.get(associations.get("phys.angSize;src")),
"_meta_property_radius": associations.get("phys.angSize;src")
})

try:
# Raio so e obrigatorio para catalogo do tipo sistema
row.update({
"_meta_radius": float(row.get(associations.get("phys.angSize;src"))),
"_meta_property_radius": associations.get("phys.angSize;src")
})
except:
pass

row.update({
"_meta_rating_id": row.get('meta_rating_id', None)
Expand Down Expand Up @@ -172,6 +171,41 @@ def list(self, request):
"_meta_comments": None
})

essential_props = dict({
# Id
'meta.id;meta.main': '_meta_id',
# Coordinates
'pos.eq.ra;meta.main': '_meta_ra',
'pos.eq.dec;meta.main': '_meta_dec',
# Elipse
'phys.size.smajAxis;instr.det;meta.main': '_meta_a_image',
'phys.size.sminAxis;instr.det;meta.main': '_meta_b_image',
'pos.posAng;instr.det;meta.main': '_meta_theta_image',
# Magnitudes
'phot.mag;meta.main;em.opt.g': '_meta_mag_auto_g',
'phot.mag;meta.main;em.opt.r': '_meta_mag_auto_r',
'phot.mag;meta.main;em.opt.i': '_meta_mag_auto_i',
'phot.mag;meta.main;em.opt.z': '_meta_mag_auto_z',
'phot.mag;meta.main;em.opt.Y': '_meta_mag_auto_y',
# Photo Z
'src.redshift.phot': '_meta_photo_z',

})


for ucd in associations:
try:
meta_prop = essential_props.get(ucd)
if meta_prop:
value = row.get(associations.get(ucd))

row.update({
meta_prop: value
})

except:
pass

return Response(dict({
'count': count,
'results': rows
Expand Down Expand Up @@ -204,6 +238,8 @@ def list(self, request):
# Criar uma lista de colunas baseda nas associacoes isso para limitar a query de nao usar *
columns = Association().get_properties_associated(product_id)

print(columns)

catalog_db = CatalogObjectsDBHelper(
table=catalog.tbl_name,
schema=catalog.tbl_schema,
Expand Down Expand Up @@ -235,6 +271,8 @@ def list(self, request):
'phot.mag;meta.main;em.opt.i': '_meta_mag_auto_i',
'phot.mag;meta.main;em.opt.z': '_meta_mag_auto_z',
'phot.mag;meta.main;em.opt.Y': '_meta_mag_auto_y',
# Photo Z
'src.redshift.phot': '_meta_photo_z',
})

for row in rows:
Expand Down Expand Up @@ -267,6 +305,7 @@ def list(self, request):
try:
meta_prop = essential_props.get(ucd)
if meta_prop:
# print(row.get(associations.get(ucd)))
value = row.get(associations.get(ucd))

# TODO: Esse Bloco precisa de um refactoring
Expand Down Expand Up @@ -309,6 +348,8 @@ def list(self, request):
meta_prop: value
})

# print("%s:%s" % (meta_prop, value))

except:
pass
return Response(dict({
Expand Down
20 changes: 17 additions & 3 deletions api/common/notify.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ class Notify():
def __init__(self):
self.logger = logging.getLogger('django')

def send_email(self, subject, body, to, copy_to_adms=True):
def send_email(self, subject, body, to, copy_to_adms=True, html=True):
"""
Envia um email, o body do email pode ser uma mensagem html.
:param subject: Assunto do Email
Expand Down Expand Up @@ -56,8 +56,22 @@ def send_email(self, subject, body, to, copy_to_adms=True):
from_email=from_email,
to=to,
)
msg.content_subtype = "html"

if html is True:
msg.content_subtype = "html"

msg.send(fail_silently=False)

except Exception as e:
self.logger.error(e)
self.logger.error(e)


def send_email_failure_helpdesk(self, subject, original_message):


try:
to = settings.EMAIL_HELPDESK
except:
raise Exception("The EMAIL_HELPDESK variable is not configured in settings.")

self.send_email(subject, original_message, to, False, False)
24 changes: 20 additions & 4 deletions api/common/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,9 +280,25 @@ def send_statistic_email(request):
@api_view(['GET'])
def teste(request):
if request.method == 'GET':

from product.garbagecolector import GarbageColectorProduct

GarbageColectorProduct().purge_products_expiration_time()
# from product.models import CutOutJob
# from product.tasks import start_des_cutout_job_by_id, check_jobs_running
#
# cutoutjob = CutOutJob.objects.get(pk=78)
# print(cutoutjob.pk)


# Testar submissao
# cutoutjob.cjb_status = 'st'
# cutoutjob.save()
# start_des_cutout_job_by_id(78)

# Testar Check Status
# cutoutjob.cjb_status = 'rn'
# cutoutjob.save()
# check_jobs_running()

# from product.garbagecolector import GarbageColectorProduct
#
# GarbageColectorProduct().purge_products_expiration_time()

return Response(dict({'status': "success"}))
31 changes: 29 additions & 2 deletions api/dri/settings/local_vars.py.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ EMAIL_ADMIN = '[email protected]'

# DES Cutout Service
DES_CUTOUT_SERVICE = {
# 1 para a versao do Descut Colaboracao 2 para versao Descut Public
'API_VERSION': 1,
'HOST': 'https://descut.cosmology.illinois.edu',
'USER': '<User required for cutout>',
'PASSWORD': '<password required for cutout>',
Expand All @@ -80,12 +82,24 @@ DES_CUTOUT_SERVICE = {
# Url base que sera usada para exibir as imagens geradas esse parametro deve ser mapeado no dri.conf no apache
'CUTOUT_SOURCE': '/data',
# Tempo de delay para a task check_jobs em minutos
'CUTOUT_TASK_CHECK_JOBS_DELAY': 1
'CUTOUT_TASK_CHECK_JOBS_DELAY': 1,
# Lista dos Releases que podem ser usados para cutout em lowercase. use [] para permitir todos e None para
# desabilitar a opcao na interface
'AVAILABLE_RELEASES': None,
# Quantidade limit de objetos a ser passada para o descutout
'MAX_OBJECTS': 300
'MAX_OBJECTS': 300,
# Token de authenticacao utilizado apenas para o DescutPublico para colaboracao usar None
'TOKEN': None,
# Esta opcao deve ser False para o DescutPublico e True para Colaboracao
'DELETE_JOB_AFTER_DOWNLOAD': True,
# Url para gerar o token, para o publico usar None.
'API_GET_TOKEN': '/api/token/',
# Url para a API reponsavel por criar os jobs
'API_CREATE_JOBS': '/api/jobs/',
# Url para a API responsavel por retornar o status dos jobs
'API_CHECK_JOBS': '/api/jobs/',
# No DescutPublico e necessario passar um email para onde seram enviadas as notificacoes do descut.
'EMAIL': '[email protected]'
}

# Tempo limite em horas para que um produto fique disponivel, apos este tempo
Expand Down Expand Up @@ -159,6 +173,14 @@ LOGGING = {
# 'filename': os.path.join(LOG_DIR, 'ncsa_authentication.log'),
# 'formatter': 'simple',
# },
# 'garbage_colector': {
# 'level': 'DEBUG',
# 'class': 'logging.handlers.RotatingFileHandler',
# 'maxBytes': 1024 * 1024 * 5, # 5 MB
# 'backupCount': 5,
# 'filename': os.path.join(LOG_DIR, 'garbage_colector.log'),
# 'formatter': 'standard',
# },
# },
# 'loggers': {
# 'django': {
Expand Down Expand Up @@ -191,5 +213,10 @@ LOGGING = {
# 'level': 'DEBUG',
# 'propagate': True,
# },
# 'garbage_colector': {
# 'handlers': ['garbage_colector'],
# 'level': 'DEBUG',
# 'propagate': True,
# },
# }
}
65 changes: 50 additions & 15 deletions api/lib/CatalogDB.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ def set_query_columns(self, columns):
def set_filters(self, filters):
# Seta os filtros a serem aplicados no create_stm
if filters is not None and len(filters) > 0:

for condition in filters:
column = condition.get("column").lower().strip()

Expand All @@ -173,9 +174,23 @@ def set_url_filters(self, url_filters):
self.filters.append(condition)

def parse_url_filters(self, url_filters):

conditions = list()

if url_filters is not None and len(url_filters) > 0:

square_condition = dict({
"lon": None,
"lat": None,
"radius": None,
"column": "coordinates",
"op": "coordinates",
"upperright": [],
"lowerleft": [],
"property_ra": self.associations.get("pos.eq.ra;meta.main"),
"property_dec": self.associations.get("pos.eq.dec;meta.main")
})

for param in url_filters:
# para cara parametro vindo da url checar se e uma propriedade valida da tabela
# e Criar um filtro para ela
Expand Down Expand Up @@ -217,6 +232,33 @@ def parse_url_filters(self, url_filters):
# do atributo association
pass

# Query por quadrado usando formato lon, lat do centro e raio.
elif column == "lon" or column == "lat" or column == "radius":
# Prepara um objeto com as variaveis necessarias para a criar um quadrado.
square_condition[column] = url_filters.get(param)

# Verificar se foi criado um filtro por tipo quadrado.
if square_condition['lon'] is not None or square_condition['lat'] is not None or square_condition[
'radius'] is not None:
# criar as variaveis lowerleft e upperright

lon = float(square_condition['lon'])
lat = float(square_condition['lat'])
radius = float(square_condition['radius'])

lowerleft = [lon - radius,
lat - radius]

upperright = [lon + radius,
lat + radius]

square_condition.update({
"lowerleft": lowerleft,
"upperright": upperright
})

conditions.append(square_condition)

return conditions

def get_condition_square(self, lowerleft, upperright, property_ra, property_dec):
Expand Down Expand Up @@ -450,7 +492,9 @@ def create_stm(self, columns=list(), filters=None, ordering=None, limit=None, st

# Targets podem ter filtros especias checar a existencia deles
if self.filters is not None and len(self.filters) > 0:

for condition in self.filters:

if condition.get("column").find("_meta_") is not -1:
# Filtro Especial onde a propriedade nao faz parte da tabela original

Expand All @@ -466,22 +510,13 @@ def create_stm(self, columns=list(), filters=None, ordering=None, limit=None, st
if condition.get("value") in ['True', 'true', '1', 't', 'y', 'yes']:
reject_filters = catalog_reject.c.reject == 1

elif condition.get("column") == 'coordinates':
value = json.loads(condition.get("value"))

# Upper Right
upperright = value[0]
# Lower Left
lowerleft = value[1]

property_ra = self.associations.get("pos.eq.ra;meta.main")
property_dec = self.associations.get("pos.eq.dec;meta.main")
elif condition.get("column") == 'coordinates':

coordinate_filters = self.get_condition_square(
lowerleft,
upperright,
property_ra,
property_dec)
coordinate_filters = self.get_condition_square(
condition.get("lowerleft"),
condition.get("upperright"),
condition.get("property_ra"),
condition.get("property_dec"))

else:
filters.append(condition)
Expand Down
Loading

0 comments on commit d1b26f0

Please sign in to comment.