Skip to content

Commit

Permalink
The final path of each map was changed to a structure compatible with…
Browse files Browse the repository at this point in the history 
… the maps already registered, separated by release. the displayname and internal name fields have been changed to generate more descriptive names. Frontend has been changed to display the map's displayname instead of the class. #1356 (#1357)

Co-authored-by: Matheus Freitas <[email protected]>
  • Loading branch information
@glaubervila @matheusallein
glaubervila and matheusallein authored May 25, 2021
1 parent a745795 commit 2fd1d0e
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 29 deletions.
65 changes: 65 additions & 0 deletions Docs/exemplo_importacao_maps_table_and_aladin.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
{
"process": {
"process_end_date": "2019-10-22 11:12:13",
"process_name": "Systematic Maps",
"fields": ["Y6A1_COADD"],
"process_description": null,
"process_id": "5705",
"products": [{
"pg_table_info": {
"table_size_in_bytes": 1313914880,
"number_of_columns": 4,
"number_of_rows": 25174128,
"table": "systematic_maps.y6a1_coadd_4096_airmass_wmean_y_5705"
},
"nside": "4096",
"oracle_table": "e_5705_16581",
"class_name": "Airmass Wmean",
"file_name": null,
"selected_name": null,
"product_type": "Systematic Maps",
"process_id": 5705,
"releases": ["Y6A1_COADD"],
"number_columns": null,
"table": "e_5705_16581",
"number_objects": null,
"map_filter": "Y",
"association": [],
"product_id": 16581,
"number_tiles": null,
"pipeline": "Systematic Maps",
"display_name": "Airmass Wmean 13",
"job_id": "734276",
"ordering": "nest",
"name": "Airmass Wmean 13",
"type_name": "Systematic Maps",
"namespace": null,
"ora_table_info": {
"n_imported_rows": 25174128,
"table_size_in_bytes": 998244352,
"oracle_table_name": "brportal.e_5705_16581",
"n_imported_columns": 4
},
"class": "systematic_maps_airmass_wmean",
"version": "13",
"mask_filter": null,
"fields": ["Y6A1_COADD"],
"release": "Y6A1_COADD",
"type": "map",
"file_path": "/archive/staging/DES/tmp/exports/systematic_maps.y6a1_coadd_4096_airmass_wmean_y_5705.csv",
"columns": "pixel,signal,ra,dec",
"schema": "brportal",
"database": "dessci",
"is_permanent": true,
"aladin": {
"download_url": "ftp://srvdatatransfer.linea.gov.br/production/y6a2_i_o_4096_t_32768_frac_EQU.zip",
"filename": "y6a2_i_o_4096_t_32768_frac_EQU.zip"
}
}],
"process_comment": null,
"process_start_date": "2019-10-22 11:12:13",
"owner_username": "adriano.pieres"
},
"ticket": "VDB377",
"register_username": "adriano.pieres"
}
2 changes: 1 addition & 1 deletion api/product/admin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class CatalogAdmin(admin.ModelAdmin):

class MapAdmin(admin.ModelAdmin):
list_display = (
'id', 'prd_name', 'prd_display_name', 'prd_class', 'mpa_nside', 'mpa_ordering',
'id', 'prd_name', 'prd_display_name', 'prd_class', 'mpa_nside', 'mpa_ordering', 'prd_filter', 'prd_is_public', 'prd_is_permanent'
)
list_display_links = ('id', 'prd_name')
search_fields = ('prd_name',)
Expand Down
77 changes: 67 additions & 10 deletions api/product_register/ImportProcess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,7 @@ def register_catalog(self, data):
"prd_description": data.get('description', None),
"prd_date": date,
"prd_is_public": data.get('is_public', False),
"prd_is_permanent": data.get('is_permanent', False),
"tbl_rows": int(tbl_rows),
"tbl_num_columns": int(tbl_num_columns),
"tbl_size": int(tbl_size),
Expand Down Expand Up @@ -658,25 +659,45 @@ def register_map(self, data):

# Data do produto caso o produto tenha processo a data do produto = data de start do processo
date = None
process_id = None
if self.process is not None:
date = self.process.epr_start_date
process_id = self.process.epr_original_id

# Cria um internal name e display name para ficar no formato
# release_nome_process_band
internal_name = data.get('name').replace(' ', '_').lower()
display_name = data.get('display_name')

if data.get('version', None) is not None:
if process_id is not None:
internal_name = internal_name.replace(data.get('version'), process_id)
display_name = display_name.replace(data.get('version'), process_id)
else:
internal_name = internal_name.replace('_%s' % data.get('version'), '')
display_name = display_name.replace(data.get('version'), '')

if map_filter is not None:
internal_name += '_%s' % map_filter.filter
display_name += ' ' + map_filter.filter

product, created = Map.objects.update_or_create(
prd_owner=self.owner,
prd_name=data.get('name').replace(' ', '_').lower(),
prd_name=internal_name,
tbl_schema=tbl_schema,
tbl_name=tbl_name,
prd_filter=map_filter,
defaults={
"prd_process_id": self.process,
"prd_class": cls,
"prd_display_name": data.get('display_name'),
"prd_display_name": display_name,
"prd_user_display_name": data.get('user_display_name'),
"prd_product_id": data.get('product_id', None),
"prd_version": data.get('version', None),
"prd_description": data.get('description', None),
"prd_date": date,
"prd_is_public": data.get('is_public', True),
"prd_is_permanent": data.get('is_permanent', True),
"mpa_nside": self.check_nside(data.get('nside')),
"mpa_ordering": self.check_ordering(data.get('ordering')),
"tbl_rows": tbl_rows,
Expand All @@ -692,6 +713,13 @@ def register_map(self, data):
self.product_release(product, data.get('releases'))
add_release = False

# Alterar o nome adicionando o prefixo do release.
release_display_name = product.releases.first().rls_display_name
product.prd_name = '%s_%s' % (release_display_name.lower(), product.prd_name)
product.prd_display_name = '%s %s' % (release_display_name, product.prd_display_name)
product.save()
product.refresh_from_db()

# Registrar O produto a seus respectivos Tags
if 'fields' in data:
self.product_tag(product, data.get('fields'), add_release)
Expand Down Expand Up @@ -744,26 +772,52 @@ def register_aladin_image(self, product, data):
self.logger.debug(data)
if 'download_url' in data and 'filename' in data:
try:

# O diretório deve ser maps/release/class/band/

# Diretório de mapas
relative_map_path = os.path.join(settings.DATA_DIR, "maps")
base_path = os.path.join(relative_map_path, product.prd_class.pcl_name)
if not os.path.exists(base_path):
os.mkdir(base_path)
sleep(1)
# Diretório do release
release_display_name = product.releases.first().rls_display_name.lower()
base_path = os.path.join(relative_map_path, release_display_name)

# Diretório da classe do mapa
base_path = os.path.join(base_path, product.prd_class.pcl_name)

# Diretório da banda do mapa
map_path = os.path.join(base_path, product.prd_filter.filter)

# raise Exception("Parou aqui!")

map_path = os.path.join(base_path, str(product.id))
if not os.path.exists(map_path):
os.mkdir(map_path)
os.makedirs(map_path)
sleep(1)
self.logger.debug("Creted path: %s" % map_path)

self.logger.debug("Filepath: %s" % map_path)

aladin_path_zip = self.download_aladin_image(data.get('download_url'), data.get('filename'), map_path)

with zipfile.ZipFile(aladin_path_zip, "r") as zip_ref:
zip_ref.extractall(map_path)

self.logger.debug("Filepath: %s" % map_path)
# Remove o arquivo zip
if os.path.exists(aladin_path_zip):
os.remove(aladin_path_zip)
self.logger.info("Deleted zip file: %s" % aladin_path_zip)

# Rename path to aladin
new_path = os.path.join(map_path, "aladin")
extrated_path = aladin_path_zip.replace('.zip', '')
# self.logger.debug("Old Path: %s" % extrated_path)
os.rename(extrated_path, new_path)

self.logger.debug("New Path: %s" % new_path)

# raise Exception("Parou aqui!")

host = settings.BASE_HOST
result_dir = os.path.join(relative_map_path, aladin_path_zip.replace('.zip', ''))
result_dir = os.path.join(relative_map_path, new_path)

result_sorce = result_dir.replace(settings.DATA_DIR, settings.DATA_SOURCE)

Expand All @@ -773,6 +827,8 @@ def register_aladin_image(self, product, data):
product=product, defaults={
'img_url': map_final_url,
})

self.logger.debug("Aladin Image URL: %s" % map_final_url)
except Exception as e:
self.logger.error(e)
raise Exception(e)
Expand Down Expand Up @@ -851,6 +907,7 @@ def register_mask(self, data):
"prd_description": data.get('description', None),
"prd_date": date,
"prd_is_public": data.get('is_public', True),
"prd_is_permanent": data.get('is_permanent', True),
"msk_filter": filter,
"tbl_rows": tbl_rows,
"tbl_num_columns": tbl_num_columns,
Expand Down
5 changes: 3 additions & 2 deletions frontend/nginx-proxy-develop.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,12 @@ server {
}

# Favicon
location /favicon.ico {
root /var/www/images/;
location ~* favicon.png {
alias /var/www/images/favicon.png;
autoindex off;
}


# Frontend Development with Sencha app Watch
location /sky/ {
proxy_pass http://sky:1841$request_uri;
Expand Down
4 changes: 2 additions & 2 deletions frontend/nginx-proxy.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ server {
}

# Favicon
location /favicon.ico {
root /var/www/images/;
location ~* favicon.png {
alias /var/www/images/favicon.png;
autoindex off;
}
# Logotipo para os emails
Expand Down
27 changes: 13 additions & 14 deletions frontend/packages/local/aladin/src/maps/MapSelectionController.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ Ext.define('aladin.maps.MapSelectionController', {
]);

store.load({
callback: function() {
callback: function () {
cmb_type.reset();
store_types.removeAll();
cmb_class.reset();
store_classes.removeAll();
cmb_filter.reset();
store_filters.removeAll();

store.each(function(record){
store.each(function (record) {
if (
store_types.findRecord(
'pgr_group', record.get('pgr_group')) == null
Expand Down Expand Up @@ -85,10 +85,10 @@ Ext.define('aladin.maps.MapSelectionController', {
cmb_filter.reset();
store_filters.removeAll();

store.each(function(record) {
store.each(function (record) {
if (
record.get('pgr_group') == map_model.get('pgr_group') &&
store_classes.findRecord('prd_class', record.get('prd_class')) == null
store_classes.findRecord('prd_class', record.get('prd_class')) == null
) {
store_classes.add(record);
}
Expand All @@ -108,10 +108,10 @@ Ext.define('aladin.maps.MapSelectionController', {
cmb_filter.reset();
store_filters.removeAll();

store.each(function(record) {
store.each(function (record) {
if (
record.get('pgr_group') == map_model.get('pgr_group') &&
record.get('prd_class') == map_model.get('prd_class')
record.get('prd_class') == map_model.get('prd_class')
) {
store_filters.add(record);
}
Expand All @@ -136,15 +136,14 @@ Ext.define('aladin.maps.MapSelectionController', {
]);

aladin_images_store.load({
callback: function() {
callback: function () {

if (aladin_images_store.getCount() != 1) {
Ext.MessageBox.alert(
'Warning',
aladin_images_store.getCount().toString() + ' images found for same map!');
}
else
{
else {
// retrieve the first non-map layer to restore
aladin_last_nonmap_survey = vm.get('aladin_last_nonmap_survey');

Expand All @@ -154,11 +153,12 @@ Ext.define('aladin.maps.MapSelectionController', {

var img_url = aladin_images_store.getAt(0).get('img_url');

console.log(map_model)
// default survey object
survey = {
'id': 'map_' + map_model.get('id').toString(),
'url': img_url,
'name': map_model.get('pcl_display_name'),
'name': map_model.get('prd_display_name'),
'filter': map_model.get('prd_filter'),
'maxOrder': 3,
'frame': 'equatorial',
Expand All @@ -170,11 +170,11 @@ Ext.define('aladin.maps.MapSelectionController', {
// retrieving maxOrder value from properties file
aladin.readProperties(
img_url,
function(properties) {
function (properties) {
survey['maxOrder'] = properties["maxOrder"];
me.setMapSurvey(survey);
},
function(error) {
function (error) {
console.log('aladin.readProperties() error: %o', error);
me.setMapSurvey(survey);
}
Expand Down Expand Up @@ -211,8 +211,7 @@ Ext.define('aladin.maps.MapSelectionController', {
if (vm.get('map_selected')) {
aladin_survey = vm.get('aladin_last_map_survey');
}
else
{
else {
aladin_survey = vm.get('aladin_last_nonmap_survey');
}

Expand Down

0 comments on commit 2fd1d0e

Please sign in to comment.