Skip to content

Commit

Permalink
Removing dependencies on h5py
Browse files Browse the repository at this point in the history 
In some installations, the dependency on h5py (HDF5) results in
the Python interpreter bombing if the h5py library has been compiled
with different library version to the one being used in the current
system. Since I never really got around to saving stuff as HDF5
anyway, I switched it off as a dependency.
  • Loading branch information
Jose Gomez-Dans committed May 23, 2017
1 parent 9e70f55 commit 72fa5ee
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 44 deletions.
87 changes: 45 additions & 42 deletions gp_emulator/multivariate_gp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import os
import shutil

import h5py
#import h5py
import numpy as np
import matplotlib.pyplot as plt

Expand Down Expand Up @@ -71,17 +71,19 @@ def __init__ ( self, dump=None, X=None, y=None,
"""
if dump is not None:
if X is None and y is None:
if dump.find (".h5") > 0 or dump.find(".hdf5") > 0:
f = h5py.File ( dump, 'r+')
group = "%s_%03d_%03d_%03d" % ( model, sza, vza, raa )
X = f[group + '/X_train'][:,:]
y = f[group + '/y_train'][:,:]
hyperparams = f[group+'/hyperparams'][:,:]
thresh = f[group+'/thresh'].value
basis_functions = f[group+"/basis_functions"][:,:]
n_pcs = f[group+"/n_pcs"].value
f.close()
elif dump.find(".npz"):
if dump.find (".h5") > 0 or dump.find(".hdf5") > 0:
raise IOError, "I can't be bothered working with HDF5 files"

##f = h5py.File ( dump, 'r+')
##group = "%s_%03d_%03d_%03d" % ( model, sza, vza, raa )
##X = f[group + '/X_train'][:,:]
##y = f[group + '/y_train'][:,:]
##hyperparams = f[group+'/hyperparams'][:,:]
##thresh = f[group+'/thresh'].value
##basis_functions = f[group+"/basis_functions"][:,:]
##n_pcs = f[group+"/n_pcs"].value
##f.close()
elif dump.find(".npz"):
f = np.load ( dump )
X = f[ 'X' ]
y = f[ 'y' ]
Expand All @@ -94,15 +96,15 @@ def __init__ ( self, dump=None, X=None, y=None,

else:

f = h5py.File ( dump, 'r+')
group = "/%s_%03d_%03d_%03d" % ( model, sza, vza, raa )
X = f[group + '/X_train'][:,:]
y = f[group + '/y_train'][:,:]
hyperparams = f[group+'/hyperparams'][:,:]
thresh = f[group+'/thresh'].value
basis_functions = f[group+"/basis_functions"][:,:]
n_pcs = f[group+"/n_pcs"].value
f.close()
#f = h5py.File ( dump, 'r+')
#group = "/%s_%03d_%03d_%03d" % ( model, sza, vza, raa )
#X = f[group + '/X_train'][:,:]
#y = f[group + '/y_train'][:,:]
#hyperparams = f[group+'/hyperparams'][:,:]
#thresh = f[group+'/thresh'].value
#basis_functions = f[group+"/basis_functions"][:,:]
#n_pcs = f[group+"/n_pcs"].value
#f.close()
else:
raise ValueError, "You specified both a dump file and X and y"
else:
Expand Down Expand Up @@ -150,31 +152,32 @@ def dump_emulator ( self, fname, model_name, sza, vza, raa ):
raa = int ( raa )
if fname.find ( ".npz" ) < 0 and ( fname.find ( "h5" ) >= 0 \
or fname.find ( ".hdf" ) >= 0 ):
try:
f = h5py.File (fname, 'r+')
except IOError:
print "The file %s did not exist. Creating it" % fname
f = h5py.File (fname, 'w')
f
group = '%s_%03d_%03d_%03d' % ( model_name, sza, vza, raa )
if group in f.keys():
raise ValueError, "Emulator already exists!"
f.create_group ("/%s" % group )
f.create_dataset ( "/%s/X_train" % group, data=self.X_train, compression="gzip" )
f.create_dataset ( "/%s/y_train" % group, data=self.y_train, compression="gzip" )
f.create_dataset ( "/%s/hyperparams" % group, data=self.hyperparams,
compression="gzip" )
f.create_dataset ( "/%s/basis_functions" % group, data=self.basis_functions,
compression="gzip" )
f.create_dataset ( "/%s/thresh" % group, data=self.thresh )
f.create_dataset ( "/%s/n_pcs" % group, data=self.n_pcs)
f.close()
print "Emulator safely saved"
raise IOError, "I can't be bothered working with HDF5 files"
#try:
#f = h5py.File (fname, 'r+')
#except IOError:
#print "The file %s did not exist. Creating it" % fname
#f = h5py.File (fname, 'w')
#f
#group = '%s_%03d_%03d_%03d' % ( model_name, sza, vza, raa )
#if group in f.keys():
#raise ValueError, "Emulator already exists!"
#f.create_group ("/%s" % group )
#f.create_dataset ( "/%s/X_train" % group, data=self.X_train, compression="gzip" )
#f.create_dataset ( "/%s/y_train" % group, data=self.y_train, compression="gzip" )
#f.create_dataset ( "/%s/hyperparams" % group, data=self.hyperparams,
#compression="gzip" )
#f.create_dataset ( "/%s/basis_functions" % group, data=self.basis_functions,
#compression="gzip" )
#f.create_dataset ( "/%s/thresh" % group, data=self.thresh )
#f.create_dataset ( "/%s/n_pcs" % group, data=self.n_pcs)
#f.close()
#print "Emulator safely saved"
else:
np.savez_compressed ( fname, X=self.X_train, y=self.y_train, \
hyperparams=self.hyperparams, thresh=self.thresh, \
basis_functions=self.basis_functions, n_pcs=self.n_pcs )

print "Emulator safely saved"

def calculate_decomposition ( self, X, thresh ):
"""Does PCA decomposition
Expand Down
4 changes: 2 additions & 2 deletions gp_emulator/save_emulators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import shelve

import numpy as np
import h5py
#import h5py

from GaussianProcess import GaussianProcess
from multivariate_gp import MultivariateEmulator
Expand Down Expand Up @@ -110,7 +110,7 @@ def get_emulator ( self, tag ):
def convert_npz_to_hdf5 ( npz_file, hdf5_file ):
"""A utility to convert from the old and time honoured .npz format to the
new HDF5 format."""

import h5py # HACK Nasty!
f = np.load ( npz_file )
X = f[ 'X' ]
y = f[ 'y' ]
Expand Down

0 comments on commit 72fa5ee

Please sign in to comment.