HDF5File.h

Go to the documentation of this file.

#ifndef HDF5FILE_H_
#define HDF5FILE_H_
 
#include <Eigen/Core>
 
#include "MUQ/Utilities/HDF5/HDF5Types.h"
#include "MUQ/Utilities/HDF5/PathTools.h"
 
#include <vector>
#include <memory>
 
#include <hdf5.h>
#include <hdf5_hl.h>
 
 
namespace muq
{
 
namespace Utilities
{
 
    class HDF5File : public std::enable_shared_from_this<HDF5File> {
    public:
 
 
    HDF5File(std::string const& filename_);
 
    virtual ~HDF5File();
 
 
    void Open(std::string const& filename_);
 
 
    void Close();
 
 
        void Copy(std::string const& destName, std::shared_ptr<HDF5File> srcFile, std::string const& srcName);
 
 
    template<typename scalarType, int fixedRows, int fixedCols>
    void WriteMatrix(std::string name, Eigen::Matrix<scalarType, fixedRows, fixedCols> const& dataset) {
 
      // data set name must begin with '/'
      if( name.at(0)!='/' ) {
        std::cerr << std::endl << "ERROR: Paths in the HDF5 file must start with a forward slash (/)" << std::endl << "\tHDF5File::WriteMatrix(std::string const&, Eigen::Matrix<scalarType, fixedRows, fixedCols> const&)" << std::endl << std::endl;
        assert(name.at(0) == '/');
        }
 
      if((name.at(0)=='/')&&(name.size()>0)){
        if(name.at(1)=='/'){
          name.erase(name.begin());
        }
      }
 
        // make sure the file is open
        assert(fileID>0);
 
        // set the maximum size of the data set to be unlimited
        const hsize_t maxdim[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
 
        const hsize_t dimsf[2] = {(hsize_t)dataset.rows(), (hsize_t)dataset.cols()};
 
        // create the dataspace for the dataset.
        const hid_t filespace = H5Screate_simple(2, dimsf, maxdim);
        assert(filespace>0);
 
        // the dataset we are writing to
        hid_t dataID;
        if( DoesDataSetExist(name) ) {
 
            // open the data
            dataID = H5Dopen(fileID, name.c_str(), H5P_DEFAULT);
 
            // get the dataspace dimension size
            hsize_t dims[2];
            H5Sget_simple_extent_dims(H5Dget_space(dataID), dims, nullptr);
 
            if( (dimsf[0]!=dims[0]) || (dimsf[1]!=dims[1]) ) { // if the data size changes ...
                // Extend the dataset
                H5Dset_extent(dataID, dimsf);
            }
 
        } else {
 
            // get the parent path
            std::string parentPath = GetParentPath(name);
 
            // make sure the parent exists (and create it if it does not)
            CreateGroup(parentPath);
 
            // modify data set creation properties --- enable chunking
            const hid_t prop = H5Pcreate(H5P_DATASET_CREATE);
 
            // set chunk size to be (1,1) -- may be a bad idea of the data set is extremely large
            hsize_t chunk[2] = {100,100};
            if(fixedRows>0){
            chunk[0] = std::min<hsize_t>(chunk[0],fixedRows);
            }
 
            if(fixedCols>0){
            chunk[1] = std::min<hsize_t>(chunk[1],fixedRows);
            }
            H5Pset_chunk(prop, 2, chunk);
 
            // create a dataset for each process
            dataID = H5Dcreate(fileID,  // Location identifier
                            name.c_str(), // Dataset name
                            HDF5_Type<scalarType>::GetFlag(), // Datatype identifier
                            filespace,  // Dataspace identifier
                            H5P_DEFAULT,  // Link creation property list
                            prop,  // Dataset creation property list
                            H5P_DEFAULT); // Dataset access property list
 
            // close the creation properties
            H5Pclose(prop);
        }
 
        // constant reference (avoid copying) to the data transpose because of column versus row major conventions
        Eigen::Matrix<scalarType, fixedRows, fixedCols> dataset_ = dataset.transpose();
 
        // write the data to file
        H5Dwrite(dataID, HDF5_Type<scalarType>::GetFlag(), H5S_ALL, H5S_ALL, H5P_DEFAULT, dataset_.data());
 
        // close the filespace
        H5Sclose(filespace);
 
        // close the dataset
        H5Dclose(dataID);
 
    }
 
 
    template<typename scalarType, int fixedRows, int fixedCols>
        void WritePartialMatrix(std::string const& name,
                Eigen::Matrix<scalarType, fixedRows, fixedCols> const& data,
                unsigned int const row, unsigned int const col)
    {
        // data set name must begin with '/'
        if( name.at(0)!='/' ) {
        std::cerr << std::endl << "ERROR: Paths in the HDF5 file must start with a forward slash (/)" << std::endl << "\tHDF5File::WriteMatrix(std::string const&, Eigen::Matrix<scalarType, fixedRows, fixedCols> const&)" << std::endl << std::endl;
        assert(name.at(0) == '/');
        }
 
        // make sure the file is open
        assert(fileID>0);
 
        // how many elements to move in each dimension
        const hsize_t stride[2] = {1, 1};
 
        // how many blocks to move in each dimension
        const hsize_t count[2] = {1, 1};
 
        // make sure the data set exists
        if( !DoesDataSetExist(name) ) {
        std::cerr << std::endl << "ERROR: Dataset " << name << " does not exsts." << std::endl << std::endl;
        assert(DoesDataSetExist(name));
        }
 
        // open the data set
        const hid_t dataID = H5Dopen(fileID, name.c_str(), H5P_DEFAULT);
 
        // get the space in the file
        const hid_t fullspace = H5Dget_space(dataID);
 
        // the location in the dataset where we start writing
        const hsize_t start[2] = {row, col};
 
        // the size of each block
        const hsize_t block[2] = {(hsize_t)data.rows(), (hsize_t)data.cols()};
 
        // get the hyperslab (the subspace needed for the partial write)
        H5Sselect_hyperslab(fullspace, H5S_SELECT_SET, start, stride, count, block);
 
        // get the dataspace for the partial write
        const hsize_t limitedBlock = H5Screate_simple(2, block, block);
 
        // constant reference (prevent copying) to transpose data
        const Eigen::Matrix<scalarType, fixedRows, fixedCols>& data_ = data.transpose();
 
        // write to file
        auto write = H5Dwrite(dataID,
                  HDF5_Type<scalarType>::GetFlag(),
                  limitedBlock,
                  fullspace,
                  H5P_DEFAULT,
                  data_.data());
 
        // close the spaces
        H5Sclose(limitedBlock);
        H5Sclose(fullspace);
        H5Dclose(dataID);
    }
 
 
    template<typename scalarType = double, int fixedRows = Eigen::Dynamic, int fixedCols = Eigen::Dynamic>
        Eigen::Matrix<scalarType, fixedRows, fixedCols> ReadMatrix(std::string const& name) const {
 
        // make sure the file is open
        assert(fileID>0);
 
        if( !DoesDataSetExist(name) ) { // if the data does not exist
              // ... return an empty matrix.
          return Eigen::Matrix<scalarType, fixedRows, fixedCols>();
        }
 
        // get the dataset ID
        const hid_t dataset = H5Dopen2(hid_t(fileID), name.c_str(), H5P_DEFAULT);
 
        // get the space associated with this data set
        const hid_t filespace = H5Dget_space(dataset);
 
      const int ndims = H5Sget_simple_extent_ndims(filespace);
 
        // get the size of the dataset
        std::vector<hsize_t> dimsr(ndims);
        H5Sget_simple_extent_dims(filespace, dimsr.data(), nullptr);
 
        // declare the matrix that we will read into
        Eigen::Matrix<scalarType, Eigen::Dynamic, Eigen::Dynamic> pt;
 
        if( ndims==2 ) { // if it is a matrix ...
              pt.resize(dimsr[1], dimsr[0]);
        } else { // if it is a vector ...
          if(fixedRows==1){
              pt = Eigen::Matrix<scalarType, 1, fixedCols>(dimsr[0]);
          }else{
              pt = Eigen::Matrix<scalarType, fixedRows, 1>(dimsr[0]);
          }
 
        }
 
        // get the property list
        const hid_t prop = H5Dget_create_plist(dataset);
 
        // get the memory for this data set
        const hid_t memspace = H5Screate_simple(ndims, dimsr.data(), nullptr);
 
        // read the data
      H5Dread(dataset, HDF5_Type<scalarType>::GetFlag(), memspace, filespace, H5P_DEFAULT, pt.data());
 
        // close the creation properties
        H5Pclose(prop);
 
        // close the filespace
        H5Sclose(filespace);
 
        // close the memory space
        H5Sclose(memspace);
 
        // close the dataset
        H5Dclose(dataset);
 
        if(ndims==2) {
            // hdf5 and eigen have different memory convention so we read in the matrix transpose
        if(fixedRows==1){
          return pt;
        }else{
          return pt.transpose();
        }
        }
 
        // return the data
        return pt;
    }
 
 
    template<typename scalarType=double, int fixedRows = Eigen::Dynamic, int fixedCols = Eigen::Dynamic>
    Eigen::Matrix<scalarType, fixedRows, fixedCols> ReadPartialMatrix(std::string const& name,
                                      int rowStart,
                                      int colStart,
                                      int numRows,
                                      int numCols) const
    {
 
 
        // make sure the file is open
        assert(fileID>0);
 
        if( !DoesDataSetExist(name) ) { // if the data does not exist
            // ... return an empty matrix.
            return Eigen::Matrix<scalarType, fixedRows, fixedCols>();
        }
 
        // get the dataset ID
        const hid_t dataset = H5Dopen2(fileID, name.c_str(), H5P_DEFAULT);
 
        // get the space associated with this data set
        const hid_t dataspace = H5Dget_space(dataset);
 
        // get the size of the dataset
        hsize_t dimsr[2];
        H5Sget_simple_extent_dims(dataspace, dimsr, nullptr);
 
        // declare the matrix that we will read into
        Eigen::Matrix<scalarType, fixedCols, fixedRows> pt;
 
        pt.resize(numCols,numRows); //<- will flip to proper orientation below
        if( dimsr[1]>0 ) { // if it is a matrix ...
            assert(numRows<=dimsr[0]);
            assert(numCols<=dimsr[1]);
        } else { // if it is a vector ...
            assert(numRows<dimsr[0]);
        }
 
        const hsize_t start[2]  = {(hsize_t) rowStart, (hsize_t) colStart};
        const hsize_t count[2]  = {(hsize_t) numRows, (hsize_t) numCols};
 
        herr_t status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, NULL, count, NULL);
        // get the memory for this data set
        dimsr[0] = numRows;
        dimsr[1] = numCols;
        hid_t memspace = H5Screate_simple(2, dimsr, nullptr);
 
        // read the data
        H5Dread(dataset, HDF5_Type<scalarType>::GetFlag(), memspace, dataspace, H5P_DEFAULT, pt.data());
 
        // close the filespace
        H5Sclose(dataspace);
 
        // close the memory space
        H5Sclose(memspace);
 
        // close the dataset
        H5Dclose(dataset);
 
        if( dimsr[1]>0 ) {
            // hdf5 and eigen have different memory convention so we read in the matrix transpose
            return pt.transpose();
        }
 
        // return the data
        return pt;
 
    }
 
 
    bool DoesGroupExist(std::string const& name) const;
 
 
    bool DoesDataSetExist(std::string const& name) const;
 
    bool IsDataSet(std::string const& name) const;
    bool IsGroup(std::string const& name) const;
 
 
    Eigen::VectorXi GetDataSetSize(std::string const name) const;
 
 
    void CreateGroup(std::string const& name);
 
 
 
 
    template<typename scalarType>
  void CreateDataset(std::string const& name, int rows, int cols)
  {
 
    // make sure the file is open
    assert(fileID>0);
 
    // set the maximum size of the data set to be unlimited
    const hsize_t maxdim[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
 
    // the sizes of the data set
    const hsize_t dimsf[2] = {(hsize_t)rows, (hsize_t)cols};
 
    // create the dataspace for the dataset.
    const hid_t filespace = H5Screate_simple(2, dimsf, maxdim);
    assert(filespace>0);
 
    // the dataset we are writing to
    hid_t dataID;
 
    if( DoesDataSetExist(name) ) {
 
        // open the data
        dataID = H5Dopen(fileID, name.c_str(), H5P_DEFAULT);
 
        // get the dataspace dimension size
        hsize_t dims[2];
        H5Sget_simple_extent_dims(H5Dget_space(dataID), dims, nullptr);
 
        if( (dimsf[0]!=dims[0]) || (dimsf[1]!=dims[1]) ) { // if the data size changes ...
            // Extend the dataset
            H5Dset_extent(dataID, dimsf);
        }
 
    } else {
 
        // get the parent path
        std::string parentPath = GetParentPath(name);
 
        // make sure the parent exists (and create it if it does not)
        CreateGroup(parentPath);
 
        // modify data set creation properties --- enable chunking
        const hid_t prop = H5Pcreate(H5P_DATASET_CREATE);
 
        // set chunk size to be (1,1) -- may be a bad idea of the data set is extremely large
        hsize_t chunk[2] = {std::min<hsize_t>(100,rows), std::min<hsize_t>(100,cols)};
        H5Pset_chunk(prop, 2, chunk);
 
        // create a dataset for each process
        dataID = H5Dcreate(fileID, name.c_str(), HDF5_Type<scalarType>::GetFlag(), filespace, H5P_DEFAULT, prop, H5P_DEFAULT);
        assert(dataID>0);
 
        // close the creation properties
        H5Pclose(prop);
    }
 
    // close the filespace
    H5Sclose(filespace);
 
    // close the dataset
    H5Dclose(dataID);
  }
 
 
    template<typename scalarType, int fixedRows>
    void WriteVectorAttribute(std::string const& datasetName,
                  std::string const& attributeName,
                  Eigen::Matrix<scalarType, fixedRows, 1> const& attribute)
    {
 
        // make sure the file is open
        assert(fileID>0);
 
        // Create the group if necessary
        if( !DoesDataSetExist(datasetName) || !DoesGroupExist(datasetName) )
        CreateGroup(datasetName);
 
        // set the attribute
        const herr_t set_result = H5LTset_attribute(fileID,
                            datasetName.c_str(),
                            attributeName.c_str(),
                            HDF5_Type<scalarType>::GetFlag(),
                            (void*) attribute.data(),
                            attribute.rows());
        assert(set_result>=0);
    }
 
 
    template<typename attType = double>
    void WriteScalarAttribute(std::string const& datasetName,
                  std::string const& attributeName,
                  attType const& attribute)
    {
        // make the scalar a vector
        Eigen::Matrix<attType, 1, 1> vec(1);
        vec << attribute;
 
        // call the vector attribute version
        WriteVectorAttribute<attType, 1>(datasetName, attributeName, vec);
    }
 
 
    void WriteStringAttribute(std::string const& datasetName,
                  std::string const& attributeName,
                  std::string const& attribute);
 
 
    template<typename scalarType>
    scalarType GetScalarAttribute(std::string const& datasetName,
                      std::string const& attributeName) const {
 
        // read as a vector attribute
        Eigen::Matrix<scalarType, Eigen::Dynamic, 1> att = GetVectorAttribute<scalarType>(datasetName, attributeName);
        assert(att.size() == 1);
 
        // return the sclar
        return att(0);
    }
 
 
    template<typename scalarType>
    Eigen::Matrix<scalarType, Eigen::Dynamic, 1> GetVectorAttribute(std::string const& datasetName,
                                    std::string const& attributeName) const
    {
        // make sure the file is open
        assert(fileID>0);
 
        // make sure the dataset exists
        assert(DoesDataSetExist(datasetName) || DoesGroupExist(datasetName));
 
        // get the dimensions of the attribute
        int rank[1];
        hsize_t dims[2];
        auto get_ndim = H5LTget_attribute_ndims(fileID, datasetName.c_str(), attributeName.c_str(), rank);
        assert(get_ndim >= 0);
        assert(rank[0]==1);
 
        H5T_class_t typeClass;
        size_t typeSize;
        auto status = H5LTget_attribute_info(fileID, datasetName.c_str(), attributeName.c_str(), dims, &typeClass, &typeSize);
        assert(status >= 0);
 
        // create the vector to return
        Eigen::Matrix<scalarType, Eigen::Dynamic, 1> data(dims[0]);
 
        // get the attribute
 
        auto get_att = H5LTget_attribute(fileID, datasetName.c_str(), attributeName.c_str(), HDF5_Type<scalarType>::GetFlag(), data.data());
        assert(get_att >= 0);
 
        // translate it into and Eigen type
        //Translater<std::vector<scalarType>, Eigen::Matrix<scalarType, Eigen::Dynamic, 1> > trans(data);
        return data;
    }
 
 
    std::string GetStringAttribute(std::string const& datasetName, std::string const& attributeName) const;
 
 
    std::vector<std::string> GetChildren(std::string base = "/") const;
 
    void FlushFile();
 
 
    void MergeFile(std::shared_ptr<HDF5File> const& otherFile);
 
 
    hid_t fileID = -1;
 
 
    std::string filename = "";
 
    private:
 
      bool DoesFileExist(const std::string& name) const;
 
    };
 
} // namespace Utilities
} // namespace muq
 
#endif