Field3D
SparseDataReader.h
Go to the documentation of this file.
1//----------------------------------------------------------------------------//
2
3/*
4 * Copyright (c) 2009 Sony Pictures Imageworks Inc
5 *
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the
17 * distribution. Neither the name of Sony Pictures Imageworks nor the
18 * names of its contributors may be used to endorse or promote
19 * products derived from this software without specific prior written
20 * permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33 * OF THE POSSIBILITY OF SUCH DAMAGE.
34 */
35
36//----------------------------------------------------------------------------//
37
42//----------------------------------------------------------------------------//
43
44#ifndef _INCLUDED_Field3D_SparseDataReader_H_
45#define _INCLUDED_Field3D_SparseDataReader_H_
46
47//----------------------------------------------------------------------------//
48
49#include <hdf5.h>
50#include <string.h> // for memcpy
51
52#include "Hdf5Util.h"
53#include "Log.h"
54
55//----------------------------------------------------------------------------//
56
57#include "ns.h"
58
60
61//----------------------------------------------------------------------------//
62// SparseDataReader
63//----------------------------------------------------------------------------//
64
69template <class Data_T>
71{
72public:
73
74 // Constructors --------------------------------------------------------------
75
78 SparseDataReader(hid_t location, int valuesPerBlock, int occupiedBlocks);
79
80 // Main methods --------------------------------------------------------------
81
84 void readBlock(int idx, Data_T &result);
85
88 void readBlockList(int idx, const std::vector<Data_T*>& memoryList);
89
90private:
91
92 // Data members --------------------------------------------------------------
93
95
98
99 const std::string k_dataStr;
100};
101
102//----------------------------------------------------------------------------//
103// SparseDataReader implementations
104//----------------------------------------------------------------------------//
105
106template <class Data_T>
107SparseDataReader<Data_T>::SparseDataReader(hid_t location, int valuesPerBlock,
108 int occupiedBlocks)
109 : m_location(location),
110 m_valuesPerBlock(valuesPerBlock),
111 m_occupiedBlocks(occupiedBlocks),
112 k_dataStr("data")
113{
114
115}
116
117//----------------------------------------------------------------------------//
118
119template <class Data_T>
120void SparseDataReader<Data_T>::readBlock(int idx, Data_T &result)
121{
122 using namespace Hdf5Util;
123 using namespace Exc;
124
126
128 Hdf5Util::H5ScopedDget_space fileDataSpace;
130 Hdf5Util::H5ScopedScreate memDataSpace;
131
132 hsize_t dims[2];
133 hsize_t memDims[1];
134
135 // Open the data set
136 dataSet.open(m_location, k_dataStr, H5P_DEFAULT);
137 if (dataSet.id() < 0)
138 throw OpenDataSetException("Couldn't open data set: " + k_dataStr);
139
140 // Get the space and type
141 fileDataSpace.open(dataSet.id());
142 dataType.open(dataSet.id());
143 if (fileDataSpace.id() < 0)
144 throw GetDataSpaceException("Couldn't get data space");
145 if (dataType.id() < 0)
146 throw GetDataTypeException("Couldn't get data type");
147
148 // Make the memory data space
149 memDims[0] = m_valuesPerBlock;
150 memDataSpace.create(H5S_SIMPLE);
151 H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
152
153 // Get the dimensions and check they match
154 H5Sget_simple_extent_dims(fileDataSpace.id(), dims, NULL);
155 if (dims[1] != static_cast<hsize_t>(m_valuesPerBlock)) {
156 throw FileIntegrityException("Block length mismatch in "
157 "SparseDataReader");
158 }
159 if (dims[0] != static_cast<hsize_t>(m_occupiedBlocks))
160 throw FileIntegrityException("Block count mismatch in "
161 "SparseDataReader");
162
163 hsize_t offset[2];
164 hsize_t count[2];
165 herr_t status;
166
167 offset[0] = idx; // Index of block
168 offset[1] = 0; // Index of first data in block. Always 0
169 count[0] = 1; // Number of columns to read. Always 1
170 count[1] = m_valuesPerBlock; // Number of values in one column
171
172 status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
173 offset, NULL, count, NULL);
174
175 if (status < 0) {
176 throw ReadHyperSlabException("Couldn't select slab in readBlock(): " +
177 boost::lexical_cast<std::string>(idx));
178 }
179
180 status = H5Dread(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
181 memDataSpace.id(), fileDataSpace.id(),
182 H5P_DEFAULT, &result);
183}
184
185//----------------------------------------------------------------------------//
186
187template <class Data_T>
189(int idxLo, const std::vector<Data_T*>& memoryList)
190{
191 using namespace Hdf5Util;
192 using namespace Exc;
193
195
197 Hdf5Util::H5ScopedDget_space fileDataSpace;
199 Hdf5Util::H5ScopedScreate memDataSpace;
200
201 hsize_t dims[2];
202 hsize_t memDims[1];
203
204 // Open the data set
205 dataSet.open(m_location, k_dataStr, H5P_DEFAULT);
206 if (dataSet.id() < 0)
207 throw OpenDataSetException("Couldn't open data set: " + k_dataStr);
208
209 // Get the space and type
210 fileDataSpace.open(dataSet.id());
211 dataType.open(dataSet.id());
212 if (fileDataSpace.id() < 0)
213 throw GetDataSpaceException("Couldn't get data space");
214 if (dataType.id() < 0)
215 throw GetDataTypeException("Couldn't get data type");
216
217 // Make the memory data space
218 memDims[0] = m_valuesPerBlock;
219 memDataSpace.create(H5S_SIMPLE);
220 H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
221
222 // Get the dimensions and check they match
223 H5Sget_simple_extent_dims(fileDataSpace.id(), dims, NULL);
224 if (dims[1] != static_cast<hsize_t>(m_valuesPerBlock)) {
225 throw FileIntegrityException("Block length mismatch in "
226 "SparseDataReader");
227 }
228 if (dims[0] != static_cast<hsize_t>(m_occupiedBlocks))
229 throw FileIntegrityException("Block count mismatch in "
230 "SparseDataReader");
231
232 hsize_t offset[2];
233 hsize_t count[2];
234 herr_t status;
235
236 offset[0] = idxLo; // Index of block
237 offset[1] = 0; // Index of first data in block. Always 0
238 count[0] = memoryList.size(); // Number of columns to read.
239 count[1] = m_valuesPerBlock; // Number of values in one column
240
241 status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
242 offset, NULL, count, NULL);
243 if (status < 0) {
244 throw ReadHyperSlabException("Couldn't select slab in readBlockList():" +
245 boost::lexical_cast<std::string>(idxLo));
246 }
247
248 // Make the memory data space ---
249
250 Hdf5Util::H5ScopedScreate localMemDataSpace;
251 hsize_t fileDims[2];
252 fileDims[0] = memoryList.size();
253 fileDims[1] = m_valuesPerBlock;
254 localMemDataSpace.create(H5S_SIMPLE);
255 H5Sset_extent_simple(localMemDataSpace.id(), 2, fileDims, NULL);
256
257 // Setup the temporary memory region ---
258
259 int bytesPerValue = 0;
260 {
262 if (t == H5T_NATIVE_CHAR)
263 bytesPerValue = 1;
264 else if (t == H5T_NATIVE_SHORT)
265 bytesPerValue = 2;
266 else if (t == H5T_NATIVE_FLOAT)
267 bytesPerValue = 4;
268 else if (t == H5T_NATIVE_DOUBLE)
269 bytesPerValue = 8;
270 }
271
272 int dim = sizeof(Data_T) / bytesPerValue;
273 std::vector<Data_T> bigblock(memoryList.size() * m_valuesPerBlock/dim);
274
275 status = H5Dread(dataSet.id(),
277 localMemDataSpace.id(),
278 fileDataSpace.id(),
279 H5P_DEFAULT, &bigblock[0]);
280
281 if (status < 0) {
282 throw Hdf5DataReadException("Couldn't read slab " +
283 boost::lexical_cast<std::string>(idxLo));
284 }
285
286 // Distribute block data into memory slots ---
287 for (size_t i = 0; i < memoryList.size(); ++i) {
288 memcpy(memoryList[i],
289 &bigblock[i * m_valuesPerBlock / dim],
290 bytesPerValue * m_valuesPerBlock);
291 }
292}
293
294//----------------------------------------------------------------------------//
295
297
298//----------------------------------------------------------------------------//
299
300#endif
Contains various utility functions for Hdf5.
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition Hdf5Util.cpp:67
boost::recursive_mutex::scoped_lock GlobalLock
Definition Hdf5Util.h:78
Contains the Log class which can be used to redirect output to an arbitrary destination.
hid_t id() const
Query the hid_t value.
Definition Hdf5Util.h:100
Scoped object - opens a dataset on creation and closes it on destruction.
Definition Hdf5Util.h:387
void open(hid_t dataset_id)
Definition Hdf5Util.h:398
Scoped object - opens a dataset on creation and closes it on destruction.
Definition Hdf5Util.h:417
void open(hid_t dataset_id)
Definition Hdf5Util.h:428
Scoped object - opens a dataset on creation and closes it on destruction.
Definition Hdf5Util.h:357
void open(hid_t parentLocation, const std::string &name, hid_t dapl_id)
Definition Hdf5Util.h:368
Scoped object - creates a dataspace on creation and closes it on destruction.
Definition Hdf5Util.h:235
void create(H5S_class_t type)
Definition Hdf5Util.h:246
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
void readBlock(int idx, Data_T &result)
Reads a block, storing the data in result, which is assumed to contain enough room for m_valuesPerBlo...
SparseDataReader(hid_t location, int valuesPerBlock, int occupiedBlocks)
Constructor. Requires knowledge of the Hdf5 location where data is stored.
const std::string k_dataStr
Namespace for Exception objects.
Definition Exception.h:57
Contains utility functions and classes for Hdf5 files.
Definition Hdf5Util.h:86
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition ns.h:58
static hid_t h5type()