ESDM
Middleware for Earth System Data
esdm.h
Go to the documentation of this file.
1 
5 #ifndef ESDM_H
6 #define ESDM_H
7 
8 #include <stdbool.h>
9 #include <stddef.h>
10 #include <sys/stat.h>
11 #include <sys/types.h>
12 #include <unistd.h>
13 
14 #include <esdm-datatypes.h>
15 
16 #ifdef __cplusplus
17 extern "C" {
18 #endif
19 
21 // ESDM ///////////////////////////////////////////////////////////////////////
23 
24 // Initialization /////////////////////////////////////////////////////////////
25 
26 // These functions must be used before calling init:
27 
37 
47 
56 esdm_status esdm_load_config_str(const char *str);
57 
69 
70 int esdm_is_initialized();
71 
79 
80 // I/O ////////////////////////////////////////////////////////////////////////
81 
92 esdm_status esdm_write(esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace);
93 
97 //esdm_status esdm_write_so(esdm_dataset_t *dataset, void *buf, int64_t *size, int64_t *offset);
98 
109 esdm_status esdm_read(esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace);
110 
114 //esdm_status esdm_read_so(esdm_dataset_t *dataset, void *buf, int64_t *size, int64_t *offset);
115 
116 
127 typedef void* (*esdm_stream_func_t)(esdm_dataspace_t *space, void * buff, void * user_ptr, void* esdm_fill_value);
128 typedef void (*esdm_reduce_func_t)(esdm_dataspace_t *space, void * user_ptr, void * stream_func_out);
129 esdm_status esdm_read_stream(esdm_dataset_t *dataset, esdm_dataspace_t *space, void * user_ptr, esdm_stream_func_t stream_func, esdm_reduce_func_t reduce_func);
130 
131 // Auxiliary //////////////////////////////////////////////////////////////////
132 
133 //size_t esdm_sizeof(esdm_type_t type);
134 #define esdm_sizeof(type) (type->size)
135 
146  ESDM_FORMAT_DELETE = 1,
147  ESDM_FORMAT_CREATE = 2,
148  ESDM_FORMAT_IGNORE_ERRORS = 4,
149  ESDM_FORMAT_PURGE_RECREATE = 7
150 };
151 
152 esdm_status esdm_mkfs(int format_flags, data_accessibility_t target);
153 
154 // LOGGING
155 
156 /*
157 Loglevel for stdout.
158 */
159 void esdm_loglevel(esdm_loglevel_e loglevel);
160 void esdm_log_on_exit(int on);
161 /*
162  Keeps a log to record last messages for crashes
163  Must be called from a single master thread
164  NOTE: logging into the shared buffer costs performance.
165 */
166 void esdm_loglevel_buffer(esdm_loglevel_e loglevel);
167 
168 // Statistics
169 
174 
179 
181 // Container //////////////////////////////////////////////////////////////////
183 
197 esdm_status esdm_container_create(const char *name, int allow_overwrite, esdm_container_t **out_container);
198 
210 esdm_status esdm_container_open(const char *name, int esdm_mode_flags, esdm_container_t **out_container);
211 
212 int esdm_container_get_mode_flags(esdm_container_t * container);
213 
228 
229 esdm_status esdm_container_delete_attribute(esdm_container_t *c, const char *name);
230 
231 esdm_status esdm_container_link_attribute(esdm_container_t *container, int overwrite, smd_attr_t *attr);
232 
233 /* This function returns the attributes */
234 esdm_status esdm_container_get_attributes(esdm_container_t *container, smd_attr_t **out_metadata);
235 
236 esdm_status esdm_container_delete(esdm_container_t *container);
237 
248 
249 /*
250  * Check if the dataset with the given name exists.
251  *
252  * @param [in] container an existing container to query
253  * @param [in] name the identifier to check, must not be NULL
254  *
255  * @return true if the identifier already exists within the container
256  */
257 bool esdm_container_dataset_exists(esdm_container_t * container, char const * name);
258 
259 /*
260  * Return the number of datasets in the container.
261  *
262  * @param [in] container an existing container to query
263  *
264  * @return the number of datasets
265  */
266 int esdm_container_dataset_count(esdm_container_t * container);
267 
268 void esdm_container_set_status_dirty(esdm_container_t * container);
269 
270 /*
271  * Return the n-th dataset in the container array.
272  *
273  * @param [in] container an existing container to query
274  * @param [in] dset_number the number (it shall exists according to esdm_container_dataset_count())
275  *
276  * @return the dataset or NULL, if dset_number >= count
277  */
278 esdm_dataset_t * esdm_container_dataset_from_array(esdm_container_t * container, int dset_number);
279 
281 // Dataset ////////////////////////////////////////////////////////////////////
283 
284 /*
285  functions to change the size of the dataspace
286  */
287 esdm_status esdm_dataset_update_size(esdm_dataset_t *dset, uint64_t * sizes);
288 int64_t const * esdm_dataset_get_size(esdm_dataset_t * dset);
289 /*
290  This function returns the actual size for ulim
291  Return a pointer to the internal size (having the same dimensions)
292  */
293 int64_t const * esdm_dataset_get_actual_size(esdm_dataset_t *dset);
294 
295 esdm_status esdm_dataset_rename(esdm_dataset_t *dataset, const char *name);
296 
297 esdm_status esdm_dataset_set_compression_hint(esdm_dataset_t * dataset, scil_user_hints_t const * hints);
298 
299 void esdm_dataset_set_status_dirty(esdm_dataset_t * dataset);
300 
301 // Dataset
302 
318 esdm_status esdm_dataset_create(esdm_container_t *container, const char *name, esdm_dataspace_t *dataspace, esdm_dataset_t **out_dataset);
319 
320 /*
321  The value to be used if data hasn't been written to a datapoint, it must be of the same type as the dataset type.
322  If the fill value was already set, overwrite it.
323  */
324 esdm_status esdm_dataset_set_fill_value(esdm_dataset_t *dataset, void const * value);
325 
326 /*
327  Copy the fill value into value
328  */
329 esdm_status esdm_dataset_get_fill_value(esdm_dataset_t *dataset, void * value);
330 
331 int esdm_dataset_is_fill_value_set(esdm_dataset_t *dataset);
332 
333 /*
334  Returns the name of the dataset, do not free or temper with it, the name is still owned by the dataset
335  */
336 char const * esdm_dataset_name(esdm_dataset_t *dataset);
337 
338 /*
339  Name the dimensions of a dataset
340  */
341 esdm_status esdm_dataset_name_dims(esdm_dataset_t *dataset, char * const * names);
342 
343 //esdm_status esdm_dataset_name_dimsv(esdm_dataset_t *dataset, ...);
344 
345 /*
346  Rename a single dimension
347  */
348 esdm_status esdm_dataset_rename_dim(esdm_dataset_t *dataset, char const *name, int i);
349 
350 /*
351  * out names still belongs to the dataset
352  */
353 esdm_status esdm_dataset_get_name_dims(esdm_dataset_t *dataset, char const *const **out_names);
354 
366 esdm_type_t esdm_dataset_get_type(esdm_dataset_t * d);
367 
368 esdm_status esdm_dataset_change_name(esdm_dataset_t *dset, char const * new_name);
369 
370 esdm_status esdm_dataset_iterator(esdm_container_t *container, esdm_dataset_iterator_t **out_iter);
371 
384 esdm_status esdm_dataset_open(esdm_container_t *container, const char *name, int esdm_mode_flags, esdm_dataset_t **out_dataset);
385 
386 /*
387  Similar to esdm_dataset_open but returns the dataset without opening it
388  */
389 esdm_status esdm_dataset_by_name(esdm_container_t *container, const char *name, int esdm_mode_flags, esdm_dataset_t **out_dataset);
390 
391 /*
392  * Obtain a reference to the dataset, if it was not yet open, it will be openend and metadata will be fetched.
393  * To return the dataset, call dataset_close()
394  *
395  * This function is *not thread-safe*.
396  * Only a single master thread must be used to call into ESDM.
397  */
398 esdm_status esdm_dataset_ref(esdm_dataset_t *dataset);
399 
409 
421 
422 esdm_status esdm_dataset_delete(esdm_dataset_t *dataset);
423 
424 esdm_status esdm_dataset_delete_attribute(esdm_dataset_t *dataset, const char *name);
425 
426 /* This function adds the metadata to the ESDM */
427 
428 esdm_status esdm_dataset_link_attribute(esdm_dataset_t *dset, int overwrite, smd_attr_t *attr);
429 
430 /* This function returns the attributes */
431 esdm_status esdm_dataset_get_attributes(esdm_dataset_t *dataset, smd_attr_t **out_metadata);
432 
434 // Dataspace //////////////////////////////////////////////////////////////////
436 
450 esdm_status esdm_dataspace_create(int64_t dims, int64_t *sizes, esdm_type_t type, esdm_dataspace_t **out_dataspace);
451 
466 esdm_status esdm_dataspace_create_full(int64_t dims, int64_t *size, int64_t *offset, esdm_type_t type, esdm_dataspace_t **out_dataspace);
467 
473 #define esdm_simple_dataspace_construct_internal(dimCount, xOffset, xSize, yOffset, ySize, zOffset, zSize, dataType) \
474  ((esdm_simple_dspace_t){ \
475  .ptr = &(esdm_dataspace_t){ \
476  .type = dataType, \
477  .dims = dimCount, \
478  .size = (int64_t [3]){xSize, ySize, zSize}, \
479  .offset = (int64_t [3]){xOffset, yOffset, zOffset}, \
480  .strideBacking = (int64_t [3]){0}, \
481  .stride = NULL \
482  } \
483  })
484 
491 #define esdm_dataspace_1do(offset, size, type) esdm_simple_dataspace_construct_internal(1, offset, size, 0, 0, 0, 0, type)
492 
499 #define esdm_dataspace_1d(size, type) esdm_simple_dataspace_construct_internal(1, 0, size, 0, 0, 0, 0, type)
500 
507 #define esdm_dataspace_2do(xOffset, xSize, yOffset, ySize, type) esdm_simple_dataspace_construct_internal(2, xOffset, xSize, yOffset, ySize, 0, 0, type)
508 
515 #define esdm_dataspace_2d(xSize, ySize, type) esdm_simple_dataspace_construct_internal(2, 0, xSize, 0, ySize, 0, 0, type)
516 
523 #define esdm_dataspace_3do(xOffset, xSize, yOffset, ySize, zOffset, zSize, type) esdm_simple_dataspace_construct_internal(3, xOffset, xSize, yOffset, ySize, zOffset, zSize, type)
524 
531 #define esdm_dataspace_3d(xSize, ySize, zSize, type) esdm_simple_dataspace_construct_internal(3, 0, xSize, 0, ySize, 0, zSize, type)
532 
533 
544 
558 esdm_status esdm_dataspace_subspace(esdm_dataspace_t *dataspace, int64_t dims, int64_t *size, int64_t *offset, esdm_dataspace_t **out_dataspace);
559 
572 
573 
574 int64_t esdm_dataspace_get_dims(esdm_dataspace_t * d);
575 int64_t const* esdm_dataspace_get_size(esdm_dataspace_t * d);
576 int64_t const* esdm_dataspace_get_offset(esdm_dataspace_t * d);
577 esdm_type_t esdm_dataspace_get_type(esdm_dataspace_t * d);
578 
583 
588 
600 void esdm_dataspace_getEffectiveStride(esdm_dataspace_t* space, int64_t* out_stride);
601 
612 int64_t esdm_dataspace_elementOffset(esdm_dataspace_t* space, int64_t* coords);
613 
614 
643 esdm_status esdm_dataspace_set_stride(esdm_dataspace_t* dataspace, int64_t* stride);
644 
661 
685 esdm_status esdm_dataspace_copy_data(esdm_dataspace_t* sourceSpace, void *sourceData, esdm_dataspace_t* destSpace, void *destData);
686 
699 esdm_status esdm_dataspace_fill(esdm_dataspace_t* dataspace, void* data, void* fillElement);
700 
711 
717 void esdm_dataspace_serialize(const esdm_dataspace_t *dataspace, smd_string_stream_t* stream);
718 
719 void esdm_dataspace_print(esdm_dataspace_t *dataspace);
720 
722 // Fragment ///////////////////////////////////////////////////////////////////
724 
729 esdm_status esdm_fragment_deserialize(void *serialized_fragment, esdm_fragment_t **_out_fragment);
730 
738 
745 
754 
765 
776 
792 void esdm_fragment_print(esdm_fragment_t *fragment);
793 
794 #ifdef __cplusplus
795 }
796 #endif
797 
798 #endif
Datatype primitives provided by ESDM.
esdm_status
Definition: esdm-datatypes.h:37
esdm_loglevel
Definition: esdm-datatypes.h:53
esdm_status esdm_load_config_str(const char *str)
Definition: esdm.c:57
esdm_status esdm_container_close(esdm_container_t *container)
Definition: esdm-datatypes.c:278
esdm_statistics_t esdm_write_stats()
Definition: esdm.c:265
esdm_status esdm_dataspace_create_full(int64_t dims, int64_t *size, int64_t *offset, esdm_type_t type, esdm_dataspace_t **out_dataspace)
Definition: esdm-datatypes.c:1282
void *(* esdm_stream_func_t)(esdm_dataspace_t *space, void *buff, void *user_ptr, void *esdm_fill_value)
Definition: esdm.h:127
esdm_status esdm_dataset_get_dataspace(esdm_dataset_t *dset, esdm_dataspace_t **out_dataspace)
Definition: esdm.c:68
esdm_status esdm_set_procs_per_node(int procs)
Definition: esdm.c:41
void esdm_fragment_print(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:1588
esdm_status esdm_dataspace_subspace(esdm_dataspace_t *dataspace, int64_t dims, int64_t *size, int64_t *offset, esdm_dataspace_t **out_dataspace)
Definition: esdm-datatypes.c:1378
esdm_status esdm_dataspace_makeContiguous(esdm_dataspace_t *dataspace, esdm_dataspace_t **out_dataspace)
Definition: esdm-datatypes.c:1431
esdm_status esdm_read(esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace)
Definition: esdm.c:199
esdm_status esdm_init()
Definition: esdm.c:79
int64_t esdm_dataspace_total_bytes(esdm_dataspace_t *d)
Definition: esdm-datatypes.c:384
esdm_status esdm_dataspace_copy_data(esdm_dataspace_t *sourceSpace, void *sourceData, esdm_dataspace_t *destSpace, void *destData)
Definition: esdm-scheduler.c:307
esdm_status esdm_dataset_create(esdm_container_t *container, const char *name, esdm_dataspace_t *dataspace, esdm_dataset_t **out_dataset)
Definition: esdm-datatypes.c:650
esdm_status esdm_container_commit(esdm_container_t *container)
Definition: esdm-datatypes.c:251
esdm_status esdm_fragment_load(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:482
esdm_status esdm_container_create(const char *name, int allow_overwrite, esdm_container_t **out_container)
Definition: esdm-datatypes.c:43
esdm_status esdm_dataspace_copy(esdm_dataspace_t *orig, esdm_dataspace_t **out_dataspace)
Definition: esdm-datatypes.c:1338
esdm_status esdm_dataset_commit(esdm_dataset_t *dataset)
Definition: esdm-datatypes.c:1076
esdm_format_flags
Definition: esdm.h:145
esdm_status esdm_fragment_retrieve(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:449
uint64_t esdm_dataspace_element_count(esdm_dataspace_t *dataspace)
Definition: esdm-datatypes.c:1604
esdm_status esdm_dataspace_fill(esdm_dataspace_t *dataspace, void *data, void *fillElement)
Definition: esdm-datatypes.c:1560
esdm_status esdm_fragment_deserialize(void *serialized_fragment, esdm_fragment_t **_out_fragment)
esdm_status esdm_dataspace_copyDatalayout(esdm_dataspace_t *dataspace, esdm_dataspace_t *strideSource)
Definition: esdm-datatypes.c:823
esdm_status esdm_fragment_destroy(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:595
esdm_status esdm_fragment_unload(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:488
void esdm_dataspace_getEffectiveStride(esdm_dataspace_t *space, int64_t *out_stride)
Definition: esdm-datatypes.c:1540
esdm_status esdm_dataset_open(esdm_container_t *container, const char *name, int esdm_mode_flags, esdm_dataset_t **out_dataset)
Definition: esdm-datatypes.c:1020
esdm_status esdm_dataspace_create(int64_t dims, int64_t *sizes, esdm_type_t type, esdm_dataspace_t **out_dataspace)
Definition: esdm-datatypes.c:1307
esdm_status esdm_finalize()
Definition: esdm.c:159
int64_t esdm_dataspace_elementOffset(esdm_dataspace_t *space, int64_t *coords)
Definition: esdm-datatypes.c:1551
esdm_status esdm_fragment_commit(esdm_fragment_t *fragment)
Definition: esdm-datatypes.c:528
esdm_status esdm_set_total_procs(int procs)
Definition: esdm.c:49
esdm_status esdm_write(esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace)
Definition: esdm.c:181
void esdm_dataspace_serialize(const esdm_dataspace_t *dataspace, smd_string_stream_t *stream)
Definition: esdm-datatypes.c:1469
esdm_status esdm_dataset_close(esdm_dataset_t *dataset)
Definition: esdm-datatypes.c:1180
esdm_status esdm_dataspace_destroy(esdm_dataspace_t *dataspace)
Definition: esdm-datatypes.c:1594
esdm_status esdm_container_open(const char *name, int esdm_mode_flags, esdm_container_t **out_container)
Definition: esdm-datatypes.c:182
esdm_status esdm_dataspace_set_stride(esdm_dataspace_t *dataspace, int64_t *stride)
Definition: esdm-datatypes.c:813
esdm_statistics_t esdm_read_stats()
Definition: esdm.c:263
Definition: esdm-datatypes-internal.h:41
Definition: esdm-datatypes-internal.h:59
Definition: esdm-datatypes.h:81
Definition: esdm-datatypes-internal.h:78
Definition: esdm-datatypes.h:101
Definition: esdm-datatypes-internal.h:16