ESDM
Middleware for Earth System Data
Macros | Typedefs | Enumerations | Functions
esdm.h File Reference

Public API of the ESDM. Inlcudes several other public interfaces. More...

#include <stdbool.h>
#include <stddef.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <esdm-datatypes.h>
Include dependency graph for esdm.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define esdm_sizeof(type)   (type->size)
 
#define esdm_simple_dataspace_construct_internal(dimCount, xOffset, xSize, yOffset, ySize, zOffset, zSize, dataType)
 
#define esdm_dataspace_1do(offset, size, type)   esdm_simple_dataspace_construct_internal(1, offset, size, 0, 0, 0, 0, type)
 
#define esdm_dataspace_1d(size, type)   esdm_simple_dataspace_construct_internal(1, 0, size, 0, 0, 0, 0, type)
 
#define esdm_dataspace_2do(xOffset, xSize, yOffset, ySize, type)   esdm_simple_dataspace_construct_internal(2, xOffset, xSize, yOffset, ySize, 0, 0, type)
 
#define esdm_dataspace_2d(xSize, ySize, type)   esdm_simple_dataspace_construct_internal(2, 0, xSize, 0, ySize, 0, 0, type)
 
#define esdm_dataspace_3do(xOffset, xSize, yOffset, ySize, zOffset, zSize, type)   esdm_simple_dataspace_construct_internal(3, xOffset, xSize, yOffset, ySize, zOffset, zSize, type)
 
#define esdm_dataspace_3d(xSize, ySize, zSize, type)   esdm_simple_dataspace_construct_internal(3, 0, xSize, 0, ySize, 0, zSize, type)
 

Typedefs

typedef void *(* esdm_stream_func_t) (esdm_dataspace_t *space, void *buff, void *user_ptr, void *esdm_fill_value)
 
typedef void(* esdm_reduce_func_t) (esdm_dataspace_t *space, void *user_ptr, void *stream_func_out)
 

Enumerations

enum  esdm_format_flags { ESDM_FORMAT_DELETE = 1 , ESDM_FORMAT_CREATE = 2 , ESDM_FORMAT_IGNORE_ERRORS = 4 , ESDM_FORMAT_PURGE_RECREATE = 7 }
 

Functions

esdm_status esdm_set_procs_per_node (int procs)
 
esdm_status esdm_set_total_procs (int procs)
 
esdm_status esdm_load_config_str (const char *str)
 
esdm_status esdm_init ()
 
int esdm_is_initialized ()
 
esdm_status esdm_finalize ()
 
esdm_status esdm_write (esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace)
 
esdm_status esdm_read (esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *subspace)
 
esdm_status esdm_read_stream (esdm_dataset_t *dataset, esdm_dataspace_t *space, void *user_ptr, esdm_stream_func_t stream_func, esdm_reduce_func_t reduce_func)
 
esdm_status esdm_mkfs (int format_flags, data_accessibility_t target)
 
void esdm_loglevel (esdm_loglevel_e loglevel)
 
void esdm_log_on_exit (int on)
 
void esdm_loglevel_buffer (esdm_loglevel_e loglevel)
 
esdm_statistics_t esdm_read_stats ()
 
esdm_statistics_t esdm_write_stats ()
 
esdm_status esdm_container_create (const char *name, int allow_overwrite, esdm_container_t **out_container)
 
esdm_status esdm_container_open (const char *name, int esdm_mode_flags, esdm_container_t **out_container)
 
int esdm_container_get_mode_flags (esdm_container_t *container)
 
esdm_status esdm_container_commit (esdm_container_t *container)
 
esdm_status esdm_container_delete_attribute (esdm_container_t *c, const char *name)
 
esdm_status esdm_container_link_attribute (esdm_container_t *container, int overwrite, smd_attr_t *attr)
 
esdm_status esdm_container_get_attributes (esdm_container_t *container, smd_attr_t **out_metadata)
 
esdm_status esdm_container_delete (esdm_container_t *container)
 
esdm_status esdm_container_close (esdm_container_t *container)
 
bool esdm_container_dataset_exists (esdm_container_t *container, char const *name)
 
int esdm_container_dataset_count (esdm_container_t *container)
 
void esdm_container_set_status_dirty (esdm_container_t *container)
 
esdm_dataset_tesdm_container_dataset_from_array (esdm_container_t *container, int dset_number)
 
esdm_status esdm_dataset_update_size (esdm_dataset_t *dset, uint64_t *sizes)
 
int64_t const * esdm_dataset_get_size (esdm_dataset_t *dset)
 
int64_t const * esdm_dataset_get_actual_size (esdm_dataset_t *dset)
 
esdm_status esdm_dataset_rename (esdm_dataset_t *dataset, const char *name)
 
esdm_status esdm_dataset_set_compression_hint (esdm_dataset_t *dataset, scil_user_hints_t const *hints)
 
void esdm_dataset_set_status_dirty (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_create (esdm_container_t *container, const char *name, esdm_dataspace_t *dataspace, esdm_dataset_t **out_dataset)
 
esdm_status esdm_dataset_set_fill_value (esdm_dataset_t *dataset, void const *value)
 
esdm_status esdm_dataset_get_fill_value (esdm_dataset_t *dataset, void *value)
 
int esdm_dataset_is_fill_value_set (esdm_dataset_t *dataset)
 
char const * esdm_dataset_name (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_name_dims (esdm_dataset_t *dataset, char *const *names)
 
esdm_status esdm_dataset_rename_dim (esdm_dataset_t *dataset, char const *name, int i)
 
esdm_status esdm_dataset_get_name_dims (esdm_dataset_t *dataset, char const *const **out_names)
 
esdm_status esdm_dataset_get_dataspace (esdm_dataset_t *dset, esdm_dataspace_t **out_dataspace)
 
esdm_type_t esdm_dataset_get_type (esdm_dataset_t *d)
 
esdm_status esdm_dataset_change_name (esdm_dataset_t *dset, char const *new_name)
 
esdm_status esdm_dataset_iterator (esdm_container_t *container, esdm_dataset_iterator_t **out_iter)
 
esdm_status esdm_dataset_open (esdm_container_t *container, const char *name, int esdm_mode_flags, esdm_dataset_t **out_dataset)
 
esdm_status esdm_dataset_by_name (esdm_container_t *container, const char *name, int esdm_mode_flags, esdm_dataset_t **out_dataset)
 
esdm_status esdm_dataset_ref (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_commit (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_close (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_delete (esdm_dataset_t *dataset)
 
esdm_status esdm_dataset_delete_attribute (esdm_dataset_t *dataset, const char *name)
 
esdm_status esdm_dataset_link_attribute (esdm_dataset_t *dset, int overwrite, smd_attr_t *attr)
 
esdm_status esdm_dataset_get_attributes (esdm_dataset_t *dataset, smd_attr_t **out_metadata)
 
esdm_status esdm_dataspace_create (int64_t dims, int64_t *sizes, esdm_type_t type, esdm_dataspace_t **out_dataspace)
 
esdm_status esdm_dataspace_create_full (int64_t dims, int64_t *size, int64_t *offset, esdm_type_t type, esdm_dataspace_t **out_dataspace)
 
esdm_status esdm_dataspace_copy (esdm_dataspace_t *orig, esdm_dataspace_t **out_dataspace)
 
esdm_status esdm_dataspace_subspace (esdm_dataspace_t *dataspace, int64_t dims, int64_t *size, int64_t *offset, esdm_dataspace_t **out_dataspace)
 
esdm_status esdm_dataspace_makeContiguous (esdm_dataspace_t *dataspace, esdm_dataspace_t **out_dataspace)
 
int64_t esdm_dataspace_get_dims (esdm_dataspace_t *d)
 
int64_t const * esdm_dataspace_get_size (esdm_dataspace_t *d)
 
int64_t const * esdm_dataspace_get_offset (esdm_dataspace_t *d)
 
esdm_type_t esdm_dataspace_get_type (esdm_dataspace_t *d)
 
uint64_t esdm_dataspace_element_count (esdm_dataspace_t *dataspace)
 
int64_t esdm_dataspace_total_bytes (esdm_dataspace_t *d)
 
void esdm_dataspace_getEffectiveStride (esdm_dataspace_t *space, int64_t *out_stride)
 
int64_t esdm_dataspace_elementOffset (esdm_dataspace_t *space, int64_t *coords)
 
esdm_status esdm_dataspace_set_stride (esdm_dataspace_t *dataspace, int64_t *stride)
 
esdm_status esdm_dataspace_copyDatalayout (esdm_dataspace_t *dataspace, esdm_dataspace_t *strideSource)
 
esdm_status esdm_dataspace_copy_data (esdm_dataspace_t *sourceSpace, void *sourceData, esdm_dataspace_t *destSpace, void *destData)
 
esdm_status esdm_dataspace_fill (esdm_dataspace_t *dataspace, void *data, void *fillElement)
 
esdm_status esdm_dataspace_destroy (esdm_dataspace_t *dataspace)
 
void esdm_dataspace_serialize (const esdm_dataspace_t *dataspace, smd_string_stream_t *stream)
 
void esdm_dataspace_print (esdm_dataspace_t *dataspace)
 
esdm_status esdm_fragment_deserialize (void *serialized_fragment, esdm_fragment_t **_out_fragment)
 
esdm_status esdm_fragment_retrieve (esdm_fragment_t *fragment)
 
esdm_status esdm_fragment_load (esdm_fragment_t *fragment)
 
esdm_status esdm_fragment_unload (esdm_fragment_t *fragment)
 
esdm_status esdm_fragment_commit (esdm_fragment_t *fragment)
 
esdm_status esdm_fragment_destroy (esdm_fragment_t *fragment)
 
void esdm_fragment_print (esdm_fragment_t *fragment)
 

Detailed Description

Public API of the ESDM. Inlcudes several other public interfaces.

Macro Definition Documentation

◆ esdm_dataspace_1d

#define esdm_dataspace_1d (   size,
  type 
)    esdm_simple_dataspace_construct_internal(1, 0, size, 0, 0, 0, 0, type)

Create a new 1D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_dataspace_1do

#define esdm_dataspace_1do (   offset,
  size,
  type 
)    esdm_simple_dataspace_construct_internal(1, offset, size, 0, 0, 0, 0, type)

Create a new 1D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_dataspace_2d

#define esdm_dataspace_2d (   xSize,
  ySize,
  type 
)    esdm_simple_dataspace_construct_internal(2, 0, xSize, 0, ySize, 0, 0, type)

Create a new 2D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_dataspace_2do

#define esdm_dataspace_2do (   xOffset,
  xSize,
  yOffset,
  ySize,
  type 
)    esdm_simple_dataspace_construct_internal(2, xOffset, xSize, yOffset, ySize, 0, 0, type)

Create a new 2D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_dataspace_3d

#define esdm_dataspace_3d (   xSize,
  ySize,
  zSize,
  type 
)    esdm_simple_dataspace_construct_internal(3, 0, xSize, 0, ySize, 0, zSize, type)

Create a new 3D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_dataspace_3do

#define esdm_dataspace_3do (   xOffset,
  xSize,
  yOffset,
  ySize,
  zOffset,
  zSize,
  type 
)    esdm_simple_dataspace_construct_internal(3, xOffset, xSize, yOffset, ySize, zOffset, zSize, type)

Create a new 3D dataspace.

Construct a simple dataspace on the stack. Since the lifetime of the dataspace is automatic, it is an error to call esdm_dataspace_destroy() on it.

◆ esdm_simple_dataspace_construct_internal

#define esdm_simple_dataspace_construct_internal (   dimCount,
  xOffset,
  xSize,
  yOffset,
  ySize,
  zOffset,
  zSize,
  dataType 
)
Value:
.ptr = &(esdm_dataspace_t){ \
.type = dataType, \
.dims = dimCount, \
.size = (int64_t [3]){xSize, ySize, zSize}, \
.offset = (int64_t [3]){xOffset, yOffset, zOffset}, \
.strideBacking = (int64_t [3]){0}, \
.stride = NULL \
} \
})
Definition: esdm-datatypes.h:81
Definition: esdm-datatypes.h:92

Create a dataspace on the stack with up to three dimensions.

Do not use this directly, use esdm_dataspace_2d() and friends instead!

Typedef Documentation

◆ esdm_stream_func_t

typedef void*(* esdm_stream_func_t) (esdm_dataspace_t *space, void *buff, void *user_ptr, void *esdm_fill_value)

Identical to esdm_read except that it uses size/offset tuples instead of the subspace This function performs the operation on the data while is streamed in. The order of the data processing is not defined. Internally, threads might be used to run the function on subspaces (typically fragment level). Hence, the function must be thread safe. You can provide a user-ptr of shared space organized by you, this ptr is passed to the function The processing is as follows: First run stream_func on each data, a stream function may output an intermediate result (return value). This function may be called multiple times and concurrently. The reduce function is called once per stream output on the master thread allowing to merge the intermediate results.

Enumeration Type Documentation

◆ esdm_format_flags

Initialize backend by invoking mkfs callback for matching target

Parameters
[in]enforce_formatforce reformatting existing system (may result in data loss)
[in]targettarget descriptor
Returns
status

Function Documentation

◆ esdm_container_close()

esdm_status esdm_container_close ( esdm_container_t container)

Close a container object. If it isn't in use any more free it.

Warning: This throws an error if there are any datasets within this container that are still open. Make sure to close all datasets first.

Parameters
[in]containeran existing container object that is no longer needed

◆ esdm_container_commit()

esdm_status esdm_container_commit ( esdm_container_t container)

Make container persistent to storage. Enqueue for writing to backends.

Calling container commit may trigger subsequent commits for datasets that are part of the container.

Parameters
[in]containerpointer to an existing container which is to be committed to storage
Returns
status

◆ esdm_container_create()

esdm_status esdm_container_create ( const char *  name,
int  allow_overwrite,
esdm_container_t **  out_container 
)

Create a new container.

  • Allocate process local memory structures.
  • Register with metadata service.
Parameters
[in]namestring to identify the container, must not be empty
[out]out_containerreturns a pointer to the new container
Returns
status

◆ esdm_container_open()

esdm_status esdm_container_open ( const char *  name,
int  esdm_mode_flags,
esdm_container_t **  out_container 
)

Open an existing container.

  • Allocate process local memory structures.
  • Register with metadata service.
Parameters
[in]namestring to identify the container, must not be empty
[out]out_containerreturns a pointer to the container
Returns
status

◆ esdm_dataset_close()

esdm_status esdm_dataset_close ( esdm_dataset_t dataset)

Close a dataset object, if it isn't used anymore, it's metadata will be unloaded

This function is not thread-safe. Only a single master thread must be used to call into ESDM.

Parameters
[in]datasetan existing dataset object that is no longer needed
Returns
status

◆ esdm_dataset_commit()

esdm_status esdm_dataset_commit ( esdm_dataset_t dataset)

Make dataset persistent to storage. Schedule for writing to backends.

Parameters
[in]datasetpointer to an existing dataset which is to be committed to storage
Returns
status

◆ esdm_dataset_create()

esdm_status esdm_dataset_create ( esdm_container_t container,
const char *  name,
esdm_dataspace_t dataspace,
esdm_dataset_t **  out_dataset 
)

Create a new dataset.

  • Allocate process local memory structures.
  • Register with metadata service.
Parameters
[in]containerpointer to an existing container to which the new dataset will be linked
[in]nameidentifier for the new dataset, must not be empty
[in]dataspacepointer to an existing dataspace which defines the shape of the data that will be stored within the dataset
[out]out_datasetreturns a pointer to the new dataset
Returns
status

◆ esdm_dataset_get_dataspace()

esdm_status esdm_dataset_get_dataspace ( esdm_dataset_t dset,
esdm_dataspace_t **  out_dataspace 
)

Inquire the shape of a dataset.

Parameters
[in]dsetthe dataset to question
[out]out_dataspacea reference to the dataset's dataspace
Returns
status

The dataset remains the owner of the dataspace, the caller must not destroy it.

◆ esdm_dataset_open()

esdm_status esdm_dataset_open ( esdm_container_t container,
const char *  name,
int  esdm_mode_flags,
esdm_dataset_t **  out_dataset 
)

Open a dataset.

  • Allocate process local memory structures
  • Retrieve metadata
Parameters
[in]containerpointer to an open container that contains the dataset that is to be opened
[in]nameidentifier of the dataset within the container, must not be empty
[out]out_datasetreturns a pointer to the opened dataset
Returns
status

◆ esdm_dataspace_copy()

esdm_status esdm_dataspace_copy ( esdm_dataspace_t orig,
esdm_dataspace_t **  out_dataspace 
)

Create a copy of a dataspace.

  • Allocate process local memory structures.
    Parameters
    [in]origthe dataspace to copy
    [out]out_dataspacepointer to the new dataspace
    Returns
    status

◆ esdm_dataspace_copy_data()

esdm_status esdm_dataspace_copy_data ( esdm_dataspace_t sourceSpace,
void *  sourceData,
esdm_dataspace_t destSpace,
void *  destData 
)

Copy data from one buffer to another, possibly partially, possibly rearranging the data as prescribed by the given dataspaces.

This function copies all the data that is contained within the intersection of the two dataspaces from the source buffer to the destination buffer. The order and layout of the data elements in each buffer is described by the associated dataspace, allowing this function to be used to

  • pack non-contiguous (= strided) data into a contiguous buffer
  • unpack a contiguous buffer into a larger, non-contiguous (= strided) dataspace
  • transpose data (for example from FORTRAN order to C order and vice versa)
  • reverse the order of the data in one or more dimensions (rather esoteric use!)

In all cases, only the intersection of the two hypercubes described by the two dataspaces is copied: If the source space is larger, only the overlapping part will be read, and if the destination space is larger, only the overlapping part will be written to. If the two dataspaces don't intersect, nothing will be done.

Parameters
[in]sourceSpacedataspace that describes the layout of the sourceData buffer
[in]sourceDatapointer to the first source data element, the logical coordinate of this data element is the offset of the source dataspace
[in]destSpacedataspace that describes the layout of the destData buffer
[out]destDatapointer to the first destination data element, the logical coordinate of this data element is the offset of the destination dataspace
Returns
status

◆ esdm_dataspace_copyDatalayout()

esdm_status esdm_dataspace_copyDatalayout ( esdm_dataspace_t dataspace,
esdm_dataspace_t strideSource 
)

Copy the stride information from one dataspace to another.

This is useful when defining a subspace that is supposed to access the same buffer as the enclosing dataspace. A simple esdm_dataspace_subspace() will assume contiguous storage for the subspace, a subsequent call esdm_dataspace_copyDatalayout(subspace, bufferSpace) will provide the subspace with the correct stride values to access its possibly non-contiguous part from the same buffer. Note that it is still necessary to adjust the buffer's address by means of esdm_dataspace_elementOffset() to compute the actual address of the subspace's first element.

The strideSource must have the same dimension count as the dataspace.

Parameters
[in,out]dataspacethe dataspace to update
[in]strideSourcethe dataspace that provides the data layout information which is to be copied
Returns
ESDM_SUCCESS

◆ esdm_dataspace_create()

esdm_status esdm_dataspace_create ( int64_t  dims,
int64_t *  sizes,
esdm_type_t  type,
esdm_dataspace_t **  out_dataspace 
)

Create a new dataspace.

  • Allocate process local memory structures.
Parameters
[in]dimscount of dimensions of the new dataspace
[in]sizesarray of the sizes of the different dimensions, the length of this array is dims. Must not be NULL unless dims == 0
[in]typethe datatype for each data point
[out]out_dataspacepointer to the new dataspace
Returns
status

◆ esdm_dataspace_create_full()

esdm_status esdm_dataspace_create_full ( int64_t  dims,
int64_t *  size,
int64_t *  offset,
esdm_type_t  type,
esdm_dataspace_t **  out_dataspace 
)

Create a new dataspace.

  • Allocate process local memory structures.
Parameters
[in]dimscount of dimensions of the new dataspace
[in]sizesarray of the sizes of the different dimensions, the length of this array is dims. Must not be NULL unless dims == 0
[in]offsetarray containing the logical coordinates of the first data point in this dataspace
[in]typethe datatype for each data point
[out]out_dataspacepointer to the new dataspace
Returns
status

◆ esdm_dataspace_destroy()

esdm_status esdm_dataspace_destroy ( esdm_dataspace_t dataspace)

Destruct and free a dataspace object.

Parameters
[in]dataspacean existing dataspace object that is no longer needed
Returns
status

"_destroy" sounds too destructive, this will be renamed to esdm_dataspace_close().

◆ esdm_dataspace_element_count()

uint64_t esdm_dataspace_element_count ( esdm_dataspace_t dataspace)

Returns the number of datapoints in the dataspace.

◆ esdm_dataspace_elementOffset()

int64_t esdm_dataspace_elementOffset ( esdm_dataspace_t space,
int64_t *  coords 
)

Get the offset in bytes of the element at the given logical position. The resulting offset may be negative if a custom stride has been set that has negative component(s). Otherwise, a contiguous C order multidimensional array is assumed, producing only positive offsets.

Parameters
[in]spacethe dataspace to query
[in]coordsan array with the coordinates of the element's logical location
Returns
an offset in bytes

◆ esdm_dataspace_fill()

esdm_status esdm_dataspace_fill ( esdm_dataspace_t dataspace,
void *  data,
void *  fillElement 
)

Overwrite a buffer with a fill value.

This functions sets all elements in the given data buffer to the value given by *fillElement. The amount and offsets of the data elements to set is controlled by the dataspace argument.

Parameters
[in]dataspacedescription of the area to overwrite
[in,out]datapointer to the first element to set
[in]fillElementpointer to a single element which is used as a prototype.
Returns
status

◆ esdm_dataspace_getEffectiveStride()

void esdm_dataspace_getEffectiveStride ( esdm_dataspace_t space,
int64_t *  out_stride 
)

Get the effective stride of a dataspace.

If a stride has been set for the dataspace, that stride is copied to the out_stride array, otherwise the effective stride is calculated and returned in that same array.

Parameters
[in]spacethe dataspace to query
[out]out_stridepointer to an array of size space->dims which will be filled with the components of the stride.

As with esdm_dataspace_set_stride(), the stride is given in terms of fundamental datatype elements and needs to be multiplied with esdm_sizeof(space->type) to get the stride in bytes.

◆ esdm_dataspace_makeContiguous()

esdm_status esdm_dataspace_makeContiguous ( esdm_dataspace_t dataspace,
esdm_dataspace_t **  out_dataspace 
)

Define a dataspace that covers the same logical hypercube as the given dataspace, but which uses the standard, contiguous C array element order. The stride of the original dataspace will be ignored totally.

  • Allocates process local memory structures.
Parameters
[in]dataspacethe dataspace that is to be copied
[out]out_dataspacepointer to the new contiguous dataspace
Returns
ESDM_SUCCESS

◆ esdm_dataspace_serialize()

void esdm_dataspace_serialize ( const esdm_dataspace_t dataspace,
smd_string_stream_t *  stream 
)

Serializes dataspace description.

e.g., to store along with fragment

◆ esdm_dataspace_set_stride()

esdm_status esdm_dataspace_set_stride ( esdm_dataspace_t dataspace,
int64_t *  stride 
)

Specify a non-standard serialization order for a dataspace.

This can be used to handle FORTRAN arrays, for example, or do some crazy stuff like inverted dimensions, or to skip over holes. Use carefully, or don't use at all. You have been warned.

Parameters
[in,out]dataspacethe dataspace that is to be modified
[in]dimsnumber of entries in the stride argument, must match the dimension count of the dataspace
[in]stridearray with dims entries, each entry gives the number of elements to skip over when increasing the respective coordinate by one.
Returns
status

Examples:

A C array int array[7][11] does not need a stride, the stride is implicitly assumed to be (11, 1).

To handle a FORTRAN array INTEGER :: array(7, 11), use the following call: esdm_dataspace_set_stride(dataspace, 2, (int64_t[2]){1, 7});

To use only a 3x5 part of an existing C array int array[7][11], starting at (1, 2), use these calls: esdm_dataspace_t* subspace; esdm_dataspace_subspace(parent, 2, (int64_t[2]){3, 5}, (int64_t[2]){1, 2}, &subspace); esdm_dataspace_set_stride(subspace, (int64_t[2]){11, 1}); After this, the 2D coordinates will be mapped to the buffer offsets like this: (1,2)=0, (1,3)=1, (1,4)=2, (1,5)=3, (1,6)=4, (2,2)=11, (2,3)=12, (2,4)=13, (2,5)=14, (2,6)=15, (3,2)=22, (3,3)=23, (3,4)=24, (3,5)=25, (3,6)=26,

◆ esdm_dataspace_subspace()

esdm_status esdm_dataspace_subspace ( esdm_dataspace_t dataspace,
int64_t  dims,
int64_t *  size,
int64_t *  offset,
esdm_dataspace_t **  out_dataspace 
)

Define a dataspace that is a subset of the given dataspace.

  • Allocates process local memory structures.
Parameters
[in]dataspacean existing dataspace that encloses the subspace
[in]dimslength of the size and offset arguments, must be equal to the number of dimensions of the given dataspace
[in]sizesize of the hypercube of data within the subspace
[in]offsetlocation of the first data point within the subspace
[out]out_dataspacepointer to the new sub-dataspace
Returns
ESDM_SUCCESS on success, ESDM_INVALID_ARGUMENT_ERROR if the provided dims, size, or offset arguments do not agree with the provided dataspace

TODO: remove dims parameter for good

◆ esdm_dataspace_total_bytes()

int64_t esdm_dataspace_total_bytes ( esdm_dataspace_t d)

Returns the number of bytes covered by the dataspace.

◆ esdm_finalize()

esdm_status esdm_finalize ( )

Display status information for objects stored in ESDM.

Returns
status

◆ esdm_fragment_commit()

esdm_status esdm_fragment_commit ( esdm_fragment_t fragment)

Make fragment persistent to storage. Schedule for writing to backends.

Parameters
[in]fragmentpointer to an existing fragment which is to be committed to storage
Returns
status

◆ esdm_fragment_deserialize()

esdm_status esdm_fragment_deserialize ( void *  serialized_fragment,
esdm_fragment_t **  _out_fragment 
)

Reinstantiate fragment from serialization.

◆ esdm_fragment_destroy()

esdm_status esdm_fragment_destroy ( esdm_fragment_t fragment)

Destruct and free a fragment object.

Parameters
[in]fragmentan existing fragment object that is no longer needed
Returns
status

"_destroy" sounds too destructive, this will be renamed to esdm_fragment_close().

◆ esdm_fragment_load()

esdm_status esdm_fragment_load ( esdm_fragment_t fragment)

Like esdm_fragment_retrieve(), but more permissive: Does not throw an ESDM_DIRTY_DATA_ERROR, simply ensures that the fragments data is available in memory.

◆ esdm_fragment_print()

void esdm_fragment_print ( esdm_fragment_t fragment)

Serializes fragment for storage.

◆ esdm_fragment_retrieve()

esdm_status esdm_fragment_retrieve ( esdm_fragment_t fragment)

Fetch data from disk if possible. Loads fragments that are not loaded, noops on those that are loaded and clean, and errors out on those that are dirty or deleted.

XXX: This should probably be turned into an internal interface.

◆ esdm_fragment_unload()

esdm_status esdm_fragment_unload ( esdm_fragment_t fragment)

Ensure that the fragment has no data in memory.

If the fragment is dirty, it is committed, turning it into a persistent fragment. If the fragment is persistent, its buffer is released, turning it into an unloaded fragment. If the fragment is deleted or not loaded, nothing is done successfully.

◆ esdm_init()

esdm_status esdm_init ( )

Initialize ESDM:

  • allocate data structures for ESDM
  • allocate memory for node local caches
  • initialize submodules
  • initialize threadpool
Returns
status

◆ esdm_load_config_str()

esdm_status esdm_load_config_str ( const char *  str)

Set the configuration to use. Must not be called after init(), and must not be called twice.

Parameters
[in]stra string containing configuration data in JSON format
Returns
status

◆ esdm_read()

esdm_status esdm_read ( esdm_dataset_t dataset,
void *  buf,
esdm_dataspace_t subspace 
)

Identical to esdm_write except that it uses size/offset tuples instead of the subspace Reads a data fragment described by desc to the dataset dset.

Parameters
[in]datasetTODO, currently a stub, we assume it has been identified/created before.... , json description?
[out]bufa contiguous memory region that shall be filled with the data from permanent storage
[in]subspacean existing dataspace that describes the shape and location of the hypercube that is to be read
Returns
status

◆ esdm_read_stats()

esdm_statistics_t esdm_read_stats ( )

Get some statistics about the reads that have been performed.

◆ esdm_set_procs_per_node()

esdm_status esdm_set_procs_per_node ( int  procs)

Set the number of processes to use per node. Must not be called after init().

Parameters
[in]procsthe number of processes to use per node
Returns
status

◆ esdm_set_total_procs()

esdm_status esdm_set_total_procs ( int  procs)

Set the total number of processes to use. Must not be called after init().

Parameters
[in]procsthe number of processes to use
Returns
status

◆ esdm_write()

esdm_status esdm_write ( esdm_dataset_t dataset,
void *  buf,
esdm_dataspace_t subspace 
)

Write data with a given size and offset.

Parameters
[in]datasetTODO, currently a stub, we assume it has been identified/created before...., json description?
[in]bufthe pointer to a contiguous memory region that shall be written to permanent storage
[in]subspacean existing dataspace that describes the shape and location of the hypercube that is to be written
Returns
status

◆ esdm_write_stats()

esdm_statistics_t esdm_write_stats ( )

Get some statistics about the writes that have been performed.