ESDM
Middleware for Earth System Data
Macros | Typedefs | Functions | Variables
esdm-internal.h File Reference

Internal ESDM functionality, not to be used by backends and plugins. More...

#include <jansson.h>
#include <glib.h>
#include <inttypes.h>
#include <esdm-datatypes-internal.h>
#include <esdm-stream.h>
#include <esdm-debug.h>
#include <esdm.h>
Include dependency graph for esdm-internal.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define min(x, y)   (x < y ? x : y)
 
#define max(x, y)   (x > y ? x : y)
 
#define esdmI_hypercubeList_nonredundantSubsets(list, inout_setCount, out_subsets)
 

Typedefs

typedef struct timespec timer
 
typedef void *(* ea_datatype_converter) (void *dest, const void *source, size_t sourceBytes)
 

Functions

esdm_config_tesdm_config_init ()
 
esdm_config_tesdm_config_init_from_str (const char *str)
 
esdm_config_tesdmI_getConfig ()
 
esdm_status esdm_config_finalize (esdm_instance_t *esdm)
 
char * esdm_config_gather ()
 
esdm_config_backends_tesdm_config_get_backends (esdm_instance_t *esdm)
 
esdm_config_backend_tesdm_config_get_metadata_coordinator (esdm_instance_t *esdm)
 
esdm_instance_tesdmI_esdm ()
 
esdm_modules_tesdm_get_modules ()
 
esdm_modules_tesdm_modules_init (esdm_instance_t *esdm)
 
esdm_backend_tesdm_modules_fastestBackend (esdm_modules_t *modules)
 
esdm_backend_tesdm_modules_randomWeightedBackend (esdm_modules_t *modules)
 
esdm_status esdm_modules_finalize ()
 
esdm_backend_t ** esdm_modules_makeBackendRecommendation (esdm_modules_t *modules, esdm_dataspace_t *space, int64_t *out_moduleCount, int64_t *out_maxFragmentSize)
 
esdm_status esdm_modules_register ()
 
esdm_status esdm_modules_get_by_type (esdm_module_type_t type, esdm_module_type_array_t **array)
 
esdm_scheduler_tesdm_scheduler_init (esdm_instance_t *esdm)
 
esdm_status esdm_scheduler_finalize (esdm_instance_t *esdm)
 
esdm_status esdm_scheduler_status_init (io_request_status_t *status)
 
esdm_status esdm_scheduler_status_finalize (io_request_status_t *status)
 
esdm_status esdm_scheduler_read_blocking (esdm_instance_t *esdm, esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *memspace, esdmI_hypercubeSet_t **out_fillRegion, bool allowWriteback, bool requestIsInternal)
 
esdm_status esdm_scheduler_write_blocking (esdm_instance_t *esdm, esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *memspace, bool requestIsInternal)
 
esdm_status esdmI_scheduler_writeFragmentBlocking (esdm_instance_t *esdm, esdm_fragment_t *fragment, bool requestIsInternal)
 
void esdmI_scheduler_writeFragmentNonblocking (esdm_instance_t *esdm, esdm_fragment_t *fragment, bool requestIsInternal, io_request_status_t *status)
 
esdm_status esdmI_scheduler_readSingleFragmentBlocking (esdm_instance_t *esdm, esdm_dataset_t *dataset, void *buffer, esdm_dataspace_t *memspace, esdm_fragment_t *fragment)
 
esdm_status esdm_scheduler_enqueue (esdm_instance_t *esdm, io_request_status_t *status, io_operation_t type, esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *memspace)
 
esdm_status esdm_scheduler_wait (io_request_status_t *status)
 
esdm_status esdmI_readWithFillRegion (esdm_dataset_t *dataset, void *buf, esdm_dataspace_t *memspace, esdmI_hypercubeSet_t **out_fillRegion)
 
void fetch_performance_from_backend (gpointer key, gpointer value, gpointer user_data)
 
esdm_performance_tesdm_performance_init (esdm_instance_t *esdm)
 
esdm_status esdm_performance_recommendation (esdm_instance_t *esdm, esdm_fragment_t *in, esdm_fragment_t *out)
 
esdm_status esdm_performance_finalize ()
 
esdm_readTimes_t esdmI_performance_read ()
 
esdm_readTimes_t esdmI_performance_read_add (const esdm_readTimes_t *a, const esdm_readTimes_t *b)
 
esdm_readTimes_t esdmI_performance_read_sub (const esdm_readTimes_t *minuend, const esdm_readTimes_t *subtrahend)
 
void esdmI_performance_read_print (FILE *stream, const char *linePrefix, const char *indentation, const esdm_readTimes_t *start, const esdm_readTimes_t *end)
 
esdm_writeTimes_t esdmI_performance_write ()
 
esdm_writeTimes_t esdmI_performance_write_add (const esdm_writeTimes_t *a, const esdm_writeTimes_t *b)
 
esdm_writeTimes_t esdmI_performance_write_sub (const esdm_writeTimes_t *minuend, const esdm_writeTimes_t *subtrahend)
 
void esdmI_performance_write_print (FILE *stream, const char *linePrefix, const char *indentation, const esdm_writeTimes_t *start, const esdm_writeTimes_t *end)
 
esdm_copyTimes_t esdmI_performance_copy ()
 
esdm_copyTimes_t esdmI_performance_copy_add (const esdm_copyTimes_t *a, const esdm_copyTimes_t *b)
 
esdm_copyTimes_t esdmI_performance_copy_sub (const esdm_copyTimes_t *minuend, const esdm_copyTimes_t *subtrahend)
 
void esdmI_performance_copy_print (FILE *stream, const char *linePrefix, const char *indentation, const esdm_copyTimes_t *start, const esdm_copyTimes_t *end)
 
esdm_backendTimes_t esdmI_performance_backend ()
 
esdm_backendTimes_t esdmI_performance_backend_add (const esdm_backendTimes_t *a, const esdm_backendTimes_t *b)
 
esdm_backendTimes_t esdmI_performance_backend_sub (const esdm_backendTimes_t *minuend, const esdm_backendTimes_t *subtrahend)
 
void esdmI_performance_backend_print (FILE *stream, const char *linePrefix, const char *indentation, const esdm_backendTimes_t *start, const esdm_backendTimes_t *end)
 
esdm_fragmentsTimes_t esdmI_performance_fragments ()
 
esdm_fragmentsTimes_t esdmI_performance_fragments_add (const esdm_fragmentsTimes_t *a, const esdm_fragmentsTimes_t *b)
 
esdm_fragmentsTimes_t esdmI_performance_fragments_sub (const esdm_fragmentsTimes_t *minuend, const esdm_fragmentsTimes_t *subtrahend)
 
void esdmI_performance_fragments_print (FILE *stream, const char *linePrefix, const char *indentation, const esdm_fragmentsTimes_t *start, const esdm_fragmentsTimes_t *end)
 
int esdmI_backend_finalize (esdm_backend_t *b)
 
int esdmI_backend_performance_estimate (esdm_backend_t *b, esdm_fragment_t *fragment, float *out_time)
 
float esdmI_backend_estimate_throughput (esdm_backend_t *b)
 
int esdmI_backend_fragment_create (esdm_backend_t *b, esdm_fragment_t *fragment)
 
int esdmI_backend_fragment_retrieve (esdm_backend_t *b, esdm_fragment_t *fragment)
 
int esdmI_backend_fragment_update (esdm_backend_t *b, esdm_fragment_t *fragment)
 
int esdmI_backend_fragment_delete (esdm_backend_t *b, esdm_fragment_t *fragment)
 
int esdmI_backend_fragment_metadata_create (esdm_backend_t *b, esdm_fragment_t *fragment, smd_string_stream_t *stream)
 
void * esdmI_backend_fragment_metadata_load (esdm_backend_t *b, esdm_fragment_t *fragment, json_t *metadata)
 
int esdmI_backend_fragment_metadata_free (esdm_backend_t *b, void *options)
 
int esdmI_backend_mkfs (esdm_backend_t *b, int format_flags)
 
int esdmI_backend_fsck (esdm_backend_t *b)
 
int esdmI_backend_fragment_write_stream_blocksize (esdm_backend_t *b, estream_write_t *state, void *cur_buf, size_t cur_offset, uint32_t cur_size)
 
double esdmI_backendOutputTime ()
 
double esdmI_backendInputTime ()
 
void esdmI_resetBackendIoTimes ()
 
void esdmI_container_init (char const *name, esdm_container_t **out_container)
 
esdm_status esdm_container_open_md_load (esdm_container_t *c, char **out_md, int *out_size)
 
esdm_status esdm_container_open_md_parse (esdm_container_t *c, char *md, int size)
 
void esdmI_container_register_dataset (esdm_container_t *c, esdm_dataset_t *dset)
 
esdm_status esdmI_container_destroy (esdm_container_t *container)
 
void esdm_dataset_init (esdm_container_t *container, const char *name, esdm_dataspace_t *dataspace, esdm_dataset_t **out_dataset)
 
esdm_status esdm_dataset_open_md_load (esdm_dataset_t *dset, char **out_md, int *out_size)
 
esdm_status esdm_dataset_open_md_parse (esdm_dataset_t *d, char *md, int size)
 
esdm_status esdmI_dataset_fragmentsCoveringRegion (esdm_dataset_t *dataset, esdmI_hypercube_t *region, int64_t *out_count, esdm_fragment_t ***out_fragments, esdmI_hypercubeSet_t **out_uncovered, bool *out_fullyCovered)
 
esdm_fragment_tesdmI_dataset_createFragment (esdm_dataset_t *dataset, esdm_dataspace_t *memspace, void *buf, bool *out_newFragment)
 
esdm_fragment_tesdmI_dataset_lookupFragmentForShape (esdm_dataset_t *dataset, esdm_dataspace_t *shape)
 
esdm_status esdmI_dataset_destroy (esdm_dataset_t *dataset)
 
esdm_status esdmI_dataspace_createFromHypercube (esdmI_hypercube_t *extends, esdm_type_t type, esdm_dataspace_t **out_space)
 
esdm_status esdmI_dataspace_createFromJson (json_t *json, esdm_dataset_t *dataset, esdm_dataspace_t **out_dataspace)
 
esdm_status esdmI_dataspace_getExtends (esdm_dataspace_t *space, esdmI_hypercube_t **out_extends)
 
esdm_status esdmI_dataspace_setExtends (esdm_dataspace_t *space, esdmI_hypercube_t *extends)
 
void esdmI_fragments_construct (esdm_fragments_t *me)
 
esdm_status esdmI_fragments_add (esdm_fragments_t *me, esdm_fragment_t *fragment) __attribute__((warn_unused_result))
 
esdm_fragment_tesdmI_fragments_lookupForShape (esdm_fragments_t *me, esdmI_hypercube_t *shape)
 
esdm_status esdmI_fragments_deleteAll (esdm_fragments_t *me)
 
esdm_fragment_t ** esdmI_fragments_makeSetCoveringRegion (esdm_fragments_t *me, esdmI_hypercube_t *region, int64_t *out_fragmentCount)
 
void esdmI_fragments_metadata_create (esdm_fragments_t *me, smd_string_stream_t *s)
 
void esdmI_fragments_purge (esdm_fragments_t *me)
 
esdm_status esdmI_fragments_destruct (esdm_fragments_t *me)
 
void esdm_fragment_metadata_create (esdm_fragment_t *f, smd_string_stream_t *stream)
 
esdm_status esdmI_create_fragment_from_metadata (esdm_dataset_t *dset, json_t *json, esdm_fragment_t **out)
 
esdm_status esdmI_fragment_create (esdm_dataset_t *dataset, esdm_dataspace_t *memspace, void *buf, esdm_fragment_t **out_fragment)
 
esdm_layout_tesdm_layout_init (esdm_instance_t *esdm)
 
esdm_status esdm_layout_finalize (esdm_instance_t *esdm)
 
esdm_fragment_tesdm_layout_reconstruction (esdm_dataset_t *dataset, esdm_dataspace_t *subspace)
 
esdm_status esdm_layout_recommendation (esdm_instance_t *esdm, esdm_fragment_t *in, esdm_fragment_t *out)
 
esdm_backend_tesdmI_init_backend (char const *name, esdm_config_backend_t *config)
 
esdm_status esdm_backend_t_estimate_performance (esdm_backend_t *backend, int fragment)
 
void print_stat (struct stat sb)
 
int mkdir_recursive (const char *path)
 
int posix_recursive_remove (const char *path)
 
int ea_read_file (char *filepath, char **buf)
 
int ea_read_check (int fd, char *buf, size_t len)
 
int ea_write_check (int fd, char *buf, size_t len)
 
json_t * load_json (const char *str)
 
esdm_backend_tesdmI_get_backend (char const *plugin_id)
 
void ea_generate_id (char *str, size_t length)
 
char * ea_make_id (size_t length)
 
void * ea_checked_malloc (size_t size) __attribute__((alloc_size(1)
 
void * ea_checked_calloc (size_t nmemb, size_t size) __attribute__((alloc_size(1
 
void * ea_checked_realloc (void *ptr, size_t size) __attribute__((alloc_size(2)
 
char * ea_checked_strdup (const char *string)
 
char * ea_checked_strndup (const char *string, size_t n)
 
void * ea_memdup (void *data, size_t size)
 
int ea_compute_hash_str (const char *str)
 
bool ea_is_valid_dataset_name (const char *str)
 
void ea_start_timer (timer *t1)
 
double ea_stop_timer (timer t1)
 
double ea_timer_subtract (timer number, timer subtract)
 
ea_datatype_converter ea_converter_for_types (esdm_type_t destType, esdm_type_t sourceType)
 
esdmI_range_t esdmI_range_intersection (esdmI_range_t a, esdmI_range_t b)
 
void esdmI_range_print (esdmI_range_t range, FILE *stream)
 
esdmI_hypercube_tesdmI_hypercube_makeDefault (int64_t dimensions)
 
esdmI_hypercube_tesdmI_hypercube_make (int64_t dimensions, int64_t *offset, int64_t *size)
 
esdmI_hypercube_tesdmI_hypercube_makeCopy (esdmI_hypercube_t *original)
 
esdmI_hypercube_tesdmI_hypercube_makeIntersection (esdmI_hypercube_t *a, esdmI_hypercube_t *b)
 
bool esdmI_hypercube_isEmpty (esdmI_hypercube_t *cube)
 
uint32_t esdmI_hypercube_hash (const esdmI_hypercube_t *me)
 
uint32_t esdmI_hypercube_hashOffsetSize (int64_t dimCount, const int64_t *offset, const int64_t *size)
 
bool esdmI_hypercube_doesIntersect (esdmI_hypercube_t *a, esdmI_hypercube_t *b)
 
bool esdmI_hypercube_touches (esdmI_hypercube_t *a, esdmI_hypercube_t *b)
 
int64_t esdmI_hypercube_overlap (esdmI_hypercube_t *a, esdmI_hypercube_t *b)
 
bool esdmI_hypercube_equal (const esdmI_hypercube_t *a, const esdmI_hypercube_t *b)
 
double esdmI_hypercube_shapeSimilarity (esdmI_hypercube_t *a, esdmI_hypercube_t *b)
 
void esdmI_hypercube_getOffsetAndSize (const esdmI_hypercube_t *cube, int64_t *out_offset, int64_t *out_size)
 
int64_t esdmI_hypercube_size (esdmI_hypercube_t *cube)
 
void esdmI_hypercube_print (esdmI_hypercube_t *cube, FILE *stream)
 
void esdmI_hypercube_destroy (esdmI_hypercube_t *cube)
 
bool esdmI_hypercubeList_doesIntersect (esdmI_hypercubeList_t *list, esdmI_hypercube_t *cube)
 
bool esdmI_hypercubeList_doesCoverFully (esdmI_hypercubeList_t *list, esdmI_hypercube_t *cube)
 
void esdmI_hypercubeList_nonredundantSubsets_internal (esdmI_hypercubeList_t *list, int64_t count, int64_t *inout_setCount, uint8_t(*out_subsets)[count])
 
void esdmI_hypercubeList_print (esdmI_hypercubeList_t *list, FILE *stream)
 
esdmI_hypercubeSet_tesdmI_hypercubeSet_make ()
 
void esdmI_hypercubeSet_construct (esdmI_hypercubeSet_t *me)
 
esdmI_hypercubeList_tesdmI_hypercubeSet_list (esdmI_hypercubeSet_t *me)
 
bool esdmI_hypercubeSet_isEmpty (esdmI_hypercubeSet_t *me)
 
int64_t esdmI_hypercubeSet_count (esdmI_hypercubeSet_t *me)
 
void esdmI_hypercubeSet_add (esdmI_hypercubeSet_t *me, esdmI_hypercube_t *cube)
 
void esdmI_hypercubeSet_subtract (esdmI_hypercubeSet_t *me, esdmI_hypercube_t *cube)
 
void esdmI_hypercubeSet_destruct (esdmI_hypercubeSet_t *me)
 
void esdmI_hypercubeSet_destroy (esdmI_hypercubeSet_t *me)
 
esdmI_hypercubeNeighbourManager_tesdmI_hypercubeNeighbourManager_make (int64_t dimensions)
 
esdmI_hypercubeList_tesdmI_hypercubeNeighbourManager_list (esdmI_hypercubeNeighbourManager_t *me)
 
void esdmI_hypercubeNeighbourManager_pushBack (esdmI_hypercubeNeighbourManager_t *me, esdmI_hypercube_t *cube)
 
int64_t * esdmI_hypercubeNeighbourManager_getNeighbours (esdmI_hypercubeNeighbourManager_t *me, int64_t index, int64_t *out_neighbourCount)
 
void esdmI_hypercubeNeighbourManager_destroy (esdmI_hypercubeNeighbourManager_t *me)
 

Variables

void malloc
 

Detailed Description

Internal ESDM functionality, not to be used by backends and plugins.

Macro Definition Documentation

◆ esdmI_hypercubeList_nonredundantSubsets

#define esdmI_hypercubeList_nonredundantSubsets (   list,
  inout_setCount,
  out_subsets 
)
Value:
do {\
esdmI_hypercubeList_t* l = list;\
esdmI_hypercubeList_nonredundantSubsets_internal(l, l->count, inout_setCount, out_subsets);\
} while(false)

Function Documentation

◆ ea_checked_calloc()

void* ea_checked_calloc ( size_t  nmemb,
size_t  size 
)

Wrapper for calloc() that checks the result for a null-pointer.

◆ ea_checked_malloc()

void* ea_checked_malloc ( size_t  size)

Wrapper for malloc() that checks the result for a null-pointer.

◆ ea_checked_realloc()

void* ea_checked_realloc ( void *  ptr,
size_t  size 
)

Wrapper for realloc() that checks the result for a null-pointer.

◆ ea_checked_strdup()

char* ea_checked_strdup ( const char *  string)

Wrapper for strdup() that checks the result for a null-pointer.

◆ ea_checked_strndup()

char* ea_checked_strndup ( const char *  string,
size_t  n 
)

Wrapper for strndup() that checks the result for a null-pointer.

◆ ea_converter_for_types()

ea_datatype_converter ea_converter_for_types ( esdm_type_t  destType,
esdm_type_t  sourceType 
)

Return a memcpy()-like function that converts an array of sourceType into an array of destType. The individual values are converted as with the respective C cast, preserving values where possible, but possibly also wrapping around or losing precision if the original values cannot be represented exactly.

Works only for some selected datatypes (noop conversions (memcpy()), int8_t through uint64_t, as well as float and double), so be sure to check the return value against NULL and handle the error condition appropriately.

Be aware that some conversions involve UB. Especially out-of-range float/double conversion to integers can yield surprising results.

◆ ea_memdup()

void* ea_memdup ( void *  data,
size_t  size 
)

Create a copy of an arbitrary memory buffer. This is essentially a strdup() for non-string data.

Parameters
[in]datapointer to the bytes to copy
[in]sizecount of bytes to copy
Returns
freshly malloc'ed buffer containing a copy of the bytes, must be free'd by the caller

◆ ea_read_check()

int ea_read_check ( int  fd,
char *  buf,
size_t  len 
)

Read while ensuring and retrying until len is read or error occured.

◆ ea_write_check()

int ea_write_check ( int  fd,
char *  buf,
size_t  len 
)

Write while ensuring and retrying until len is written or error occured.

◆ esdm_config_gather()

char* esdm_config_gather ( )

Gathers ESDM configuration settings from multiple locations to build one configuration string.

TODO: /etc/esdm/esdm.conf ~/.config/esdm/esdm.conf ~/.esdm.conf ./esdm.conf environment variable arguments

◆ esdm_config_get_backends()

esdm_config_backends_t* esdm_config_get_backends ( esdm_instance_t esdm)

Fetches backends

◆ esdm_config_init()

esdm_config_t* esdm_config_init ( )

Initializes the site configuration module.

Parameters
[in]esdmPointer to esdm instance.
Returns
Pointer to newly created configuration instance.

◆ esdm_layout_finalize()

esdm_status esdm_layout_finalize ( esdm_instance_t esdm)

Shutdown ESDM:

  • finalize submodules
  • free data structures
Returns
Status

◆ esdm_layout_init()

esdm_layout_t* esdm_layout_init ( esdm_instance_t esdm)

Initializes the init component by allocating and populating the esdm_layout structure.

Returns
Status

◆ esdm_layout_recommendation()

esdm_status esdm_layout_recommendation ( esdm_instance_t esdm,
esdm_fragment_t in,
esdm_fragment_t out 
)

Splits pending requests into one or more requests based on performance estimates obtained from available backends.

Returns
Status

◆ esdm_layout_reconstruction()

esdm_fragment_t* esdm_layout_reconstruction ( esdm_dataset_t dataset,
esdm_dataspace_t subspace 
)

The layout reconstructor finds a reconstruction for subspace of a dataset.

The reconstruction should take performance considerations into account.

Returns
Status

◆ esdm_modules_fastestBackend()

esdm_backend_t* esdm_modules_fastestBackend ( esdm_modules_t modules)

Get a pointer to the backend with the highest estimated throughput.

◆ esdm_modules_makeBackendRecommendation()

esdm_backend_t** esdm_modules_makeBackendRecommendation ( esdm_modules_t modules,
esdm_dataspace_t space,
int64_t *  out_moduleCount,
int64_t *  out_maxFragmentSize 
)

Make a recommendation on which data backend to use to store data for the given dataspace.

Parameters
[in]modulesusually esdm->modules
[in]spacethe dataspace for which the recommendation is to be made
[out]out_moduleCountreturns the number of recommended backends
[out]out_maxFragmentSizea max fragment size that is suitable for use with all the recommended backends (optional, may be NULL)
Returns
a freshly allocated array of *out_moduleCount backend pointers, must be free'd by the caller

◆ esdm_performance_recommendation()

esdm_status esdm_performance_recommendation ( esdm_instance_t esdm,
esdm_fragment_t in,
esdm_fragment_t out 
)

Splits pending requests into one or more requests based on performance estimates obtained from available backends.

◆ esdm_scheduler_read_blocking()

esdm_status esdm_scheduler_read_blocking ( esdm_instance_t esdm,
esdm_dataset_t dataset,
void *  buf,
esdm_dataspace_t memspace,
esdmI_hypercubeSet_t **  out_fillRegion,
bool  allowWriteback,
bool  requestIsInternal 
)

Calls to reads have to be completed before they can return to the application and are therefor blocking.

Note: write is also blocking right now.

Parameters
[out]out_fillRegionReturns a pointer to a hypercube set that covers the region for which no data was found. It's the callers' responsibility to either pass NULL or to destroy the hypercube set themselves.

◆ esdmI_container_destroy()

esdm_status esdmI_container_destroy ( esdm_container_t container)

Destruct and free a container object.

Parameters
[in]containeran existing container object that is no longer needed

"_destroy" sounds too destructive, this will be renamed to esdm_container_close().

◆ esdmI_dataset_destroy()

esdm_status esdmI_dataset_destroy ( esdm_dataset_t dataset)

Destruct and free a dataset object.

Parameters
[in]datasetan existing dataset object that is no longer needed
Returns
status

"_destroy" sounds too destructive, this will be renamed to esdm_dataset_close().

◆ esdmI_dataspace_createFromHypercube()

esdm_status esdmI_dataspace_createFromHypercube ( esdmI_hypercube_t extends,
esdm_type_t  type,
esdm_dataspace_t **  out_space 
)

Create a dataspace. Takes the shape in the form of an esdmI_hypercube_t.

Similar to esdm_dataspace_create(), but takes an esdmI_hypercube_t* instead of a pair of offset and size arrays.

Parameters
[in]extendsthe logical shape of the dataspace that is to be created
[out]out_spacereturns a new dataspace object that needs to be destructed by the caller

◆ esdmI_dataspace_createFromJson()

esdm_status esdmI_dataspace_createFromJson ( json_t *  json,
esdm_dataset_t dataset,
esdm_dataspace_t **  out_dataspace 
)

Create a dataspace object from its JSON description (which was produced via a call to esdm_dataspace_serialize()).

Parameters
[in]jsonthe JSON data describing the dataspace
[in]datasetthe dataset that is to be linked to the dataspace (this provides the datatype)
[out]out_dataspacewill point to a valid dataspace object after a successful return
Returns
ESDM_SUCCESS on success, ESDM_INVALID_DATA_ERROR in case of any inconsistencies in the JSON data

◆ esdmI_dataspace_getExtends()

esdm_status esdmI_dataspace_getExtends ( esdm_dataspace_t space,
esdmI_hypercube_t **  out_extends 
)

Get the logical extends covered by a dataspace in the form of an esdmI_hypercube_t.

Parameters
[in]spacethe dataspace to query
[out]out_extendsreturns a pointer to a hypercube with the extends of the dataspace, the caller is responsible to destroy the returned pointer
Returns
ESDM_SUCCESS

◆ esdmI_dataspace_setExtends()

esdm_status esdmI_dataspace_setExtends ( esdm_dataspace_t space,
esdmI_hypercube_t extends 
)

Set the logical extends covered by a dataspace in the form of an esdmI_hypercube_t.

Parameters
[in]spacethe dataspace to query
[in]extendsa hypercube with the extends of the dataspace
Returns
ESDM_SUCCESS

◆ esdmI_fragment_create()

esdm_status esdmI_fragment_create ( esdm_dataset_t d,
esdm_dataspace_t sspace,
void *  buf,
esdm_fragment_t **  out_fragment 
)

Create a new fragment. If a non-NULL buf argument is supplied, the fragment only references the data, it does not take possession of the pointer. It is the user's responsibility to ensure that the data remains alive as long as the fragment is loaded.

  • Allocate process local memory structures.
  • Does not register the fragment with the dataset. How the fragment will be owned is left to the discretion of the caller.

A fragment is part of a dataset.

Returns
Pointer to new fragment.

TODO: there should be a mode to auto-commit on creation?

How does this integrate with the scheduler? On auto-commit this merely beeing pushed to sched for dispatch?

◆ esdmI_hypercube_getOffsetAndSize()

void esdmI_hypercube_getOffsetAndSize ( const esdmI_hypercube_t cube,
int64_t *  out_offset,
int64_t *  out_size 
)

Return the shape of the hypercube as an offset and a size vector.

Parameters
[in]cubethe hypercube to get the shape of
[out]out_offsetarray of size cube->dims that will be filled with the offset vector components
[out]out_sizearray of size cube->dims that will be filled with the size vector components

◆ esdmI_hypercubeList_nonredundantSubsets_internal()

void esdmI_hypercubeList_nonredundantSubsets_internal ( esdmI_hypercubeList_t list,
int64_t  count,
int64_t *  inout_setCount,
uint8_t(*)  out_subsets[count] 
)

This function returns a number of minimal subsets of the cubes contained within the hypercubeList. The selection of the subsets is probabilistic as any complete algorithm I could think of would have had exponential complexity. The probabilistic solution allows the caller to specify how much effort should be spent in finding minimal subsets. With the probabilistic solution, the function creates as many subsets as indicated by *inout_setCount, and returns the number of actually found subsets in the same memory location. The returned value may be lower than the given value because a subset may be found several times.

The probabilistic algorithm is written in such a way that it will find small minimal subsets more easily than subsets that contain more cubes.

The complexity of the algorithm is O(*inout_setCount * list->count^2).

Parameters
[in]listthe hypercube list from which to select subsets
[in,out]inout_setCounton input the requested amount of subsets that are to be generated, on output the actual number of distinct subsets retured in out_subsets
[out]out_subsetsa 2D array of dimensions out_subsets[*inout_setCount][list->count] that will be used to store the flags which cube is selected for which subset; if out_subsets[i][j] is true then the cube with index j is selected for the subset i

◆ esdmI_readWithFillRegion()

esdm_status esdmI_readWithFillRegion ( esdm_dataset_t dataset,
void *  buf,
esdm_dataspace_t memspace,
esdmI_hypercubeSet_t **  out_fillRegion 
)

As esdm_read(), but also return the region that was filled with the fill value as a hypercube set. If no fill value is set and any region without data is detected, this call will still return an error.

Parameters
[in]datasetTODO, currently a stub, we assume it has been identified/created before.... , json description?
[out]bufa contiguous memory region that shall be filled with the data from permanent storage
[in]memspacea dataspace that describes the location, size, and memory layout of the part of the data that is to be read
[out]out_fillRegionreturns a new esdmI_hypercubeSet_t* that covers the region for which no data was found.
Returns
status

◆ fetch_performance_from_backend()

void fetch_performance_from_backend ( gpointer  key,
gpointer  value,
gpointer  user_data 
)

Queries backend for performance estimate for the given fragment.

◆ print_stat()

void print_stat ( struct stat  sb)

Print a detailed summary for the stat system call.