C API

The GSD C API consists of a single header and source file. Developers can drop the implementation into any package that needs it.

struct gsd_byte_buffer
#include <gsd.h>

Byte buffer.

Used to buffer of small data chunks held for a buffered write at the end of a frame. Also used to hold the names.

Public Members

char *data

Data.

size_t size

Number of bytes in the buffer.

size_t reserved

Number of bytes available in the buffer.

struct gsd_handle
#include <gsd.h>

File handle.

A handle to an open GSD file.

This handle is obtained when opening a GSD file and is passed into every method that operates on the file.

Warning

All members are read-only to the caller.

Public Members

int fd

File descriptor.

struct gsd_header header

The file header.

struct gsd_index_buffer file_index

Mapped data chunk index.

struct gsd_index_buffer frame_index

Index entries to append to the current frame.

struct gsd_index_buffer buffer_index

Buffered index entries to append to the current frame.

struct gsd_byte_buffer write_buffer

Buffered write data.

struct gsd_name_buffer file_names

List of names stored in the file.

struct gsd_name_buffer frame_names

List of names added in the current frame.

uint64_t cur_frame

The index of the last frame in the file.

int64_t file_size

Size of the file (in bytes)

enum gsd_open_flag open_flags

Flags passed to gsd_open() when opening this handle.

struct gsd_name_id_map name_map

Access the names in the namelist.

uint64_t pending_index_entries

Number of index entries pending in the current frame.

uint64_t maximum_write_buffer_size

Maximum write buffer size (bytes).

uint64_t index_entries_to_buffer

Number of index entries to buffer before flushing.

struct gsd_header
#include <gsd.h>

GSD file header.

The in-memory and on-disk storage of the GSD file header. Stored in the first 256 bytes of the file.

Warning

All members are read-only to the caller.

Public Members

uint64_t magic

Magic number marking that this is a GSD file.

uint64_t index_location

Location of the chunk index in the file.

uint64_t index_allocated_entries

Number of index entries that will fit in the space allocated.

uint64_t namelist_location

Location of the name list in the file.

uint64_t namelist_allocated_entries

Number of bytes in the namelist divided by GSD_NAME_SIZE.

uint32_t schema_version

Schema version: from gsd_make_version().

uint32_t gsd_version

GSD file format version from gsd_make_version().

char application[GSD_NAME_SIZE]

Name of the application that generated this file.

char schema[GSD_NAME_SIZE]

Name of data schema.

char reserved[GSD_RESERVED_BYTES]

Reserved for future use.

struct gsd_index_buffer
#include <gsd.h>

Array of index entries.

May point to a mapped location of index entries in the file or an in-memory buffer.

Public Members

struct gsd_index_entry *data

Indices in the buffer.

size_t size

Number of entries in the buffer.

size_t reserved

Number of entries available in the buffer.

void *mapped_data

Pointer to mapped data (NULL if not mapped)

size_t mapped_len

Number of bytes mapped.

struct gsd_index_entry
#include <gsd.h>

Index entry.

An index entry for a single chunk of data.

Warning

All members are read-only to the caller.

Public Members

uint64_t frame

Frame index of the chunk.

uint64_t N

Number of rows in the chunk.

int64_t location

Location of the chunk in the file.

uint32_t M

Number of columns in the chunk.

uint16_t id

Index of the chunk name in the name list.

uint8_t type

Data type of the chunk: one of gsd_type.

uint8_t flags

Flags (for internal use).

struct gsd_name_buffer
#include <gsd.h>

Name buffer.

Holds a list of string names in order separated by NULL terminators. In v1 files, each name is 64 bytes. In v2 files, only one NULL terminator is placed between each name.

Public Members

struct gsd_byte_buffer data

Data.

size_t n_names

Number of names in the list.

struct gsd_name_id_map
#include <gsd.h>

Name/id hash map.

A hash map of string names to integer identifiers.

Public Members

struct gsd_name_id_pair *v

Name/id mappings.

size_t size

Number of entries in the mapping.

struct gsd_name_id_pair
#include <gsd.h>

Name/id mapping.

A string name paired with an ID. Used for storing sorted name/id mappings in a hash map.

Public Members

char *name

Pointer to name (actual name storage is allocated in gsd_handle)

struct gsd_name_id_pair *next

Next name/id pair with the same hash.

uint16_t id

Entry id.

file gsd.h
#include <stdbool.h>
#include <stdint.h>
#include <string.h>

Declare GSD data types and C API.

Enums

enum gsd_type

Identifiers for the gsd data chunk element types.

Values:

enumerator GSD_TYPE_UINT8

Unsigned 8-bit integer.

enumerator GSD_TYPE_UINT16

Unsigned 16-bit integer.

enumerator GSD_TYPE_UINT32

Unsigned 32-bit integer.

enumerator GSD_TYPE_UINT64

Unsigned 53-bit integer.

enumerator GSD_TYPE_INT8

Signed 8-bit integer.

enumerator GSD_TYPE_INT16

Signed 16-bit integer.

enumerator GSD_TYPE_INT32

Signed 32-bit integer.

enumerator GSD_TYPE_INT64

Signed 64-bit integer.

enumerator GSD_TYPE_FLOAT

32-bit floating point number.

enumerator GSD_TYPE_DOUBLE

64-bit floating point number.

enum gsd_open_flag

Flag for GSD file open options.

Values:

enumerator GSD_OPEN_READWRITE

Open for both reading and writing.

enumerator GSD_OPEN_READONLY

Open only for reading.

enumerator GSD_OPEN_APPEND

Open only for writing.

enum gsd_error

Error return values.

Values:

enumerator GSD_SUCCESS

Success.

enumerator GSD_ERROR_IO

IO error. Check errno for details.

enumerator GSD_ERROR_INVALID_ARGUMENT

Invalid argument passed to function.

enumerator GSD_ERROR_NOT_A_GSD_FILE

The file is not a GSD file.

enumerator GSD_ERROR_INVALID_GSD_FILE_VERSION

The GSD file version cannot be read.

enumerator GSD_ERROR_FILE_CORRUPT

The GSD file is corrupt.

enumerator GSD_ERROR_MEMORY_ALLOCATION_FAILED

GSD failed to allocated memory.

enumerator GSD_ERROR_NAMELIST_FULL

The GSD file cannot store any additional unique data chunk names.

enumerator GSD_ERROR_FILE_MUST_BE_WRITABLE

This API call requires that the GSD file opened in with the mode GSD_OPEN_APPEND or GSD_OPEN_READWRITE.

enumerator GSD_ERROR_FILE_MUST_BE_READABLE

This API call requires that the GSD file opened the mode GSD_OPEN_READ or GSD_OPEN_READWRITE.

enum [anonymous]

Values:

enumerator GSD_NAME_SIZE

v1 file: Size of a GSD name in memory.

v2 file: The name buffer size is a multiple of GSD_NAME_SIZE.

enum [anonymous]

Values:

enumerator GSD_RESERVED_BYTES

Reserved bytes in the header structure.

Functions

uint32_t gsd_make_version(unsigned int major, unsigned int minor)

Specify a version.

Parameters:
  • major – major version

  • minor – minor version

Returns:

a packed version number aaaa.bbbb suitable for storing in a gsd file version entry.

int gsd_create(const char *fname, const char *application, const char *schema, uint32_t schema_version)

Create a GSD file.

The generated gsd file is not opened. Call gsd_open() to open it for writing.

Parameters:
  • fname – File name (UTF-8 encoded).

  • application – Generating application name (truncated to 63 chars).

  • schema – Schema name for data to be written in this GSD file (truncated to 63 chars).

  • schema_version – Version of the scheme data to be written (make with gsd_make_version()).

Post:

Create an empty gsd file in a file of the given name. Overwrite any existing file at that location.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

int gsd_create_and_open(struct gsd_handle *handle, const char *fname, const char *application, const char *schema, uint32_t schema_version, enum gsd_open_flag flags, int exclusive_create)

Create and open a GSD file.

Open the generated gsd file in handle.

The file descriptor is closed if there when an error opening the file.

Parameters:
  • handle – Handle to open.

  • fname – File name (UTF-8 encoded).

  • application – Generating application name (truncated to 63 chars).

  • schema – Schema name for data to be written in this GSD file (truncated to 63 chars).

  • schema_version – Version of the scheme data to be written (make with gsd_make_version()).

  • flags – Either GSD_OPEN_READWRITE, or GSD_OPEN_APPEND.

  • exclusive_create – Set to non-zero to force exclusive creation of the file.

Post:

Create an empty gsd file with the given name. Overwrite any existing file at that location.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

  • GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

  • GSD_ERROR_FILE_CORRUPT: Corrupt file.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

int gsd_open(struct gsd_handle *handle, const char *fname, enum gsd_open_flag flags)

Open a GSD file.

The file descriptor is closed if there is an error opening the file.

Parameters:
  • handle – Handle to open.

  • fname – File name to open (UTF-8 encoded).

  • flags – Either GSD_OPEN_READWRITE, GSD_OPEN_READONLY, or GSD_OPEN_APPEND.

Pre:

The file name fname is a GSD file.

Post:

Open a GSD file and populates the handle for use by API calls.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

  • GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

  • GSD_ERROR_FILE_CORRUPT: Corrupt file.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

int gsd_truncate(struct gsd_handle *handle)

Truncate a GSD file.

After truncating, a file will have no frames and no data chunks. The file size will be that of a newly created gsd file. The application, schema, and schema version metadata will be kept. Truncate does not close and reopen the file, so it is suitable for writing restart files on Lustre file systems without any metadata access.

Parameters:
  • handle – Open GSD file to truncate.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_NOT_A_GSD_FILE: Not a GSD file.

  • GSD_ERROR_INVALID_GSD_FILE_VERSION: Invalid GSD file version.

  • GSD_ERROR_FILE_CORRUPT: Corrupt file.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

int gsd_close(struct gsd_handle *handle)

Close a GSD file.

Warning

Ensure that all gsd_write_chunk() calls are completed with gsd_end_frame() before closing the file.

Parameters:
  • handle – GSD file to close.

Pre:

handle was opened by gsd_open().

Post:

Writable files: All data and index entries buffered before the previous call to gsd_end_frame() is written to the file (see gsd_flush()).

Post:

The file is closed.

Post:

handle is freed and can no longer be used.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

int gsd_end_frame(struct gsd_handle *handle)

Complete the current frame.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open().

Post:

The current frame counter is increased by 1.

Post:

Flush the write buffer if it has overflowed. See gsd_flush().

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

  • GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

int gsd_flush(struct gsd_handle *handle)

Flush the write buffer.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open().

Post:

All data buffered by gsd_write_chunk() are present in the file.

Post:

All index entries buffered by gsd_write_chunk() prior to the last call to gsd_end_frame() are present in the file.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL.

  • GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: Unable to allocate memory.

int gsd_write_chunk(struct gsd_handle *handle, const char *name, enum gsd_type type, uint64_t N, uint32_t M, uint8_t flags, const void *data)

Add a data chunk to the current frame.

Note

If the GSD file is version 1.0, the chunk name is truncated to 63 bytes. GSD version 2.0 files support arbitrarily long names.

Note

N == 0 is allowed. When N is 0, data may be NULL.

Parameters:
  • handle – Handle to an open GSD file.

  • name – Name of the data chunk.

  • type – type ID that identifies the type of data in data.

  • N – Number of rows in the data.

  • M – Number of columns in the data.

  • flags – set to 0, non-zero values reserved for future use.

  • data – Data buffer.

Pre:

handle was opened by gsd_open().

Pre:

name is a unique name for data chunks in the given frame.

Pre:

data is allocated and contains at least N * M * gsd_sizeof_type(type) bytes.

Post:

When there is space in the buffer: The given data is present in the write buffer. Otherwise, the data is present at the end of the file.

Post:

The index is present in the buffer.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL, N == 0, M == 0, type is invalid, or flags* != 0.

  • GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened read-only.

  • GSD_ERROR_NAMELIST_FULL: The file cannot store any additional unique chunk names.

  • GSD_ERROR_MEMORY_ALLOCATION_FAILED: failed to allocate memory.

const struct gsd_index_entry *gsd_find_chunk(struct gsd_handle *handle, uint64_t frame, const char *name)

Find a chunk in the GSD file.

The found entry contains size and type metadata and can be passed to gsd_read_chunk() to read the data.

Note

gsd_find_chunk() calls gsd_flush() when the file is writable.

Parameters:
  • handle – Handle to an open GSD file

  • frame – Frame to look for chunk

  • name – Name of the chunk to find

Pre:

handle was opened by gsd_open() in read or readwrite mode.

Returns:

A pointer to the found chunk, or NULL if not found.

int gsd_read_chunk(struct gsd_handle *handle, void *data, const struct gsd_index_entry *chunk)

Read a chunk from the GSD file.

Note

gsd_read_chunk() calls gsd_flush() when the file is writable.

Parameters:
  • handle – Handle to an open GSD file.

  • data – Data buffer to read into.

  • chunk – Chunk to read.

Pre:

handle was opened in read or readwrite mode.

Pre:

chunk was found by gsd_find_chunk().

Pre:

data points to an allocated buffer with at least N * M * gsd_sizeof_type(type) bytes.

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL, data is NULL, or chunk is NULL.

  • GSD_ERROR_FILE_MUST_BE_READABLE: The file was opened in append mode.

  • GSD_ERROR_FILE_CORRUPT: The GSD file is corrupt.

uint64_t gsd_get_nframes(struct gsd_handle *handle)

Get the number of frames in the GSD file.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open().

Returns:

The number of frames in the file, or 0 on error.

size_t gsd_sizeof_type(enum gsd_type type)

Query size of a GSD type ID.

Parameters:
  • type – Type ID to query.

Returns:

Size of the given type in bytes, or 0 for an unknown type ID.

const char *gsd_find_matching_chunk_name(struct gsd_handle *handle, const char *match, const char *prev)

Search for chunk names in a gsd file.

To find the first matching chunk name, pass NULL for prev. Pass in the previous found string to find the next after that, and so on. Chunk names match if they begin with the string in match*. Chunk names returned by this function may be present in at least one frame.

Note

gsd_find_matching_chunk_name() calls gsd_flush() when the file is writable.

Parameters:
  • handle – Handle to an open GSD file.

  • match – String to match.

  • prev – Search starting point.

Pre:

handle was opened by gsd_open()

Pre:

prev was returned by a previous call to gsd_find_matching_chunk_name()

Returns:

Pointer to a string, NULL if no more matching chunks are found found, or NULL if prev* is invalid

int gsd_upgrade(struct gsd_handle *handle)

Upgrade a GSD file to the latest specification.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open() with a writable mode.

Pre:

There are no pending data to write to the file in gsd_end_frame()

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_IO: IO error (check errno).

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL

  • GSD_ERROR_FILE_MUST_BE_WRITABLE: The file was opened in read-only mode.

uint64_t gsd_get_maximum_write_buffer_size(struct gsd_handle *handle)

Get the maximum write buffer size.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open().

Returns:

The maximum write buffer size in bytes, or 0 on error.

int gsd_set_maximum_write_buffer_size(struct gsd_handle *handle, uint64_t size)

Set the maximum write buffer size.

Parameters:
  • handle – Handle to an open GSD file

  • size – Maximum number of bytes to allocate in the write buffer (must be greater than 0).

Pre:

handle was opened by gsd_open().

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL

  • GSD_ERROR_INVALID_ARGUMENT: size == 0

uint64_t gsd_get_index_entries_to_buffer(struct gsd_handle *handle)

Get the number of index entries to buffer.

Parameters:
  • handle – Handle to an open GSD file

Pre:

handle was opened by gsd_open().

Returns:

The number of index entries to buffer, or 0 on error.

int gsd_set_index_entries_to_buffer(struct gsd_handle *handle, uint64_t number)

Set the number of index entries to buffer.

Note

GSD may allocate more than this number of entries in the buffer, as needed to store all index entries for the already buffered frames and the current frame.

Parameters:
  • handle – Handle to an open GSD file

  • number – Number of index entries to buffer before automatically flushing in gsd_end_frame() (must be greater than 0).

Pre:

handle was opened by gsd_open().

Returns:

  • GSD_SUCCESS (0) on success. Negative value on failure:

  • GSD_ERROR_INVALID_ARGUMENT: handle is NULL

  • GSD_ERROR_INVALID_ARGUMENT: number == 0

dir gsd
type uint8_t

8-bit unsigned integer (defined by C compiler).

type uint16_t

16-bit unsigned integer (defined by C compiler).

type uint32_t

32-bit unsigned integer (defined by C compiler).

type uint64_t

64-bit unsigned integer (defined by C compiler).

type int64_t

64-bit signed integer (defined by C compiler).

type size_t

unsigned integer (defined by C compiler).