commit 18800cc956556b2b1ac60a9d0ec75b33cc6054a9
parent abb9b9533a4dc4c853f99b81348213584aedc891
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Sat, 15 Apr 2023 16:36:13 +0200
Make memory mapping optional
By default, data is now loaded into memory. Memory mapping becomes an
option of the load functions. Their APIs are therefore updated.
Diffstat:
| M | src/smsh.c | | | 247 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------- |
| M | src/smsh.h | | | 28 | +++++++++++++++++++++++----- |
| M | src/test_smsh_load.c | | | 64 | +++++++++++++++++++++++++++++++++++++++++++++++----------------- |
3 files changed, 251 insertions(+), 88 deletions(-)
diff --git a/src/smsh.c b/src/smsh.c
@@ -21,6 +21,7 @@
#include "smsh_c.h"
#include "smsh_log.h"
+#include <rsys/cstr.h>
#include <rsys/mem_allocator.h>
#include <errno.h>
@@ -38,6 +39,22 @@ check_smsh_create_args(const struct smsh_create_args* args)
return args ? RES_OK : RES_BAD_ARG;
}
+static INLINE res_T
+check_smsh_load_args(const struct smsh_load_args* args)
+{
+ ASSERT(args);
+ if(!args || !args->path) return RES_BAD_ARG;
+ return RES_OK;
+}
+
+static INLINE res_T
+check_smsh_load_stream_args(const struct smsh_load_stream_args* args)
+{
+ ASSERT(args);
+ if(!args || !args->stream || !args->name) return RES_BAD_ARG;
+ return RES_OK;
+}
+
static void
reset_smsh(struct smsh* smsh)
{
@@ -45,20 +62,42 @@ reset_smsh(struct smsh* smsh)
smsh->nnodes = 0;
smsh->ncells = 0;
smsh->pagesize = 0;
- if(smsh->nodes && smsh->nodes != MAP_FAILED)
- munmap(smsh->nodes, smsh->map_len_nodes);
- if(smsh->cells && smsh->cells != MAP_FAILED)
- munmap(smsh->cells, smsh->map_len_cells);
+ if(smsh->nodes) {
+ if(!smsh->map_len_nodes) {
+ MEM_RM(smsh->allocator, smsh->nodes);
+ } else if(smsh->nodes != MAP_FAILED) {
+ munmap(smsh->nodes, smsh->map_len_nodes);
+ }
+ }
+ if(smsh->cells) {
+ if(!smsh->map_len_cells) {
+ MEM_RM(smsh->allocator, smsh->cells);
+ } else if(smsh->nodes != MAP_FAILED) {
+ munmap(smsh->cells, smsh->map_len_cells);
+ }
+ }
smsh->nodes = NULL;
smsh->cells = NULL;
smsh->map_len_nodes = 0;
smsh->map_len_cells = 0;
}
+static INLINE int
+is_stdin(FILE* stream)
+{
+ struct stat stream_buf;
+ struct stat stdin_buf;
+ ASSERT(stream);
+
+ CHK(fstat(fileno(stream), &stream_buf) == 0);
+ CHK(fstat(STDIN_FILENO, &stdin_buf) == 0);
+ return stream_buf.st_dev == stdin_buf.st_dev;
+}
+
static res_T
map_data
(struct smsh* smsh,
- const char* stream_name,
+ const char* name,
const int fd, /* File descriptor */
const size_t filesz, /* Overall filesize */
const char* data_name,
@@ -68,12 +107,12 @@ map_data
{
void* map = NULL;
res_T res = RES_OK;
- ASSERT(smsh && stream_name && filesz && data_name && map_len && out_map);
+ ASSERT(smsh && name && filesz && data_name && map_len && out_map);
ASSERT(IS_ALIGNED((size_t)offset, (size_t)smsh->pagesize));
if((size_t)offset + map_len > filesz) {
log_err(smsh, "%s: the %s to load exceed the file size.\n",
- stream_name, data_name);
+ name, data_name);
res = RES_IO_ERR;
goto error;
}
@@ -81,7 +120,7 @@ map_data
map = mmap(NULL, map_len, PROT_READ, MAP_PRIVATE|MAP_POPULATE, fd, offset);
if(map == MAP_FAILED) {
log_err(smsh, "%s: could not map the %s -- %s.\n",
- stream_name, data_name, strerror(errno));
+ name, data_name, strerror(errno));
res = RES_IO_ERR;
goto error;
}
@@ -94,40 +133,129 @@ error:
goto exit;
}
-static INLINE int
-is_stdin(FILE* stream)
+static res_T
+map_file(struct smsh* smsh, FILE* stream, const char* name)
{
- struct stat stream_buf;
- struct stat stdin_buf;
- ASSERT(stream);
+ off_t pos_offset;
+ off_t ids_offset;
+ size_t filesz;
+ res_T res = RES_OK;
+ ASSERT(smsh && stream && name);
- CHK(fstat(fileno(stream), &stream_buf) == 0);
- CHK(fstat(STDIN_FILENO, &stdin_buf) == 0);
- return stream_buf.st_dev == stdin_buf.st_dev;
+ /* Compute the length in bytes of the data to map */
+ smsh->map_len_nodes = smsh->nnodes * sizeof(double) * smsh->dnode;
+ smsh->map_len_cells = smsh->ncells * sizeof(uint64_t) * smsh->dcell;
+ smsh->map_len_nodes = ALIGN_SIZE(smsh->map_len_nodes, (size_t)smsh->pagesize);
+ smsh->map_len_cells = ALIGN_SIZE(smsh->map_len_cells, (size_t)smsh->pagesize);
+
+ /* Find the offsets of the positions/indices data into the stream */
+ pos_offset = (off_t)ALIGN_SIZE((uint64_t)ftell(stream), smsh->pagesize);
+ ids_offset = (off_t)((size_t)pos_offset + smsh->map_len_nodes);
+
+ /* Retrieve the overall filesize */
+ fseek(stream, 0, SEEK_END);
+ filesz = (size_t)ftell(stream);
+
+ /* Map the nodes */
+ res = map_data(smsh, name, fileno(stream), filesz, "nodes",
+ pos_offset, smsh->map_len_nodes, (void**)&smsh->nodes);
+ if(res != RES_OK) goto error;
+
+ /* Map the cells */
+ res = map_data(smsh, name, fileno(stream), filesz, "cells",
+ ids_offset, smsh->map_len_cells, (void**)&smsh->cells);
+ if(res != RES_OK) goto error;
+
+exit:
+ return res;
+error:
+ goto exit;
}
static res_T
-load_stream(struct smsh* smsh, FILE* stream, const char* stream_name)
+read_padding(FILE* stream, const size_t padding)
{
- off_t pos_offset;
- off_t ids_offset;
- size_t filesz;
+ char chunk[1024];
+ size_t remaining_nbytes = padding;
+
+ while(remaining_nbytes) {
+ const size_t nbytes = MMIN(sizeof(chunk), remaining_nbytes);
+ if(fread(chunk, 1, nbytes, stream) != nbytes) return RES_IO_ERR;
+ remaining_nbytes -= nbytes;
+ }
+ return RES_OK;
+}
+
+static res_T
+load_file(struct smsh* smsh, FILE* stream, const char* name)
+{
+ size_t ncoords;
+ size_t nindices;
+ size_t padding_bytes;
+ size_t sizeof_nodes;
+ size_t sizeof_cells;
+ size_t sizeof_header;
res_T res = RES_OK;
- ASSERT(smsh && stream && stream_name);
+ ASSERT(smsh && stream && name);
+
+ ncoords = smsh->nnodes * smsh->dnode;
+ nindices = smsh->ncells * smsh->dcell;
+ sizeof_nodes = ncoords * sizeof(*smsh->nodes);
+ sizeof_cells = nindices * sizeof(*smsh->cells);
+ sizeof_header =
+ sizeof(smsh->pagesize)
+ + sizeof(smsh->nnodes)
+ + sizeof(smsh->ncells)
+ + sizeof(smsh->dnode)
+ + sizeof(smsh->dcell);
+
+ /* Allocate the memory space where the loaded data will be stored */
+ smsh->nodes = MEM_CALLOC(smsh->allocator, ncoords, sizeof(*smsh->nodes));
+ if(!smsh->nodes) { res = RES_MEM_ERR; goto error; }
+ smsh->cells = MEM_CALLOC(smsh->allocator, nindices, sizeof(*smsh->cells));
+ if(!smsh->cells) { res = RES_MEM_ERR; goto error; }
+
+ padding_bytes = ALIGN_SIZE(sizeof_header, smsh->pagesize) - sizeof_header;
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
+
+ /* Load the nodes */
+ if(fread(smsh->nodes, sizeof(*smsh->nodes), ncoords, stream) != ncoords) {
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+ padding_bytes = ALIGN_SIZE(sizeof_nodes, smsh->pagesize) - sizeof_nodes;
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
- if(is_stdin(stream)) {
- log_err(smsh, "%s: the data cannot be load from the standard input\n",
- stream_name);
+ /* Load the cells */
+ if(fread(smsh->cells, sizeof(*smsh->cells), nindices, stream) != nindices) {
res = RES_IO_ERR;
goto error;
}
+ padding_bytes = ALIGN_SIZE(sizeof_cells, smsh->pagesize) - sizeof_cells;
+ if((res = read_padding(stream, padding_bytes)) != RES_OK) goto error;
+
+exit:
+ return res;
+error:
+ log_err(smsh, "%s: error while loading data -- %s.\n",
+ name, res_to_cstr(res));
+ goto exit;
+}
+
+static res_T
+load_stream(struct smsh* smsh, const struct smsh_load_stream_args* args)
+{
+ res_T res = RES_OK;
+ ASSERT(smsh && check_smsh_load_stream_args(args) == RES_OK);
+
reset_smsh(smsh);
/* Read file header */
#define READ(Var, N, Name) { \
- if(fread((Var), sizeof(*(Var)), (N), stream) != (N)) { \
- log_err(smsh, "%s: could not read the %s.\n", stream_name, (Name)); \
+ if(fread((Var), sizeof(*(Var)), (N), args->stream) != (N)) { \
+ log_err(smsh, "%s: could not read the %s.\n", args->name, (Name));\
res = RES_IO_ERR; \
goto error; \
} \
@@ -143,34 +271,18 @@ load_stream(struct smsh* smsh, FILE* stream, const char* stream_name)
log_err(smsh,
"%s: invalid page size %li. The page size attribute must be aligned on "
"the page size of the operating system (%lu).\n",
- stream_name, smsh->pagesize, (unsigned long)smsh->pagesize_os);
+ args->name, smsh->pagesize, (unsigned long)smsh->pagesize_os);
res = RES_BAD_ARG;
goto error;
}
- /* Compute the length in bytes of the data to map */
- smsh->map_len_nodes = smsh->nnodes * sizeof(double) * smsh->dnode;
- smsh->map_len_cells = smsh->ncells * sizeof(uint64_t) * smsh->dcell;
- smsh->map_len_nodes = ALIGN_SIZE(smsh->map_len_nodes, (size_t)smsh->pagesize);
- smsh->map_len_cells = ALIGN_SIZE(smsh->map_len_cells, (size_t)smsh->pagesize);
-
- /* Find the offsets of the positions/indices data into the stream */
- pos_offset = (off_t)ALIGN_SIZE((uint64_t)ftell(stream), smsh->pagesize);
- ids_offset = (off_t)((size_t)pos_offset + smsh->map_len_nodes);
-
- /* Retrieve the overall filesize */
- fseek(stream, 0, SEEK_END);
- filesz = (size_t)ftell(stream);
-
- /* Map the nodes */
- res = map_data(smsh, stream_name, fileno(stream), filesz, "nodes",
- pos_offset, smsh->map_len_nodes, (void**)&smsh->nodes);
- if(res != RES_OK) goto error;
-
- /* Map the cells */
- res = map_data(smsh, stream_name, fileno(stream), filesz, "cells",
- ids_offset, smsh->map_len_cells, (void**)&smsh->cells);
- if(res != RES_OK) goto error;
+ if(args->memory_mapping) {
+ res = map_file(smsh, args->stream, args->name);
+ if(res != RES_OK) goto error;
+ } else {
+ res = load_file(smsh, args->stream, args->name);
+ if(res != RES_OK) goto error;
+ }
exit:
return res;
@@ -260,24 +372,27 @@ smsh_ref_put(struct smsh* smsh)
res_T
-smsh_load(struct smsh* smsh, const char* path)
+smsh_load(struct smsh* smsh, const struct smsh_load_args* args)
{
+ struct smsh_load_stream_args stream_args = SMSH_LOAD_STREAM_ARGS_NULL;
FILE* file = NULL;
res_T res = RES_OK;
- if(!smsh || !path) {
- res = RES_BAD_ARG;
- goto error;
- }
+ if(!smsh) { res = RES_BAD_ARG; goto error; }
+ res = check_smsh_load_args(args);
+ if(res != RES_OK) goto error;
- file = fopen(path, "r");
+ file = fopen(args->path, "r");
if(!file) {
- log_err(smsh, "%s: error opening file `%s'.\n", FUNC_NAME, path);
+ log_err(smsh, "%s: error opening file `%s'.\n", FUNC_NAME, args->path);
res = RES_IO_ERR;
goto error;
}
- res = load_stream(smsh, file, path);
+ stream_args.stream = file;
+ stream_args.name = args->path;
+ stream_args.memory_mapping = args->memory_mapping;
+ res = load_stream(smsh, &stream_args);
if(res != RES_OK) goto error;
exit:
@@ -288,13 +403,13 @@ error:
}
res_T
-smsh_load_stream
- (struct smsh* smsh,
- FILE* stream,
- const char* stream_name)
+smsh_load_stream(struct smsh* smsh, const struct smsh_load_stream_args* args)
{
- if(!smsh || !stream) return RES_BAD_ARG;
- return load_stream(smsh, stream, stream_name ? stream_name : "<stream>");
+ res_T res = RES_OK;
+ if(!smsh) return RES_BAD_ARG;
+ res = check_smsh_load_stream_args(args);
+ if(res != RES_OK) return res;
+ return load_stream(smsh, args);
}
res_T
@@ -321,8 +436,8 @@ smsh_desc_compute_hash(const struct smsh_desc* desc, hash256_T hash)
sha256_ctx_update(&ctx, (const char*)(Var), sizeof(*Var)*(Nb));
sha256_ctx_init(&ctx);
- HASH(desc->nodes, desc->nnodes);
- HASH(desc->cells, desc->ncells);
+ HASH(desc->nodes, desc->nnodes*desc->dnode);
+ HASH(desc->cells, desc->ncells*desc->dcell);
HASH(&desc->nnodes, 1);
HASH(&desc->ncells, 1);
HASH(&desc->dnode, 1);
diff --git a/src/smsh.h b/src/smsh.h
@@ -50,9 +50,28 @@ struct smsh_create_args {
static const struct smsh_create_args SMSH_CREATE_ARGS_DEFAULT =
SMSH_CREATE_ARGS_DEFAULT__;
+struct smsh_load_args {
+ const char* path;
+ int memory_mapping; /* Use memory mapping instead of normal loading */
+};
+#define SMSH_LOAD_ARGS_NULL__ {NULL, 0}
+static const struct smsh_load_args SMSH_LOAD_ARGS_NULL =
+ SMSH_LOAD_ARGS_NULL__;
+
+struct smsh_load_stream_args {
+ FILE* stream;
+ const char* name; /* Name of the stream */
+ /* Use memory mapping instead of normal loading. Note that memory mapping
+ * cannot be used on some stream like stdin */
+ int memory_mapping;
+};
+#define SMSH_LOAD_STREAM_ARGS_NULL__ {NULL, "stream", 0}
+static const struct smsh_load_stream_args SMSH_LOAD_STREAM_ARGS_NULL =
+ SMSH_LOAD_STREAM_ARGS_NULL__;
+
struct smsh_desc {
- const double* nodes; /* List of double[3] */
- const uint64_t* cells; /* List of uint64_t[4] */
+ const double* nodes; /* List of double[dnode] */
+ const uint64_t* cells; /* List of uint64_t[dcell] */
size_t nnodes;
size_t ncells;
unsigned dnode; /* Dimension of a node */
@@ -85,13 +104,12 @@ smsh_ref_put
SMSH_API res_T
smsh_load
(struct smsh* smsh,
- const char* path);
+ const struct smsh_load_args* args);
SMSH_API res_T
smsh_load_stream
(struct smsh* smsh,
- FILE* stream,
- const char* stream_name); /* NULL <=> use default stream name */
+ const struct smsh_load_stream_args* args);
SMSH_API res_T
smsh_get_desc
diff --git a/src/test_smsh_load.c b/src/test_smsh_load.c
@@ -19,6 +19,7 @@
#include <rsys/math.h>
#include <rsys/mem_allocator.h>
#include <rsys/rsys.h>
+
#include <stdio.h>
#include <unistd.h>
@@ -70,6 +71,8 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell)
hash256_T hash0;
hash256_T hash1;
struct smsh_desc desc = SMSH_DESC_NULL;
+ struct smsh_load_args args = SMSH_LOAD_ARGS_NULL;
+ struct smsh_load_stream_args stream_args = SMSH_LOAD_STREAM_ARGS_NULL;
FILE* fp = NULL;
const char* filename = "test_file.smsh";
const uint64_t pagesize = 16384;
@@ -116,9 +119,17 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell)
CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1);
rewind(fp);
- CHK(smsh_load_stream(NULL, fp, filename) == RES_BAD_ARG);
- CHK(smsh_load_stream(smsh, NULL, filename) == RES_BAD_ARG);
- CHK(smsh_load_stream(smsh, fp, NULL) == RES_OK);
+
+ stream_args.stream = fp;
+ stream_args.name = filename;
+ CHK(smsh_load_stream(NULL, &stream_args) == RES_BAD_ARG);
+ stream_args.stream = NULL;
+ CHK(smsh_load_stream(smsh, &stream_args) == RES_BAD_ARG);
+ stream_args.stream = fp;
+ stream_args.name = NULL;
+ CHK(smsh_load_stream(smsh, &stream_args) == RES_BAD_ARG);
+ stream_args.name = filename;
+ CHK(smsh_load_stream(smsh, &stream_args) == RES_OK);
CHK(smsh_get_desc(NULL, &desc) == RES_BAD_ARG);
CHK(smsh_get_desc(smsh, NULL) == RES_BAD_ARG);
CHK(smsh_get_desc(smsh, &desc) == RES_OK);
@@ -129,25 +140,41 @@ test_load_mesh(struct smsh* smsh, const uint32_t dnode, const uint32_t dcell)
CHK(smsh_desc_compute_hash(&desc, hash0) == RES_OK);
rewind(fp);
- CHK(smsh_load_stream(smsh, fp, filename) == RES_OK);
+ stream_args.name = SMSH_LOAD_STREAM_ARGS_NULL.name;
+ stream_args.memory_mapping = 1;
+ CHK(smsh_load_stream(smsh, &stream_args) == RES_OK);
CHK(smsh_get_desc(smsh, &desc) == RES_OK);
check_smsh_desc(&desc, nnodes, ncells, dnode, dcell);
CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK);
CHK(hash256_eq(hash0, hash1));
+ CHK(fclose(fp) == 0);
- CHK(smsh_load(NULL, filename) == RES_BAD_ARG);
- CHK(smsh_load(smsh, NULL) == RES_BAD_ARG);
- CHK(smsh_load(smsh, "nop") == RES_IO_ERR);
- CHK(smsh_load(smsh, filename) == RES_OK);
+ args.path = filename;
+ CHK(smsh_load(NULL, &args) == RES_BAD_ARG);
+ args.path = NULL;
+ CHK(smsh_load(smsh, &args) == RES_BAD_ARG);
+ args.path = "nop";
+ CHK(smsh_load(smsh, &args) == RES_IO_ERR);
+ args.path = filename;
+ CHK(smsh_load(smsh, &args) == RES_OK);
+ CHK(smsh_get_desc(smsh, &desc) == RES_OK);
check_smsh_desc(&desc, nnodes, ncells, dnode, dcell);
+ CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK);
+ CHK(hash256_eq(hash0, hash1));
- fclose(fp);
+ args.memory_mapping = 1;
+ CHK(smsh_load(smsh, &args) == RES_OK);
+ CHK(smsh_get_desc(smsh, &desc) == RES_OK);
+ check_smsh_desc(&desc, nnodes, ncells, dnode, dcell);
+ CHK(smsh_desc_compute_hash(&desc, hash1) == RES_OK);
+ CHK(hash256_eq(hash0, hash1));
}
static void
test_load_fail(struct smsh* smsh)
{
+ struct smsh_load_stream_args args = SMSH_LOAD_STREAM_ARGS_NULL;
const char byte = 0;
FILE* fp = NULL;
uint64_t pagesize;
@@ -176,8 +203,9 @@ test_load_fail(struct smsh* smsh)
CHK(fseek(fp, (long)ALIGN_SIZE((size_t)ftell(fp), pagesize)-1, SEEK_SET) == 0);
CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1);
rewind(fp);
- CHK(smsh_load_stream(smsh, fp, NULL) == RES_BAD_ARG);
- fclose(fp);
+ args.stream = fp;
+ CHK(smsh_load_stream(smsh, &args) == RES_BAD_ARG);
+ CHK(fclose(fp) == 0);
/* Wrong size */
fp = tmpfile();
@@ -199,16 +227,15 @@ test_load_fail(struct smsh* smsh)
CHK(fseek(fp, (long)ALIGN_SIZE((size_t)ftell(fp), pagesize)-2, SEEK_SET) == 0);
CHK(fwrite(&byte, sizeof(byte), 1, fp) == 1);
rewind(fp);
- CHK(smsh_load_stream(smsh, fp, NULL) == RES_IO_ERR);
- fclose(fp);
-
- /* Wrong stream */
- CHK(smsh_load_stream(smsh, stdin, "stdin") == RES_IO_ERR);
+ args.stream = fp;
+ CHK(smsh_load_stream(smsh, &args) == RES_IO_ERR);
+ CHK(fclose(fp) == 0);
}
static void
test_load_files(struct smsh* smsh, int argc, char** argv)
{
+ struct smsh_load_args args = SMSH_LOAD_ARGS_NULL;
hash256_T hash;
int i;
CHK(smsh);
@@ -219,7 +246,10 @@ test_load_files(struct smsh* smsh, int argc, char** argv)
size_t icell;
printf("Load %s\n", argv[i]);
- CHK(smsh_load(smsh, argv[i]) == RES_OK);
+
+ args.path = argv[i];
+ args.memory_mapping = 1;
+ CHK(smsh_load(smsh, &args) == RES_OK);
CHK(smsh_get_desc(smsh, &desc) == RES_OK);
CHK(smsh_desc_compute_hash(&desc, hash) == RES_OK);
