commit 29f1c4044fa51e46890c6618992c82812b6ec463
parent 39c7f999f3adc1665695a1aa58f2db9eeaa7c094
Author: Vincent Forest <vincent.forest@meso-star.com>
Date: Thu, 9 Aug 2018 14:45:25 +0200
Add the grid API
This API should be used to store the cloud data on storage device in
order to reuse them in the next runs.
Diffstat:
6 files changed, 439 insertions(+), 51 deletions(-)
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -65,6 +65,7 @@ set(HTRDR_FILES_SRC
htrdr_camera.c
htrdr_compute_radiance_sw.c
htrdr_draw_radiance_sw.c
+ htrdr_grid.c
htrdr_main.c
htrdr_rectangle.c
htrdr_sky.c
@@ -75,6 +76,7 @@ set(HTRDR_FILES_INC
htrdr_args.h
htrdr_buffer.h
htrdr_camera.h
+ htrdr_grid.h
htrdr_rectangle.h
htrdr_sky.h
htrdr_sun.h
diff --git a/src/htrdr.c b/src/htrdr.c
@@ -97,55 +97,6 @@ log_msg
}
static res_T
-open_output_stream(struct htrdr* htrdr, const struct htrdr_args* args)
-{
- FILE* fp = NULL;
- int fd = -1;
- res_T res = RES_OK;
- ASSERT(htrdr && args);
-
- if(args->force_overwriting) {
- fp = fopen(args->output, "w");
- if(!fp) {
- htrdr_log_err(htrdr,
- "could not open the output file `%s'.\n", args->output);
- goto error;
- }
- } else {
- fd = open(args->output, O_CREAT|O_WRONLY|O_EXCL|O_TRUNC, S_IRUSR|S_IWUSR);
- if(fd >= 0) {
- fp = fdopen(fd, "w");
- if(fp == NULL) {
- htrdr_log_err(htrdr,
- "could not open the output file `%s'.\n", args->output);
- goto error;
- }
- } else if(errno == EEXIST) {
- htrdr_log_err(htrdr,
- "the output file `%s' already exists. Use -f to overwrite it.\n",
- args->output);
- goto error;
- } else {
- htrdr_log_err(htrdr,
- "unexpected error while opening the output file `%s'.\n", args->output);
- goto error;
- }
- }
-exit:
- htrdr->output = fp;
- return res;
-error:
- res = RES_IO_ERR;
- if(fp) {
- CHK(fclose(fp) == 0);
- fp = NULL;
- } else if(fd >= 0) {
- CHK(close(fd) == 0);
- }
- goto exit;
-}
-
-static res_T
dump_accum_buffer
(struct htrdr* htrdr,
struct htrdr_buffer* buf,
@@ -268,7 +219,8 @@ htrdr_init
htrdr->output = stdout;
output_name = "<stdout>";
} else {
- res = open_output_stream(htrdr, args);
+ res = open_output_stream
+ (htrdr, args->output, args->force_overwriting, &htrdr->output);
if(res != RES_OK) goto error;
output_name = args->output;
}
@@ -439,6 +391,56 @@ htrdr_log_warn(struct htrdr* htrdr, const char* msg, ...)
/*******************************************************************************
* Local functions
******************************************************************************/
+extern LOCAL_SYM res_T
+open_output_stream
+ (struct htrdr* htrdr,
+ const char* filename,
+ int force_overwrite,
+ FILE** out_fp)
+{
+ FILE* fp = NULL;
+ int fd = -1;
+ res_T res = RES_OK;
+ ASSERT(htrdr && filename && out_fp);
+
+ if(force_overwrite) {
+ fp = fopen(filename, "w");
+ if(!fp) {
+ htrdr_log_err(htrdr, "could not open the output file `%s'.\n", filename);
+ goto error;
+ }
+ } else {
+ fd = open(filename, O_CREAT|O_WRONLY|O_EXCL|O_TRUNC, S_IRUSR|S_IWUSR);
+ if(fd >= 0) {
+ fp = fdopen(fd, "w");
+ if(fp == NULL) {
+ htrdr_log_err(htrdr, "could not open the output file `%s'.\n", filename);
+ goto error;
+ }
+ } else if(errno == EEXIST) {
+ htrdr_log_err(htrdr, "the output file `%s' already exists. \n",
+ filename);
+ goto error;
+ } else {
+ htrdr_log_err(htrdr,
+ "unexpected error while opening the output file `%s'.\n", filename);
+ goto error;
+ }
+ }
+exit:
+ *out_fp = fp;
+ return res;
+error:
+ res = RES_IO_ERR;
+ if(fp) {
+ CHK(fclose(fp) == 0);
+ fp = NULL;
+ } else if(fd >= 0) {
+ CHK(close(fd) == 0);
+ }
+ goto exit;
+}
+
res_T
is_file_updated(struct htrdr* htrdr, const char* filename, int* out_upd)
{
@@ -449,7 +451,7 @@ is_file_updated(struct htrdr* htrdr, const char* filename, int* out_upd)
struct timespec mtime;
int fd = -1;
int err;
- int upd;
+ int upd = 1;
res_T res = RES_OK;
ASSERT(htrdr && filename && out_upd);
diff --git a/src/htrdr_c.h b/src/htrdr_c.h
@@ -33,6 +33,13 @@ wavelength_to_wavenumber(const double lambda/*In nanometer*/)
return wavenumber_to_wavelength(lambda);
}
+extern LOCAL_SYM res_T
+open_output_stream
+ (struct htrdr* htrdr,
+ const char* filename,
+ int force_overwrite,
+ FILE** out_fp);
+
extern LOCAL_SYM res_T
is_file_updated
(struct htrdr* htrdr,
diff --git a/src/htrdr_grid.c b/src/htrdr_grid.c
@@ -0,0 +1,309 @@
+/* Copyright (C) 2018 Université Paul Sabatier, |Meso|Star>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#define _POSIX_C_SOURCE 200809L /* mmap support */
+#define _DEFAULT_SOURCE 1 /* MAP_POPULATE support */
+
+#include "htrdr.h"
+#include "htrdr_c.h"
+#include "htrdr_grid.h"
+
+#include <rsys/mem_allocator.h>
+#include <rsys/ref_count.h>
+
+#include <errno.h>
+#include <sys/mman.h> /* mmap/munmap */
+#include <fcntl.h>
+#include <unistd.h> /* sysconf */
+
+struct htrdr_grid {
+ FILE* fp;
+ char* data;
+ size_t definition[3];
+ size_t cell_sz;
+ size_t pagesize;
+
+ ref_T ref;
+ struct htrdr* htrdr;
+};
+
+/*******************************************************************************
+ * Helper functions
+ ******************************************************************************/
+static void
+grid_release(ref_T* ref)
+{
+ struct htrdr_grid* grid;
+ ASSERT(ref);
+ grid = CONTAINER_OF(ref, struct htrdr_grid, ref);
+ if(grid->fp) fclose(grid->fp);
+ if(grid->data) {
+ size_t grid_sz;
+ grid_sz =
+ grid->definition[0]
+ * grid->definition[1]
+ * grid->definition[2]
+ * grid->cell_sz;
+ grid_sz = ALIGN_SIZE(grid_sz, grid->pagesize);
+ if(munmap(grid->data, grid_sz)) {
+ htrdr_log_err(grid->htrdr, "error unmapping the grid data -- %s.\n",
+ strerror(errno));
+ ASSERT(0);
+ }
+ }
+ MEM_RM(grid->htrdr->allocator, grid);
+}
+
+/*******************************************************************************
+ * Local functions
+ ******************************************************************************/
+res_T
+htrdr_grid_create
+ (struct htrdr* htrdr,
+ const size_t definition[3],
+ const size_t sizeof_cell, /* Size of an cell in Bytes */
+ const char* filename,
+ const int force_overwrite,
+ struct htrdr_grid** out_grid)
+{
+ const char byte = 0;
+ struct htrdr_grid* grid = NULL;
+ size_t grid_sz;
+ long grid_offset;
+ int n;
+ res_T res = RES_OK;
+ ASSERT(!htrdr || !out_grid || !filename || !definition);
+
+ if(!definition[0] || !definition[1] || !definition[2]) {
+ htrdr_log_err(htrdr, "%s: invalid definition [%lu, %lu, %lu].\n", FUNC_NAME,
+ (unsigned long)definition[0],
+ (unsigned long)definition[1],
+ (unsigned long)definition[2]);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ if(!sizeof_cell) {
+ htrdr_log_err(htrdr, "%s: invalid cell size `%lu'.\n", FUNC_NAME,
+ (unsigned long)sizeof_cell);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ grid = MEM_CALLOC(htrdr->allocator, 1, sizeof(*grid));
+ if(!grid) {
+ res = RES_MEM_ERR;
+ goto error;
+ }
+ ref_init(&grid->ref);
+ grid->definition[0] = definition[0];
+ grid->definition[1] = definition[1];
+ grid->definition[2] = definition[2];
+ grid->cell_sz = sizeof_cell;
+ grid->pagesize = (size_t)sysconf(_SC_PAGESIZE);
+ grid->htrdr = htrdr;
+
+ res = open_output_stream(htrdr, filename, force_overwrite, &grid->fp);
+ if(res != RES_OK) goto error;
+
+ #define WRITE(Var, N, Name) { \
+ if(fwrite((Var), sizeof(*(Var)), (N), grid->fp) != (N)) { \
+ htrdr_log_err(htrdr, "%s:%s: could not write `%s'.\n", \
+ FUNC_NAME, filename, (Name)); \
+ res = RES_IO_ERR; \
+ goto error; \
+ } \
+ } (void)0
+ WRITE(&grid->pagesize, 1, "pagesize");
+ WRITE(&grid->cell_sz, 1, "cell_sz");
+ WRITE(grid->definition, 3, "definition");
+
+ /* Align the grid data on pagesize */
+ n = fseek
+ (grid->fp, ALIGN_SIZE(ftell(grid->fp),(off_t)grid->pagesize), SEEK_SET);
+ if(n < 0) {
+ htrdr_log_err(htrdr,
+ "%s:%s: could not align the grid data on page size -- %s.\n",
+ FUNC_NAME, filename, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+ /* Define the grid size */
+ grid_sz = definition[0] * definition[1] * definition[2] * sizeof_cell;
+ grid_sz = ALIGN_SIZE(grid_sz, grid->pagesize);
+
+ /* Save the position of the grid data into the file */
+ grid_offset = ftell(grid->fp);
+
+ /* Reserve the space for the grid data */
+ n = fseek(grid->fp, (long)grid_sz, SEEK_CUR);
+ if(n < 0) {
+ htrdr_log_err(htrdr,
+ "%s:%s: could reserve the space to store the grid -- %s.\n", FUNC_NAME,
+ filename, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+ /* Write one char at the end of the file to position the EOF indicator */
+ CHK(fseek(grid->fp, -1, SEEK_CUR) == 1);
+ WRITE(&byte, 1, "Dummy Byte");
+ #undef WRITE
+
+ grid->data = mmap(NULL, grid_sz, PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_POPULATE, fileno(grid->fp), grid_offset);
+ if(grid->data == MAP_FAILED) {
+ htrdr_log_err(htrdr, "%s:%s: could not map the grid data -- %s.\n",
+ FUNC_NAME, filename, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+exit:
+ *out_grid = grid;
+ return res;
+error:
+ if(grid) {
+ htrdr_grid_ref_put(grid);
+ grid = NULL;
+ }
+ goto exit;
+}
+
+res_T
+htrdr_grid_open
+ (struct htrdr* htrdr,
+ const char* filename,
+ struct htrdr_grid** out_grid)
+{
+ struct htrdr_grid* grid = NULL;
+ size_t grid_sz;
+ size_t grid_offset;
+ size_t pagesize;
+ int fd = -1;
+ res_T res = RES_OK;
+ ASSERT(htrdr && filename && out_grid);
+
+ grid = MEM_CALLOC(htrdr->allocator, 1, sizeof(*grid));
+ if(!grid) {
+ res = RES_MEM_ERR;
+ goto error;
+ }
+ ref_init(&grid->ref);
+ grid->pagesize = (size_t)sysconf(_SC_PAGESIZE);
+ grid->htrdr = htrdr;
+
+ fd = open(filename, O_RDWR, 0);
+ if(fd < 0) {
+ htrdr_log_err(htrdr, "%s: could not open `%s' -- %s.\n", FUNC_NAME,
+ filename, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+ CHK(grid->fp = fdopen(fd, "rw"));
+
+ #define READ(Var, N, Name) { \
+ if(fread((Var), sizeof(*(Var)), (N), grid->fp) != (N)) { \
+ htrdr_log_err(htrdr, "%s:%s: could not read `%s'.\n", \
+ FUNC_NAME, filename, Name); \
+ res = RES_IO_ERR; \
+ goto error; \
+ } \
+ } (void)0
+ READ(&pagesize, 1, "pagesize");
+ if(pagesize != grid->pagesize) {
+ htrdr_log_err(htrdr, "%s:%s: invalid pagesize `%lu'.\n", FUNC_NAME,
+ filename, (unsigned long)pagesize);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ READ(&grid->cell_sz, 1, "sizeof_cell");
+ if(grid->cell_sz == 0) {
+ htrdr_log_err(htrdr, "%s:%s: invalid cell size `%lu'.\n", FUNC_NAME,
+ filename, (unsigned long)grid->cell_sz);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ READ(&grid->definition, 3, "definition");
+ if(!grid->definition[0] || !grid->definition[1] || !grid->definition[2]) {
+ htrdr_log_err(htrdr, "%s:%s: invalid definition [%lu, %lu, %lu].\n",
+ FUNC_NAME, filename,
+ (unsigned long)grid->definition[0],
+ (unsigned long)grid->definition[1],
+ (unsigned long)grid->definition[2]);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+
+ grid_offset = ALIGN_SIZE((size_t)ftell(grid->fp), grid->pagesize);
+ grid_sz =
+ grid->definition[0]
+ * grid->definition[1]
+ * grid->definition[2]
+ * grid->cell_sz;
+ grid_sz = ALIGN_SIZE(grid_sz, grid->pagesize);
+
+ grid->data = mmap(NULL, grid_sz, PROT_READ|PROT_WRITE,
+ MAP_PRIVATE|MAP_POPULATE, fileno(grid->fp), (off_t)grid_offset);
+
+ if(grid->data == MAP_FAILED) {
+ htrdr_log_err(htrdr, "%s:%s: could not map the grid data -- %s.\n",
+ FUNC_NAME, filename, strerror(errno));
+ res = RES_IO_ERR;
+ goto error;
+ }
+
+exit:
+ *out_grid = grid;
+ return res;
+error:
+ if(grid) {
+ htrdr_grid_ref_put(grid);
+ grid = NULL;
+ }
+ goto exit;
+}
+
+void
+htrdr_grid_ref_get(struct htrdr_grid* grid)
+{
+ ASSERT(grid);
+ ref_get(&grid->ref);
+}
+
+void
+htrdr_grid_ref_put(struct htrdr_grid* grid)
+{
+ ASSERT(grid);
+ ref_put(&grid->ref, grid_release);
+}
+
+void*
+htrdr_grid_at(struct htrdr_grid* grid, const size_t xyz[3])
+{
+ size_t slice;
+ size_t pitch;
+ ASSERT(grid && xyz);
+ ASSERT(xyz[0] < grid->definition[0]);
+ ASSERT(xyz[1] < grid->definition[1]);
+ ASSERT(xyz[2] < grid->definition[2]);
+ pitch = grid->definition[0] * grid->cell_sz;
+ slice = grid->definition[1] * pitch;
+ return grid->data + xyz[2]*slice + xyz[1]*pitch + xyz[0]*grid->cell_sz;
+}
+
diff --git a/src/htrdr_grid.h b/src/htrdr_grid.h
@@ -0,0 +1,54 @@
+/* Copyright (C) 2018 Université Paul Sabatier, |Meso|Star>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#ifndef HTRDR_GRID_H
+#define HTRDR_GRID_H
+
+#include <rsys/rsys.h>
+
+/* Forwared declarations */
+struct htrdr;
+struct htrdr_grid;
+
+extern LOCAL_SYM res_T
+htrdr_grid_create
+ (struct htrdr* htrdr,
+ const size_t definition[3],
+ const size_t sizeof_cell, /* Size of an cell in Bytes */
+ const char* filename,
+ const int force_overwrite,
+ struct htrdr_grid** grid);
+
+extern LOCAL_SYM res_T
+htrdr_grid_open
+ (struct htrdr* htrdr,
+ const char* filename,
+ struct htrdr_grid** grid);
+
+extern LOCAL_SYM void
+htrdr_grid_ref_get
+ (struct htrdr_grid* grid);
+
+extern LOCAL_SYM void
+htrdr_grid_ref_put
+ (struct htrdr_grid* grid);
+
+extern LOCAL_SYM void*
+htrdr_grid_at
+ (struct htrdr_grid* grid,
+ const size_t xyz[3]);
+
+#endif /* HTRDR_GRID_H */
+
diff --git a/src/htrdr_sky.c b/src/htrdr_sky.c
@@ -755,6 +755,9 @@ htrdr_sky_create
struct htrdr_sky** out_sky)
{
struct htrdr_sky* sky = NULL;
+ int htcp_upd = 1;
+ int htmie_upd = 1;
+ int htgop_upd = 1;
res_T res = RES_OK;
ASSERT(htrdr && sun && htcp_filename && htmie_filename && out_sky);
@@ -811,6 +814,17 @@ htrdr_sky_create
goto error;
}
+ res = is_file_updated(sky->htrdr, htcp_filename, &htcp_upd);
+ if(res != RES_OK) goto error;
+ res = is_file_updated(sky->htrdr, htmie_filename, &htmie_upd);
+ if(res != RES_OK) goto error;
+ res = is_file_updated(sky->htrdr, htgop_filename, &htgop_upd);
+ if(res != RES_OK) goto error;
+
+ if(htcp_upd || htmie_upd || htgop_upd) {
+ htrdr_log(sky->htrdr, "Cloud cache needs to be rebuid.\n");
+ }
+
res = setup_sw_bands_properties(sky);
if(res != RES_OK) goto error;
