commit 647092dd08bdd48d7b544e37e63f12d84ca85800
parent 9678203d05e409a0fecf1a99d4320ebf18c0fa33
Author: Christophe Coustet <christophe.coustet@meso-star.com>
Date: Thu, 10 Feb 2022 14:49:35 +0100
Add optional MPI support
Diffstat:
7 files changed, 454 insertions(+), 214 deletions(-)
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -18,6 +18,10 @@ project(stardis C)
set(SDIS_SOURCE_DIR ${PROJECT_SOURCE_DIR}/../src)
+option(ENABLE_MPI
+ "Enable the support of distributed parallelism \
+using the Message Passing Interface specification." OFF)
+
if(CMAKE_HOST_UNIX)
set(STARDIS_DOC "TROFF" CACHE STRING
"Type of documentation to generate and install.")
@@ -80,6 +84,12 @@ find_package(StarSP 0.12 REQUIRED)
if(MSVC)
find_package(MuslGetopt REQUIRED)
endif()
+if(ENABLE_MPI)
+ find_package(MPI 2 REQUIRED)
+ set(CMAKE_C_COMPILER ${MPI_C_COMPILER})
+ include_directories(${MPI_INCLUDE_PATH})
+endif()
+
include_directories(
${RSys_INCLUDE_DIR}
@@ -163,6 +173,10 @@ set_target_properties(stardis
target_link_libraries(stardis
Stardis Star3D StarGeom3D StarEnc3D StarSTL StarSP RSys ${GETOPT_LIB} ${MATH_LIB})
+if(ENABLE_MPI)
+ set_target_properties(stardis PROPERTIES COMPILE_DEFINITIONS "STARDIS_ENABLE_MPI")
+endif()
+
###############################################################################
# Define output & install directories
###############################################################################
diff --git a/src/stardis-app.c b/src/stardis-app.c
@@ -13,6 +13,10 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. */
+#ifdef STARDIS_ENABLE_MPI
+#define _POSIX_C_SOURCE 200112L
+#endif
+
#include "stardis-app.h"
#include "stardis-output.h"
#include "stardis-compute.h"
@@ -32,6 +36,11 @@
#include <string.h>
+#ifdef STARDIS_ENABLE_MPI
+#include <stdio.h>
+#include <mpi.h>
+#endif
+
static const struct dummies DUMMIES_NULL = DUMMIES_NULL__;
static const struct counts COUNTS_NULL = COUNTS_NULL__;
@@ -197,6 +206,60 @@ error:
* Public Functions
******************************************************************************/
+#ifdef STARDIS_ENABLE_MPI
+/* To be called after logger has been initialized
+ * and before stardis is initialized */
+res_T
+init_mpi
+ (int* pargc,
+ char** pargv[],
+ void (*prt_err_fn)(const char* msg, void* ctx),
+ void (*prt_warn_fn)(const char* msg, void* ctx))
+{
+ res_T res = RES_OK;
+ char buf[64];
+ int mpi_provided;
+
+ ASSERT(pargc && pargv && prt_err_fn && prt_warn_fn);
+
+ if(MPI_Init_thread(pargc, pargv, MPI_THREAD_MULTIPLE, &mpi_provided)
+ != MPI_SUCCESS)
+ {
+ prt_err_fn("Cannot init MPI\n", NULL);
+ res = RES_BAD_ARG;
+ goto error;
+ }
+ else if(mpi_provided != MPI_THREAD_MULTIPLE) {
+ const char* lvl;
+ switch(mpi_provided) {
+ case MPI_THREAD_SINGLE: lvl = "MPI_THREAD_SINGLE"; break;
+ case MPI_THREAD_FUNNELED: lvl = "MPI_THREAD_FUNNELED"; break;
+ case MPI_THREAD_SERIALIZED: lvl = "MPI_THREAD_SERIALIZED"; break;
+ default: FATAL("Unreachable code.\n"); break;
+ }
+ snprintf(buf, sizeof(buf)-1, "MPI support restricted to %s\n", lvl);
+ prt_warn_fn(buf, NULL);
+ }
+
+end:
+ return res;
+error:
+ goto end;
+}
+
+void
+finalize_mpi
+ (struct stardis* stardis)
+{
+ int initialized;
+ ASSERT(stardis);
+
+ CHK(MPI_Initialized(&initialized) == MPI_SUCCESS);
+ if(initialized)
+ CHK(MPI_Finalize() == MPI_SUCCESS);
+}
+#endif
+
res_T
stardis_init
(const struct args* args,
@@ -267,7 +330,21 @@ stardis_init
dev_args.allocator = stardis->allocator;
dev_args.nthreads_hint = stardis->nthreads;
dev_args.verbosity = stardis->verbose;
- dev_args.use_mpi = 0; /* No MPI yet */
+
+#ifdef STARDIS_ENABLE_MPI
+ logger_print(stardis->logger, LOG_OUTPUT, "MPI is enabled.\n");
+ /* Open MPI accepts the C/C++ argc and argv arguments to main,
+ * but neither modifies, interprets, nor distributes them: use NULL */
+ CHK(MPI_Initialized(&stardis->mpi_initialized) == MPI_SUCCESS);
+ if(stardis->mpi_initialized)
+ CHK(MPI_Comm_rank(MPI_COMM_WORLD, &stardis->mpi_rank) == MPI_SUCCESS);
+#else
+ logger_print(stardis->logger, LOG_OUTPUT, "MPI is disabled.\n");
+ stardis->mpi_initialized = 0;
+#endif
+
+ dev_args.use_mpi = stardis->mpi_initialized;
+
ERR(sdis_device_create(&dev_args, &stardis->dev));
ERR(init_geometry(stardis->logger, stardis->allocator, stardis->verbose,
diff --git a/src/stardis-app.h b/src/stardis-app.h
@@ -38,6 +38,10 @@
#include <limits.h>
#include <stdarg.h>
+#ifdef STARDIS_ENABLE_MPI
+#include <mpi/mpi.h>
+#endif
+
/* Forward declarations */
struct logger;
struct mem_allocator;
@@ -711,8 +715,20 @@ release_camera(struct camera* cam) {
static INLINE void
log_err_fn(const char* msg, void* ctx)
{
+#ifdef STARDIS_ENABLE_MPI
+ int initialized, rank = 0;
+#endif
+
ASSERT(msg);
(void)ctx;
+
+#ifdef STARDIS_ENABLE_MPI
+ CHK(MPI_Initialized(&initialized) == MPI_SUCCESS);
+ if(initialized) CHK(MPI_Comm_rank(MPI_COMM_WORLD, &rank) == MPI_SUCCESS);
+ /* Only master prints */
+ if(rank != 0) return;
+#endif
+
#ifdef OS_WINDOWS
fprintf(stderr, "error: %s", msg);
#else
@@ -723,8 +739,20 @@ log_err_fn(const char* msg, void* ctx)
static INLINE void
log_warn_fn(const char* msg, void* ctx)
{
+#ifdef STARDIS_ENABLE_MPI
+ int initialized, rank = 0;
+#endif
+
ASSERT(msg);
(void)ctx;
+
+#ifdef STARDIS_ENABLE_MPI
+ CHK(MPI_Initialized(&initialized) == MPI_SUCCESS);
+ if(initialized) CHK(MPI_Comm_rank(MPI_COMM_WORLD, &rank) == MPI_SUCCESS);
+ /* Only master prints */
+ if(rank != 0) return;
+#endif
+
#ifdef OS_WINDOWS
fprintf(stderr, "warning: %s", msg);
#else
@@ -735,8 +763,20 @@ log_warn_fn(const char* msg, void* ctx)
static INLINE void
log_prt_fn(const char* msg, void* ctx)
{
+#ifdef STARDIS_ENABLE_MPI
+ int initialized, rank = 0;
+#endif
+
ASSERT(msg);
(void)ctx;
+
+#ifdef STARDIS_ENABLE_MPI
+ CHK(MPI_Initialized(&initialized) == MPI_SUCCESS);
+ if(initialized) CHK(MPI_Comm_rank(MPI_COMM_WORLD, &rank) == MPI_SUCCESS);
+ /* Only master prints */
+ if(rank != 0) return;
+#endif
+
#ifdef OS_WINDOWS
fprintf(stderr, "message: %s", msg);
#else
@@ -809,6 +849,8 @@ struct stardis {
int dump_paths;
int verbose;
int geometry_initialized;
+ int mpi_initialized;
+ int mpi_rank;
};
static INLINE unsigned
@@ -818,6 +860,19 @@ allocate_stardis_medium_id(struct stardis* stardis)
return stardis->next_medium_id++;
}
+#ifdef STARDIS_ENABLE_MPI
+extern LOCAL_SYM res_T
+init_mpi
+ (int* pargc,
+ char** pargv[],
+ void (*prt_err_fn)(const char* msg, void* ctx),
+ void (*prt_warn_fn)(const char* msg, void* ctx));
+
+extern LOCAL_SYM void
+finalize_mpi
+ (struct stardis* stardis);
+#endif
+
extern LOCAL_SYM res_T
stardis_init
(const struct args* args,
diff --git a/src/stardis-args.c b/src/stardis-args.c
@@ -30,12 +30,17 @@
#include <stdio.h>
#include <ctype.h>
#include <string.h>
+
#ifdef COMPILER_GCC
#include <strings.h> /* strcasecmp */
#else
#define strcasecmp(s1, s2) _stricmp((s1), (s2))
#endif
+#ifdef STARDIS_ENABLE_MPI
+#include <mpi/mpi.h>
+#endif
+
/*******************************************************************************
* Local Functions
******************************************************************************/
@@ -157,7 +162,12 @@ print_version
{
ASSERT(stream);
fprintf(stream,
- "Stardis version %i.%i.%i built on stardis solver version %i.%i.%i\n",
+ "Stardis version %i.%i.%i built on stardis solver version %i.%i.%i; MPI is "
+#ifdef STARDIS_ENABLE_MPI
+ "enabled.\n",
+#else
+ "disabled.\n",
+#endif
STARDIS_APP_VERSION_MAJOR, STARDIS_APP_VERSION_MINOR, STARDIS_APP_VERSION_PATCH,
Stardis_VERSION_MAJOR, Stardis_VERSION_MINOR, Stardis_VERSION_PATCH);
}
@@ -212,8 +222,17 @@ short_help
const char* prog)
{
const char* name;
+#ifdef STARDIS_ENABLE_MPI
+ int rank;
+#endif
ASSERT(stream && prog);
+#ifdef STARDIS_ENABLE_MPI
+ CHK(MPI_Comm_rank(MPI_COMM_WORLD, &rank) == MPI_SUCCESS);
+ /* Only master prints */
+ if(rank != 0) return;
+#endif
+
#ifdef COMPILER_GCC
name = strrchr(prog, '/');
#else
diff --git a/src/stardis-compute.c b/src/stardis-compute.c
@@ -375,40 +375,44 @@ error:
}
#define READ_RANDOM_STATE(Name) \
- if(!str_is_empty(Name)) { \
- const char* name = str_cget(Name); \
- stream = fopen(name, "r"); \
- if(!stream) { \
- res = RES_IO_ERR; \
- logger_print(stardis->logger, LOG_ERROR, \
- "Could not open generator's state file ('%s').\n", \
- name); \
- goto error; \
+ if(!stardis->mpi_initialized || stardis->mpi_rank == 0) { \
+ if(!str_is_empty(Name)) { \
+ const char* name = str_cget(Name); \
+ stream = fopen(name, "r"); \
+ if(!stream) { \
+ res = RES_IO_ERR; \
+ logger_print(stardis->logger, LOG_ERROR, \
+ "Could not open generator's state file ('%s').\n", \
+ name); \
+ goto error; \
+ } \
+ ERR(ssp_rng_create(stardis->allocator, SSP_RNG_MT19937_64, &args.rng_state)); \
+ res = read_random_generator_state(args.rng_state, stream); \
+ if(res != RES_OK) { \
+ logger_print(stardis->logger, LOG_ERROR, \
+ "Could not read random generator's state ('%s').\n", \
+ name); \
+ goto error; \
+ } \
+ fclose(stream); stream = NULL; \
} \
- ERR(ssp_rng_create(stardis->allocator, SSP_RNG_MT19937_64, &args.rng_state)); \
- res = read_random_generator_state(args.rng_state, stream); \
- if(res != RES_OK) { \
- logger_print(stardis->logger, LOG_ERROR, \
- "Could not read random generator's state ('%s').\n", \
- name); \
- goto error; \
- } \
- fclose(stream); stream = NULL; \
}
#define WRITE_RANDOM_STATE(Name) \
- if(!str_is_empty(Name)) { \
- const char* name = str_cget(Name); \
- stream = fopen(name, "wb"); \
- if(!stream) { \
- res = RES_IO_ERR; \
- logger_print(stardis->logger, LOG_ERROR, \
- "Could not write random generator's state ('%s').\n", \
- name); \
- goto error; \
+ if(!stardis->mpi_initialized || stardis->mpi_rank == 0) { \
+ if(!str_is_empty(Name)) { \
+ const char* name = str_cget(Name); \
+ stream = fopen(name, "wb"); \
+ if(!stream) { \
+ res = RES_IO_ERR; \
+ logger_print(stardis->logger, LOG_ERROR, \
+ "Could not write random generator's state ('%s').\n", \
+ name); \
+ goto error; \
+ } \
+ ERR(write_random_generator_state(estimator, stream)); \
+ fclose(stream); stream = NULL; \
} \
- ERR(write_random_generator_state(estimator, stream)); \
- fclose(stream); stream = NULL; \
}
static res_T
@@ -436,51 +440,59 @@ compute_probe(struct stardis* stardis, struct time* start)
READ_RANDOM_STATE(&stardis->rndgen_state_in_filename);
if(stardis->mode & (MODE_BIN_GREEN | MODE_GREEN)) {
- time_current(&compute_start);
- ERR(sdis_solve_probe_green_function(stardis->sdis_scn, &args, &green));
- time_current(&compute_end);
- if(stardis->mode & MODE_BIN_GREEN) {
- struct time output_end;
- ASSERT(!str_is_empty(&stardis->bin_green_filename));
- stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
- if(!stream) {
- res = RES_IO_ERR;
- goto error;
- }
- ERR(dump_green_bin(green, stardis, stream));
- fclose(stream); stream = NULL;
- if(str_cget(&stardis->end_paths_filename)
- && strlen(str_cget(&stardis->end_paths_filename)))
- {
- stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_probe_green_function(stardis->sdis_scn, &args, &green));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_probe_green_function(stardis->sdis_scn, &args, &green));
+ time_current(&compute_end);
+ if(stardis->mode & MODE_BIN_GREEN) {
+ struct time output_end;
+ ASSERT(!str_is_empty(&stardis->bin_green_filename));
+ stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
if(!stream) {
res = RES_IO_ERR;
goto error;
}
- ERR(dump_paths_end(green, stardis, stream));
+ ERR(dump_green_bin(green, stardis, stream));
fclose(stream); stream = NULL;
+ if(str_cget(&stardis->end_paths_filename)
+ && strlen(str_cget(&stardis->end_paths_filename)))
+ {
+ stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(!stream) {
+ res = RES_IO_ERR;
+ goto error;
+ }
+ ERR(dump_paths_end(green, stardis, stream));
+ fclose(stream); stream = NULL;
+ }
+ time_current(&output_end);
+ ERR(print_computation_time(NULL, stardis,
+ start, &compute_start, &compute_end, &output_end));
+ }
+ if(stardis->mode & MODE_GREEN) {
+ ERR(dump_green_ascii(green, stardis, stdout));
}
- time_current(&output_end);
- ERR(print_computation_time(NULL, stardis,
- start, &compute_start, &compute_end, &output_end));
- }
- if(stardis->mode & MODE_GREEN) {
- ERR(dump_green_ascii(green, stardis, stdout));
}
} else {
args.register_paths = stardis->dump_paths;
args.picard_order = stardis->picard_order;
- time_current(&compute_start);
- ERR(sdis_solve_probe(stardis->sdis_scn, &args, &estimator));
- time_current(&compute_end);
- ERR(print_computation_time(estimator, stardis,
- start, &compute_start, &compute_end, NULL));
- ERR(print_single_MC_result(estimator, stardis, stdout));
-
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_probe(stardis->sdis_scn, &args, &estimator));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_probe(stardis->sdis_scn, &args, &estimator));
+ time_current(&compute_end);
+ ERR(print_computation_time(estimator, stardis,
+ start, &compute_start, &compute_end, NULL));
+ ERR(print_single_MC_result(estimator, stardis, stdout));
+
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
}
/* Output random state? */
@@ -670,52 +682,61 @@ compute_probe_on_interface(struct stardis* stardis, struct time* start)
READ_RANDOM_STATE(&stardis->rndgen_state_in_filename);
if(stardis->mode & (MODE_BIN_GREEN | MODE_GREEN)) {
- time_current(&compute_start);
- ERR(sdis_solve_probe_boundary_green_function(stardis->sdis_scn, &args,
- &green));
- time_current(&compute_end);
- if(stardis->mode & MODE_BIN_GREEN) {
- struct time output_end;
- ASSERT(!str_is_empty(&stardis->bin_green_filename));
- stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
- if(!stream) {
- res = RES_IO_ERR;
- goto error;
- }
- ERR(dump_green_bin(green, stardis, stream));
- fclose(stream); stream = NULL;
- if(str_cget(&stardis->end_paths_filename)
- && strlen(str_cget(&stardis->end_paths_filename)))
- {
- stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_probe_boundary_green_function(stardis->sdis_scn, &args,
+ &green));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_probe_boundary_green_function(stardis->sdis_scn, &args,
+ &green));
+ time_current(&compute_end);
+ if(stardis->mode & MODE_BIN_GREEN) {
+ struct time output_end;
+ ASSERT(!str_is_empty(&stardis->bin_green_filename));
+ stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
if(!stream) {
res = RES_IO_ERR;
goto error;
}
- ERR(dump_paths_end(green, stardis, stream));
+ ERR(dump_green_bin(green, stardis, stream));
fclose(stream); stream = NULL;
+ if(str_cget(&stardis->end_paths_filename)
+ && strlen(str_cget(&stardis->end_paths_filename)))
+ {
+ stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(!stream) {
+ res = RES_IO_ERR;
+ goto error;
+ }
+ ERR(dump_paths_end(green, stardis, stream));
+ fclose(stream); stream = NULL;
+ }
+ time_current(&output_end);
+ ERR(print_computation_time(NULL, stardis,
+ start, &compute_start, &compute_end, &output_end));
+ }
+ if(stardis->mode & MODE_GREEN) {
+ ERR(dump_green_ascii(green, stardis, stdout));
}
- time_current(&output_end);
- ERR(print_computation_time(NULL, stardis,
- start, &compute_start, &compute_end, &output_end));
- }
- if(stardis->mode & MODE_GREEN) {
- ERR(dump_green_ascii(green, stardis, stdout));
}
} else {
args.register_paths = stardis->dump_paths;
args.picard_order = stardis->picard_order;
- time_current(&compute_start);
- ERR(sdis_solve_probe_boundary(stardis->sdis_scn, &args, &estimator));
- time_current(&compute_end);
- ERR(print_computation_time(estimator, stardis,
- start, &compute_start, &compute_end, NULL));
- ERR(print_single_MC_result(estimator, stardis, stdout));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_probe_boundary(stardis->sdis_scn, &args, &estimator));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_probe_boundary(stardis->sdis_scn, &args, &estimator));
+ time_current(&compute_end);
+ ERR(print_computation_time(estimator, stardis,
+ start, &compute_start, &compute_end, NULL));
+ ERR(print_single_MC_result(estimator, stardis, stdout));
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
}
/* Output random state? */
@@ -856,42 +877,46 @@ compute_camera(struct stardis* stardis, struct time* start)
args.picard_order = stardis->picard_order;
/* Launch the simulation */
- time_current(&compute_start);
- ERR(sdis_solve_camera(stardis->sdis_scn, &args, &buf));
- time_current(&compute_end);
-
- /* Write the image */
- if(str_is_empty(&stardis->camera.file_name))
- stream = stdout;
- else {
- stream = fopen(str_cget(&stardis->camera.file_name), "w");
- if(!stream) {
- res = RES_IO_ERR;
- goto error;
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_camera(stardis->sdis_scn, &args, &buf));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_camera(stardis->sdis_scn, &args, &buf));
+ time_current(&compute_end);
+
+ /* Write the image */
+ if(str_is_empty(&stardis->camera.file_name))
+ stream = stdout;
+ else {
+ stream = fopen(str_cget(&stardis->camera.file_name), "w");
+ if(!stream) {
+ res = RES_IO_ERR;
+ goto error;
+ }
}
- }
- ASSERT(stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_VTK
- || stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_HT);
- if(stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_VTK)
- ERR(dump_vtk_image(buf, stream));
- else ERR(dump_ht_image(buf, stream));
- if(!str_is_empty(&stardis->camera.file_name))
- fclose(stream);
-
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_buffer_get_definition(buf, definition));
- FOR_EACH(iy, 0, definition[1]) {
- FOR_EACH(ix, 0, definition[0]) {
- const struct sdis_estimator* estimator;
- ERR(sdis_estimator_buffer_at(buf, ix, iy, &estimator));
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ ASSERT(stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_VTK
+ || stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_HT);
+ if(stardis->camera.fmt == STARDIS_RENDERING_OUTPUT_FILE_FMT_VTK)
+ ERR(dump_vtk_image(buf, stream));
+ else ERR(dump_ht_image(buf, stream));
+ if(!str_is_empty(&stardis->camera.file_name))
+ fclose(stream);
+
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_buffer_get_definition(buf, definition));
+ FOR_EACH(iy, 0, definition[1]) {
+ FOR_EACH(ix, 0, definition[0]) {
+ const struct sdis_estimator* estimator;
+ ERR(sdis_estimator_buffer_at(buf, ix, iy, &estimator));
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
}
+ time_current(&output_end);
+ ERR(print_computation_time(NULL, stardis,
+ start, &compute_start, &compute_end, NULL));
}
- time_current(&output_end);
- ERR(print_computation_time(NULL, stardis,
- start, &compute_start, &compute_end, NULL));
end:
if(cam) SDIS(camera_ref_put(cam));
@@ -933,50 +958,58 @@ compute_medium(struct stardis* stardis, struct time* start)
READ_RANDOM_STATE(&stardis->rndgen_state_in_filename);
if(stardis->mode & (MODE_BIN_GREEN | MODE_GREEN)) {
- time_current(&compute_start);
- ERR(sdis_solve_medium_green_function(stardis->sdis_scn, &args, &green));
- time_current(&compute_end);
- if(stardis->mode & MODE_BIN_GREEN) {
- struct time output_end;
- ASSERT(!str_is_empty(&stardis->bin_green_filename));
- stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
- if(!stream) {
- res = RES_IO_ERR;
- goto error;
- }
- ERR(dump_green_bin(green, stardis, stream));
- fclose(stream); stream = NULL;
- if(str_cget(&stardis->end_paths_filename)
- && strlen(str_cget(&stardis->end_paths_filename)))
- {
- stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_medium_green_function(stardis->sdis_scn, &args, &green));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_medium_green_function(stardis->sdis_scn, &args, &green));
+ time_current(&compute_end);
+ if(stardis->mode & MODE_BIN_GREEN) {
+ struct time output_end;
+ ASSERT(!str_is_empty(&stardis->bin_green_filename));
+ stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
if(!stream) {
res = RES_IO_ERR;
goto error;
}
- ERR(dump_paths_end(green, stardis, stream));
+ ERR(dump_green_bin(green, stardis, stream));
fclose(stream); stream = NULL;
+ if(str_cget(&stardis->end_paths_filename)
+ && strlen(str_cget(&stardis->end_paths_filename)))
+ {
+ stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(!stream) {
+ res = RES_IO_ERR;
+ goto error;
+ }
+ ERR(dump_paths_end(green, stardis, stream));
+ fclose(stream); stream = NULL;
+ }
+ time_current(&output_end);
+ ERR(print_computation_time(NULL, stardis,
+ start, &compute_start, &compute_end, &output_end));
+ }
+ if(stardis->mode & MODE_GREEN) {
+ ERR(dump_green_ascii(green, stardis, stdout));
}
- time_current(&output_end);
- ERR(print_computation_time(NULL, stardis,
- start, &compute_start, &compute_end, &output_end));
- }
- if(stardis->mode & MODE_GREEN) {
- ERR(dump_green_ascii(green, stardis, stdout));
}
} else {
args.register_paths = stardis->dump_paths;
args.picard_order = stardis->picard_order;
- time_current(&compute_start);
- ERR(sdis_solve_medium(stardis->sdis_scn, &args, &estimator));
- time_current(&compute_end);
- ERR(print_computation_time(estimator, stardis,
- start, &compute_start, &compute_end, NULL));
- ERR(print_single_MC_result(estimator, stardis, stdout));
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_medium(stardis->sdis_scn, &args, &estimator));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_medium(stardis->sdis_scn, &args, &estimator));
+ time_current(&compute_end);
+ ERR(print_computation_time(estimator, stardis,
+ start, &compute_start, &compute_end, NULL));
+ ERR(print_single_MC_result(estimator, stardis, stdout));
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
}
/* Output random state? */
@@ -1065,50 +1098,58 @@ compute_boundary(struct stardis* stardis, struct time* start)
READ_RANDOM_STATE(&stardis->rndgen_state_in_filename);
if(stardis->mode & (MODE_BIN_GREEN | MODE_GREEN)) {
- time_current(&compute_start);
- ERR(sdis_solve_boundary_green_function(stardis->sdis_scn, &args, &green));
- time_current(&compute_end);
- if(stardis->mode & MODE_BIN_GREEN) {
- struct time output_end;
- ASSERT(!str_is_empty(&stardis->bin_green_filename));
- stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
- if(!stream) {
- res = RES_IO_ERR;
- goto error;
- }
- ERR(dump_green_bin(green, stardis, stream));
- fclose(stream); stream = NULL;
- if(str_cget(&stardis->end_paths_filename)
- && strlen(str_cget(&stardis->end_paths_filename)))
- {
- stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_boundary_green_function(stardis->sdis_scn, &args, &green));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_boundary_green_function(stardis->sdis_scn, &args, &green));
+ time_current(&compute_end);
+ if(stardis->mode & MODE_BIN_GREEN) {
+ struct time output_end;
+ ASSERT(!str_is_empty(&stardis->bin_green_filename));
+ stream = fopen(str_cget(&stardis->bin_green_filename), "wb");
if(!stream) {
res = RES_IO_ERR;
goto error;
}
- ERR(dump_paths_end(green, stardis, stream));
+ ERR(dump_green_bin(green, stardis, stream));
fclose(stream); stream = NULL;
+ if(str_cget(&stardis->end_paths_filename)
+ && strlen(str_cget(&stardis->end_paths_filename)))
+ {
+ stream = fopen(str_cget(&stardis->end_paths_filename), "w");
+ if(!stream) {
+ res = RES_IO_ERR;
+ goto error;
+ }
+ ERR(dump_paths_end(green, stardis, stream));
+ fclose(stream); stream = NULL;
+ }
+ time_current(&output_end);
+ ERR(print_computation_time(NULL, stardis,
+ start, &compute_start, &compute_end, &output_end));
+ }
+ if(stardis->mode & MODE_GREEN) {
+ ERR(dump_green_ascii(green, stardis, stdout));
}
- time_current(&output_end);
- ERR(print_computation_time(NULL, stardis,
- start, &compute_start, &compute_end, &output_end));
- }
- if(stardis->mode & MODE_GREEN) {
- ERR(dump_green_ascii(green, stardis, stdout));
}
} else {
args.register_paths = stardis->dump_paths;
args.picard_order = stardis->picard_order;
- time_current(&compute_start);
- ERR(sdis_solve_boundary(stardis->sdis_scn, &args, &estimator));
- time_current(&compute_end);
- ERR(print_computation_time(estimator, stardis,
- start, &compute_start, &compute_end, NULL));
- ERR(print_single_MC_result(estimator, stardis, stdout));
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_boundary(stardis->sdis_scn, &args, &estimator));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_boundary(stardis->sdis_scn, &args, &estimator));
+ time_current(&compute_end);
+ ERR(print_computation_time(estimator, stardis,
+ start, &compute_start, &compute_end, NULL));
+ ERR(print_single_MC_result(estimator, stardis, stdout));
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
}
/* Output random state? */
@@ -1149,17 +1190,21 @@ compute_flux_boundary(struct stardis* stardis, struct time* start)
/* Input random state? */
READ_RANDOM_STATE(&stardis->rndgen_state_in_filename);
- time_current(&compute_start);
- ERR(sdis_solve_boundary_flux(stardis->sdis_scn, &args, &estimator));
- time_current(&compute_end);
- ERR(print_computation_time(estimator, stardis,
- start, &compute_start, &compute_end, NULL));
- ERR(print_single_MC_result(estimator, stardis, stdout));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(sdis_solve_boundary_flux(stardis->sdis_scn, &args, &estimator));
+ } else {
+ time_current(&compute_start);
+ ERR(sdis_solve_boundary_flux(stardis->sdis_scn, &args, &estimator));
+ time_current(&compute_end);
+ ERR(print_computation_time(estimator, stardis,
+ start, &compute_start, &compute_end, NULL));
+ ERR(print_single_MC_result(estimator, stardis, stdout));
- /* Dump recorded paths according to user settings */
- dump_ctx.stardis = stardis;
- dump_ctx.rank = 0;
- ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ /* Dump recorded paths according to user settings */
+ dump_ctx.stardis = stardis;
+ dump_ctx.rank = 0;
+ ERR(sdis_estimator_for_each_path(estimator, dump_path, &dump_ctx));
+ }
/* Output random state? */
WRITE_RANDOM_STATE(&stardis->rndgen_state_out_filename);
@@ -1210,7 +1255,9 @@ compute_map(struct stardis* stardis, struct time* start)
ERR(sdis_solve_boundary(stardis->sdis_scn, &args,
darray_estimators_data_get(&estimators) + p));
}
- ERR(dump_map(stardis, &estimators, stdout));
+ if(stardis->mpi_initialized && stardis->mpi_rank != 0) {
+ ERR(dump_map(stardis, &estimators, stdout));
+ }
end:
if(estimators_initialized) {
diff --git a/src/stardis-main.c b/src/stardis-main.c
@@ -26,6 +26,10 @@
#include <stdlib.h>
#include <stdio.h>
+#ifdef STARDIS_ENABLE_MPI
+#include <mpi.h>
+#endif
+
int
main
(int argc,
@@ -44,12 +48,16 @@ main
time_current(&start);
+#ifdef STARDIS_ENABLE_MPI
+ ERR(init_mpi(&argc, &argv, log_err_fn, log_warn_fn));
+#endif
+
ERR(mem_init_proxy_allocator(&allocator, &mem_default_allocator));
allocator_initialized = 1;
ERR(logger_init(&allocator, &logger));
logger_initialized = 1;
- /* Active loggin for args pasing */
+ /* Active loggin for args parsing */
logger_set_stream(&logger, LOG_ERROR, log_err_fn, NULL);
logger_set_stream(&logger, LOG_WARNING, log_warn_fn, NULL);
logger_set_stream(&logger, LOG_OUTPUT, log_prt_fn, NULL);
@@ -119,6 +127,10 @@ exit:
}
mem_shutdown_proxy_allocator(&allocator);
}
+#ifdef STARDIS_ENABLE_MPI
+ finalize_mpi(&stardis);
+#endif
+
return err;
error:
if(mode & COMPUTE_MODES)
diff --git a/src/stardis-output.c b/src/stardis-output.c
@@ -1500,6 +1500,9 @@ print_computation_time
ASSERT(stardis && start && compute_start && compute_end);
+ /* Only master prints or reads estimators */
+ ASSERT(!stardis->mpi_initialized || stardis->mpi_rank == 0);
+
time_sub(&tmp, compute_start, start);
time_dump(&tmp, flag, NULL, buf, sizeof(buf));
logger_print(stardis->logger, LOG_OUTPUT,
@@ -1540,6 +1543,9 @@ print_single_MC_result
ASSERT(estimator && stardis && stream);
+ /* Only master prints or reads estimators */
+ ASSERT(!stardis->mpi_initialized || stardis->mpi_rank == 0);
+
/* Fetch the estimation data */
ERR(sdis_estimator_get_temperature(estimator, &result));
ERR(sdis_estimator_get_failure_count(estimator, &nfailures_));
@@ -1743,6 +1749,9 @@ dump_map
ASSERT(stardis && estimators && stream);
+ /* Only master prints or reads estimators */
+ ASSERT(!stardis->mpi_initialized || stardis->mpi_rank == 0);
+
est = darray_estimators_cdata_get(estimators);
idx = darray_size_t_cdata_get(&stardis->compute_surface.primitives);
sz = darray_size_t_size_get(&stardis->compute_surface.primitives);
@@ -1840,6 +1849,10 @@ dump_compute_region_at_the_end_of_vtk
unsigned tsz, i;
size_t j, psz;
ASSERT(stardis && stream);
+
+ /* Only master prints or reads estimators */
+ ASSERT(!stardis->mpi_initialized || stardis->mpi_rank == 0);
+
psz = darray_size_t_size_get(&stardis->compute_surface.primitives);
ASSERT(psz == darray_sides_size_get(&stardis->compute_surface.sides));
@@ -1925,6 +1938,9 @@ dump_model_as_c_chunks
ASSERT(stardis && stream);
+ /* Only master prints or reads estimators */
+ ASSERT(!stardis->mpi_initialized || stardis->mpi_rank == 0);
+
prefix = str_cget(&stardis->chunks_prefix);
ERR(sg3d_geometry_get_unique_vertices_count(stardis->geometry.sg3d, &vcount));
ERR(sg3d_geometry_get_unique_triangles_count(stardis->geometry.sg3d, &tcount));
