HyPar  1.0
Finite-Difference Hyperbolic-Parabolic PDE Solver on Cartesian Grids
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
main.cpp
Go to the documentation of this file.
1 
146 #include <stdio.h>
147 #include <stdlib.h>
148 #include <sys/time.h>
149 #include <string>
150 
151 #ifdef with_petsc
152 #include <petscinterface.h>
153 #endif
154 
155 #ifdef with_python
156 #include <Python.h>
157 #ifdef with_python_numpy
158 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
159 #include <numpy/arrayobject.h>
160 #endif
161 #endif
162 
163 #include <mpivars_cpp.h>
164 #include <simulation_library.h>
165 
166 static const char help[] = "HyPar - A finite-difference algorithm for solving hyperbolic-parabolic PDEs";
167 
168 #ifdef with_python
169 static void initializePython(int);
170 static void initializePythonPlotting(int);
171 #endif
172 
178 int main(int argc, char **argv)
179 {
180  int ierr = 0, d, n;
181  struct timeval main_start, solve_start;
182  struct timeval main_end , solve_end ;
183 #ifdef with_petsc
184  PetscBool use_petscts;
185 #endif
186  int use_petsc = 0;
187 
188 #ifdef serial
189  int world = 0;
190  int rank = 0;
191  int nproc = 1;
192  printf("HyPar - Serial Version\n");
193 #else
194  MPI_Comm world;
195  int rank, nproc;
196  MPI_Init(&argc,&argv);
197  MPI_Comm_dup(MPI_COMM_WORLD, &world);
198  MPI_Comm_rank(MPI_COMM_WORLD,&rank );
199  MPI_Comm_size(MPI_COMM_WORLD,&nproc);
200  if (!rank) printf("HyPar - Parallel (MPI) version with %d processes\n",nproc);
201 #endif
202 
203 #ifdef with_petsc
204  PetscInitialize(&argc,&argv,(char*)0,help);
205  if (!rank) printf("Compiled with PETSc time integration.\n");
206 #endif
207 
208 #ifdef with_python
209  initializePython(rank);
211 #endif
212 
213  gettimeofday(&main_start,NULL);
214 
215  int sim_type = -1;
216  Simulation *sim = NULL;
217 
218  if (!rank) {
219 
220  std::string ensemble_sim_fname(_ENSEMBLE_SIM_INP_FNAME_);
221  std::string sparsegrids_sim_fname(_SPARSEGRIDS_SIM_INP_FNAME_);
222 
223  FILE *f_ensemble_sim = fopen(ensemble_sim_fname.c_str(), "r");
224  FILE *f_sparsegrids_sim = fopen(sparsegrids_sim_fname.c_str(), "r");
225 
226  if (f_ensemble_sim && f_sparsegrids_sim) {
227 
228  fprintf(stderr,"Error: Cannot have both %s and %s input files.\n",
230  fprintf(stderr, "Remove one or both of them depending on the kind of simulation you want to run.\n");
231  fclose(f_ensemble_sim);
232  fclose(f_sparsegrids_sim);
233 
234  } else if (f_ensemble_sim) {
235 
236  sim_type = _SIM_TYPE_ENSEMBLE_;
237  fclose(f_ensemble_sim);
238 
239  } else if (f_sparsegrids_sim) {
240 
241  sim_type = _SIM_TYPE_SPARSE_GRIDS_;
242  fclose(f_sparsegrids_sim);
243 
244  } else {
245 
246  sim_type = _SIM_TYPE_SINGLE_;
247 
248  }
249 
250  }
251 
252 #ifndef serial
253  MPI_Bcast(&sim_type, 1, MPI_INT, 0, MPI_COMM_WORLD);
254 #endif
255 
256  if (sim_type == _SIM_TYPE_SINGLE_) {
257  sim = new SingleSimulation;
258  } else if (sim_type == _SIM_TYPE_ENSEMBLE_) {
259  if (!rank) printf("-- Ensemble Simulation --\n");
260  sim = new EnsembleSimulation;
261  } else if (sim_type == _SIM_TYPE_SPARSE_GRIDS_) {
262  if (!rank) printf("-- Sparse Grids Simulation --\n");
263  sim = new SparseGridsSimulation;
264  } else {
265  fprintf(stderr, "ERROR: invalid sim_type (%d) on rank %d.\n",
266  sim_type, rank);
267  }
268 
269  if (sim == NULL) {
270  fprintf(stderr, "ERROR: unable to create sim on rank %d.\n",
271  rank );
272  return 1;
273  }
274 
275  /* Allocate simulation objects */
276  ierr = sim->define(rank, nproc);
277  if (!sim->isDefined()) {
278  printf("Error: Simulation::define() failed on rank %d\n",
279  rank);
280  return 1;
281  }
282  if (ierr) {
283  printf("Error: Simulation::define() returned with status %d on process %d.\n",
284  ierr, rank);
285  return(ierr);
286  }
287 
288 #ifndef serial
289  ierr = sim->mpiCommDup();
290 #endif
291 
292 #ifdef with_petsc
293  use_petscts = PETSC_FALSE; /* default value */
294  ierr = PetscOptionsGetBool( nullptr,nullptr,
295  "-use-petscts",
296  &use_petscts,
297  nullptr); CHKERRQ(ierr);
298  if (use_petscts == PETSC_TRUE) use_petsc = 1;
299  sim->usePetscTS(use_petscts);
300 #endif
301 
302  /* Read Inputs */
303  ierr = sim->ReadInputs();
304  if (ierr) {
305  printf("Error: Simulation::ReadInputs() returned with status %d on process %d.\n",ierr,rank);
306  return(ierr);
307  }
308 
309  /* Initialize and allocate arrays */
310  ierr = sim->Initialize();
311  if (ierr) {
312  printf("Error: Simulation::Initialize() returned with status %d on process %d.\n",ierr,rank);
313  return(ierr);
314  }
315 
316  /* read and set grid & initial solution */
317  ierr = sim->InitialSolution();
318  if (ierr) {
319  printf("Error: Simulation::InitialSolution() returned with status %d on process %d.\n",ierr,rank);
320  return(ierr);
321  }
322 
323  /* Initialize domain boundaries */
324  ierr = sim->InitializeBoundaries();
325  if (ierr) {
326  printf("Error: Simulation::InitializeBoundaries() returned with status %d on process %d.\n",ierr,rank);
327  return(ierr);
328  }
329 
330  /* Initialize immersed boundaries */
331  ierr = sim->InitializeImmersedBoundaries();
332  if (ierr) {
333  printf("Error: Simulation::InitializeImmersedBoundaries() returned with status %d on process %d.\n",ierr,rank);
334  return(ierr);
335  }
336 
337  /* Initialize solvers */
338  ierr = sim->InitializeSolvers();
339  if (ierr) {
340  printf("Error: Simulation::InitializeSolvers() returned with status %d on process %d.\n",ierr,rank);
341  return(ierr);
342  }
343 
344  /* Initialize physics */
345  ierr = sim->InitializePhysics();
346  if (ierr) {
347  printf("Error: Simulation::InitializePhysics() returned with status %d on process %d.\n",ierr,rank);
348  return(ierr);
349  }
350 
351  /* Initialize physics data */
352  ierr = sim->InitializePhysicsData();
353  if (ierr) {
354  printf("Error: Simulation::InitializePhysicsData() returned with status %d on process %d.\n",ierr,rank);
355  return(ierr);
356  }
357 
358  /* Wrap up initializations */
359  ierr = sim->InitializationWrapup();
360  if (ierr) {
361  printf("Error: Simulation::InitializationWrapup() returned with status %d on process %d.\n",ierr,rank);
362  return(ierr);
363  }
364 
365  /* Initializations complete */
366 
367  /* Run the solver */
368 #ifndef serial
369  MPI_Barrier(MPI_COMM_WORLD);
370 #endif
371  gettimeofday(&solve_start,NULL);
372 #ifdef with_petsc
373  if (use_petsc == 1) {
374  /* Use PETSc time-integration */
375  ierr = sim->SolvePETSc();
376  if (ierr) {
377  printf("Error: Simulation::SolvePETSc() returned with status %d on process %d.\n",ierr,rank);
378  return(ierr);
379  }
380  } else {
381  /* Use native time-integration */
382  ierr = sim->Solve();
383  if (ierr) {
384  printf("Error: Simulation::Solve() returned with status %d on process %d.\n",ierr,rank);
385  return(ierr);
386  }
387  }
388 #else
389  /* Use native time-integration */
390  ierr = sim->Solve();
391  if (ierr) {
392  printf("Error: Simulation::Solve() returned with status %d on process %d.\n",ierr,rank);
393  return(ierr);
394  }
395 #endif
396  gettimeofday(&solve_end,NULL);
397 #ifndef serial
398  MPI_Barrier(MPI_COMM_WORLD);
399 #endif
400  gettimeofday(&main_end,NULL);
401 
402  /* calculate solver and total runtimes */
403  long long walltime;
404  walltime = ( (main_end.tv_sec * 1000000 + main_end.tv_usec )
405  - (main_start.tv_sec * 1000000 + main_start.tv_usec));
406  double main_runtime = (double) walltime / 1000000.0;
407  ierr = MPIMax_double(&main_runtime,&main_runtime,1,&world); if(ierr) return(ierr);
408  walltime = ( (solve_end.tv_sec * 1000000 + solve_end.tv_usec )
409  - (solve_start.tv_sec * 1000000 + solve_start.tv_usec));
410  double solver_runtime = (double) walltime / 1000000.0;
411  ierr = MPIMax_double(&solver_runtime,&solver_runtime,1,&world); if(ierr) return(ierr);
412 
413  /* Write errors and other data */
414  sim->WriteErrors(solver_runtime, main_runtime);
415 
416  /* Cleaning up */
417  delete sim;
418  if (!rank) printf("Finished.\n");
419 
420 #ifdef with_python
421  Py_Finalize();
422 #endif
423 
424 #ifdef with_petsc
425  PetscFinalize();
426 #endif
427 
428 #ifndef serial
429  MPI_Comm_free(&world);
430  MPI_Finalize();
431 #endif
432 
433  return(0);
434 }
435 
436 #ifdef with_python
437 
438 void initializePython(int a_rank )
439 {
440  Py_Initialize();
441  if (!a_rank) printf("Initialized Python.\n");
442  PyRun_SimpleString("import os");
443  PyRun_SimpleString("hypar_dir = os.environ.get('HYPAR_DIR')");
444 #ifndef serial
445  MPI_Barrier(MPI_COMM_WORLD);
446 #endif
447  return;
448 }
449 
451 void initializePythonPlotting(int a_rank )
452 {
453  /* load plotting tools and scripts */
454  PyRun_SimpleString("hypar_dir_plt_py = hypar_dir + '/src/PlottingFunctions'");
455  PyRun_SimpleString("import sys");
456  PyRun_SimpleString("sys.path.append(hypar_dir_plt_py)");
457  if (!a_rank) {
458  PyRun_SimpleString
459  ("print('Added plotting script directory (%s) to Python path.' % hypar_dir_plt_py)");
460  }
461 #ifndef serial
462  MPI_Barrier(MPI_COMM_WORLD);
463 #endif
464  return;
465 }
466 
467 #endif
static void initializePython(int)
Definition: main.cpp:438
Class describing sparse grids simulations.
static const char help[]
Definition: main.cpp:166
#define _SPARSEGRIDS_SIM_INP_FNAME_
virtual int InitializePhysicsData()=0
virtual int mpiCommDup()=0
#define _SIM_TYPE_SINGLE_
virtual void usePetscTS(PetscBool)=0
Class describing a single simulation.
virtual int Solve()=0
Class describing ensemble simulations.
int MPIMax_double(double *, double *, int, void *)
Definition: MPIMax.c:38
#define _SIM_TYPE_SPARSE_GRIDS_
virtual int Initialize()=0
virtual void WriteErrors(double, double)=0
#define _SIM_TYPE_ENSEMBLE_
virtual int SolvePETSc()=0
virtual bool isDefined() const =0
#define _ENSEMBLE_SIM_INP_FNAME_
virtual int InitializationWrapup()
Definition: simulation.h:83
virtual int InitialSolution()=0
virtual int define(int, int)=0
static void initializePythonPlotting(int)
Definition: main.cpp:451
virtual int InitializePhysics()=0
virtual int InitializeImmersedBoundaries()=0
C++ declarations for MPI-related functions.
virtual int InitializeBoundaries()=0
virtual int ReadInputs()=0
virtual int InitializeSolvers()=0
int main(int argc, char **argv)
Main driver.
Definition: main.cpp:178
Base class for a simulation.
Definition: simulation.h:46