Merge pull request #8 from IskanderI/petsc-stages

add more PETSc logging stages

Project.toml

@@ -1,7 +1,7 @@
 name = "LaMEM_C"
 uuid = "29b6043d-3311-4c16-b469-b02bf85cb680"
 authors = ["Anton Popov <[email protected]>", "Boris Kaus <[email protected]>"]
-version = "2.1.1"
+version = "2.1.2"
 
 [deps]
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"

src/LaMEM.cpp

@@ -22,7 +22,6 @@ int main(int argc, char **argv)
 
 	// Initialize PETSC
 	ierr = PetscInitialize(&argc,&argv,(char *)0, help); CHKERRQ(ierr);
-
 	ModParam IOparam;
 	char      str[_str_len_];
 
@@ -45,7 +44,7 @@ int main(int argc, char **argv)
 	if(IOparam.use == 0)
 	{
 		// Forward simulation	
-		ierr = LaMEMLibMain(NULL); CHKERRQ(ierr);
+		ierr = LaMEMLibMain(NULL,IOparam.stages); CHKERRQ(ierr);
 	}
 	else
 	{

src/LaMEM.h

@@ -189,7 +189,7 @@ typedef pair <PetscInt,    PetscInt> ipair;
 
 // LaMEM library main function
 
-PetscErrorCode LaMEMLibMain(void *param);
+PetscErrorCode LaMEMLibMain(void *param,PetscLogStage stages[4]);
 
 //-----------------------------------------------------------------------------
 #endif

src/LaMEMLib.cpp

@@ -42,7 +42,7 @@
 #include "passive_tracer.h"
 
 //---------------------------------------------------------------------------
-PetscErrorCode LaMEMLibMain(void *param)
+PetscErrorCode LaMEMLibMain(void *param,PetscLogStage stages[4])
 {
 	LaMEMLib       lm;
 	RunMode        mode;
@@ -60,7 +60,7 @@ PetscErrorCode LaMEMLibMain(void *param)
 	PetscPrintf(PETSC_COMM_WORLD,"-------------------------------------------------------------------------- \n");
 	PetscPrintf(PETSC_COMM_WORLD,"                   Lithosphere and Mantle Evolution Model                   \n");
 	PetscPrintf(PETSC_COMM_WORLD,"     Compiled: Date: %s - Time: %s 	    \n",__DATE__,__TIME__ );
-	PetscPrintf(PETSC_COMM_WORLD,"     Version : 2.1.0 \n");
+	PetscPrintf(PETSC_COMM_WORLD,"     Version : 2.1.2 \n");
 	PetscPrintf(PETSC_COMM_WORLD,"-------------------------------------------------------------------------- \n");
 	PetscPrintf(PETSC_COMM_WORLD,"        STAGGERED-GRID FINITE DIFFERENCE CANONICAL IMPLEMENTATION           \n");
 	PetscPrintf(PETSC_COMM_WORLD,"-------------------------------------------------------------------------- \n");
@@ -130,7 +130,7 @@ PetscErrorCode LaMEMLibMain(void *param)
 	else if(mode == _NORMAL_ || mode == _RESTART_)
 	{
 		// solve coupled nonlinear equations
-		ierr = LaMEMLibSolve(&lm, param); CHKERRQ(ierr);
+		ierr = LaMEMLibSolve(&lm, param,stages); CHKERRQ(ierr);
 	}
 
 	// destroy library objects
@@ -604,7 +604,7 @@ PetscErrorCode LaMEMLibSaveOutput(LaMEMLib *lm)
 	PetscFunctionReturn(0);
 }
 //---------------------------------------------------------------------------
-PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
+PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param, PetscLogStage stages[4])
 {
 	PMat           pm;     // preconditioner matrix    (to be removed!)
 	PCStokes       pc;     // Stokes preconditioner    (to be removed!)
@@ -614,6 +614,7 @@ PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
 	PetscInt       restart;
 	PetscLogDouble t;
 
+
 	PetscErrorCode ierr;
 	PetscFunctionBeginUser;
 
@@ -649,17 +650,23 @@ PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
 		// initialize boundary constraint vectors
 		ierr = BCApply(&lm->bc); CHKERRQ(ierr);
 
+		PetscCall(PetscLogStagePush(stages[0])); /* Start profiling stage*/
+
 		// initialize temperature
 		ierr = JacResInitTemp(&lm->jr); CHKERRQ(ierr);
 
+		PetscCall(PetscLogStagePop()); /* Stop profiling stage*/
 		// compute elastic parameters
 		ierr = JacResGetI2Gdt(&lm->jr); CHKERRQ(ierr);
 
 		// solve nonlinear equation system with SNES
 		PetscTime(&t);
 
+		PetscCall(PetscLogStagePush(stages[1])); /* Start profiling stage*/
+
 		ierr = SNESSolve(snes, NULL, lm->jr.gsol); CHKERRQ(ierr);
 
+		PetscCall(PetscLogStagePop()); /* Stop profiling stage*/
 		// print analyze convergence/divergence reason & iteration count
 		ierr = SNESPrintConvergedReason(snes, t); CHKERRQ(ierr);
 
@@ -686,6 +693,8 @@ PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
 		// MARKER & FREE SURFACE ADVECTION + EROSION
 		//==========================================
 
+		PetscCall(PetscLogStagePush(stages[2])); /* Start profiling stage*/
+
 		// calculate current time step
 		ierr = ADVSelectTimeStep(&lm->actx, &restart); CHKERRQ(ierr);
 
@@ -707,6 +716,8 @@ PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
 		// Advect Passive tracers
 		ierr = ADVAdvectPassiveTracer(&lm->actx); CHKERRQ(ierr);
 
+		PetscCall(PetscLogStagePop()); /* Stop profiling stage*/
+
 		// apply erosion to the free surface
 		ierr = FreeSurfAppErosion(&lm->surf); CHKERRQ(ierr);
 
@@ -725,10 +736,14 @@ PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param)
 
 		// update time stamp and counter
 		ierr = TSSolStepForward(&lm->ts); CHKERRQ(ierr);
-
+
+		PetscCall(PetscLogStagePush(stages[3])); /* Start profiling stage*/
+
 		// grid & marker output
 		ierr = LaMEMLibSaveOutput(lm); CHKERRQ(ierr);
 
+		PetscCall(PetscLogStagePop()); /* Stop profiling stage*/
+
 		// restart database
 		ierr = LaMEMLibSaveRestart(lm); CHKERRQ(ierr);
 

src/LaMEMLib.h

@@ -68,7 +68,7 @@ PetscErrorCode LaMEMLibSetLinks(LaMEMLib *lm);
 
 PetscErrorCode LaMEMLibSaveOutput(LaMEMLib *lm, PetscInt dirInd);
 
-PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param);
+PetscErrorCode LaMEMLibSolve(LaMEMLib *lm, void *param, PetscLogStage stages[4]);
 
 PetscErrorCode LaMEMLibDryRun(LaMEMLib *lm);
 

src/adjoint.cpp

@@ -984,7 +984,7 @@ PetscErrorCode LaMEMAdjointMain(ModParam *IOparam)
 		// Only compute a forward model and the corresponding misfit 
 
 		IOparam->BruteForce_FD = PETSC_TRUE;		// to return early w/out cmomputing FD gradients
-		ierr = LaMEMLibMain(IOparam); 															CHKERRQ(ierr);
+		ierr = LaMEMLibMain(IOparam,IOparam->stages); 															CHKERRQ(ierr);
 
 		// Print overview of cost function & gradients 
 		ierr = PrintCostFunction(IOparam);					CHKERRQ(ierr);
@@ -1079,7 +1079,7 @@ PetscErrorCode LaMEMAdjointMain(ModParam *IOparam)
  	else if(IOparam->use == _syntheticforwardrun_)
  	{
  		// call LaMEM main library function
- 		ierr = LaMEMLibMain(IOparam); CHKERRQ(ierr);
+ 		ierr = LaMEMLibMain(IOparam,IOparam->stages); CHKERRQ(ierr);
 
  		// Save output
  		PetscViewer     viewerVel;
@@ -1212,7 +1212,7 @@ PetscErrorCode ComputeGradientsAndObjectiveFunction(Vec Parameters, PetscScalar
 
 	// Adjoint gradients: Call LaMEM main library function once (computes gradients @ the end)
  	IOparam->BruteForce_FD = PETSC_FALSE;
-	ierr = LaMEMLibMain(IOparam); 															CHKERRQ(ierr);
+	ierr = LaMEMLibMain(IOparam,IOparam->stages); 															CHKERRQ(ierr);
 
 	// Print overview of cost function & gradients 
 	ierr = PrintCostFunction(IOparam);					CHKERRQ(ierr);
@@ -1827,7 +1827,7 @@ PetscErrorCode AdjointFiniteDifferenceGradients(ModParam *IOparam)
 	if (FD_Adjoint){
 
 		// Call LaMEM
-		ierr 		= 	LaMEMLibMain(IOparam); CHKERRQ(ierr);		// call LaMEM
+		ierr 		= 	LaMEMLibMain(IOparam,IOparam->stages); CHKERRQ(ierr);		// call LaMEM
 		Misfit_ref	=	IOparam->mfit;
 
 		PetscPrintf(PETSC_COMM_WORLD,"| ************************************************************************ \n");
@@ -1863,7 +1863,7 @@ PetscErrorCode AdjointFiniteDifferenceGradients(ModParam *IOparam)
 				ierr		=	CopyParameterToLaMEMCommandLine(IOparam, CurVal + Perturb, j);		CHKERRQ(ierr);
 
 				// Compute solution with updated parameter
-				ierr 		= 	LaMEMLibMain(IOparam); 												CHKERRQ(ierr);
+				ierr 		= 	LaMEMLibMain(IOparam,IOparam->stages); 												CHKERRQ(ierr);
 				Misfit_pert = 	IOparam->mfit;
 
 				// FD gradient

src/objFunct.h

@@ -115,6 +115,8 @@ struct ModParam
 	PetscScalar      Avel_num[_MAX_OBS_];             	// Numerically computed velocity at the comparison points
 	PetscBool        Apoint_on_proc[_MAX_OBS_];         // Is the observation point on the current processor or not (simplified printing)?
 	char   			 ScalLawFilename[_str_len_];		// Name of scaling law file
+
+	PetscLogStage	 stages[4]; 						/* Create stages for PETSC profiling */
 };
 
 // observation type