From db42526f644d2862da1e7e9e856ed50d99cc468d Mon Sep 17 00:00:00 2001 From: Alex Lindsay Date: Thu, 5 Nov 2020 09:20:33 -0800 Subject: [PATCH] Speed up nonsparse AD initial setup significantly We have been setting `ADReal::do_derivatives=true` by default and only toggling to `false` during residual computation. This can be very bad for non-sparse calculations. For instance in a navier-stokes input file supplied by Gavin Ridley, I saw 10 seconds spent in initial condition computation and 9 seconds in `ComputeMaterialsObjectThread` for a total of 19 seconds in `FEProblemBase::initialSetup`. This was 27% of the total computation time! With the changes here `FEProblemBase::initialSetup` no longer even appears in the graph, which is how it should be. I recall that I was forced to do derivative calculations by default for the reasons stated in the comment I'm deleting. My memory says some phase field object was initializing its material properties in its constructor, or more likely during `initialSetup`, so we had to enable derivative calculations then or else its results would be garbage. Hopefully, however, if there are objects still doing things like that we can do a little more fine-grained control to make their objects work without significantly sabotaging everyone's performance. Refs #14701 --- framework/src/base/MooseApp.C | 2 ++ framework/src/problems/FEProblemBase.C | 14 ++++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/framework/src/base/MooseApp.C b/framework/src/base/MooseApp.C index cc84d2b8d636..f55610c1d64c 100644 --- a/framework/src/base/MooseApp.C +++ b/framework/src/base/MooseApp.C @@ -352,6 +352,8 @@ MooseApp::MooseApp(InputParameters parameters) } #endif + ADReal::do_derivatives = false; + Registry::addKnownLabel(_type); Moose::registerAll(_factory, _action_factory, _syntax); diff --git a/framework/src/problems/FEProblemBase.C b/framework/src/problems/FEProblemBase.C index 79ddbcd048d1..e983b5fe7484 100644 --- a/framework/src/problems/FEProblemBase.C +++ b/framework/src/problems/FEProblemBase.C @@ -349,11 +349,6 @@ FEProblemBase::FEProblemBase(const InputParameters & parameters) _num_grid_steps(0), _displaced_neighbor_ref_pts("invert_elem_phys use_undisplaced_ref unset", "unset") { - // Initialize static do_derivatives member. We initialize this to true so that all the default AD - // things that we setup early in the simulation actually get their derivative vectors initalized. - // We will toggle this to false when doing residual evaluations - ADReal::do_derivatives = true; - _time = 0.0; _time_old = 0.0; _t_step = 0; @@ -5156,12 +5151,7 @@ FEProblemBase::computeResidualSys(NonlinearImplicitSystem & /*sys*/, NumericVector & residual) { TIME_SECTION(_compute_residual_sys_timer); - - ADReal::do_derivatives = false; - computeResidual(soln, residual); - - ADReal::do_derivatives = true; } void @@ -5449,6 +5439,8 @@ FEProblemBase::computeJacobianInternal(const NumericVector & soln, void FEProblemBase::computeJacobianTags(const std::set & tags) { + ADReal::do_derivatives = true; + if (!_has_jacobian || !_const_jacobian) { TIME_SECTION(_compute_jacobian_tags_timer); @@ -5538,6 +5530,8 @@ FEProblemBase::computeJacobianTags(const std::set & tags) _displaced_problem->setCurrentlyComputingJacobian(false); _safe_access_tagged_matrices = true; } + + ADReal::do_derivatives = false; } void