Merge branch 'sbrus89/ocn/subgrid-openacc-fix' (PR #6472)

Fix OpenACC rountine issue for subgrid wetting and drying

This PR moves a subgrid subroutine call out of a OpenACC parallel region
to fix the compile problems noted in #6470. Since subgrid wetting and
drying is strictly a MPAS-Ocean standalone feature, it should be fine
for this code to remain CPU-only.

This PR also fixes a couple issues in mpas_ocn_vmix.F:
* An OpenACC bug related to the use of gang vector collapse(3) on a
  double nested loop with variable inner loop bounds.
* A missing !$omp parallel region in a calculation for the
  config_use_gotm option.

[BFB] -- mpas-ocean standalone only

components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F

@@ -886,9 +886,15 @@ subroutine ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, &
          ! Use centered (mean) thickness as flux value
 
          if ( config_use_subgrid_wetting_drying ) then
+#ifdef MPAS_OPENACC
+         !$acc update host(ssh)
+#endif
 
             call ocn_subgrid_layerThickEdgeFlux_center(ssh, layerThickEdgeFlux)
 
+#ifdef MPAS_OPENACC
+         !$acc update device(layerThickEdgeFlux)
+#endif
          else
 
 #ifdef MPAS_OPENACC
@@ -915,9 +921,16 @@ subroutine ocn_diagnostic_solve_layerThicknessEdge(normalVelocity, &
          ! Use upwind thickness as the edge flux value
 
          if (config_use_subgrid_wetting_drying) then
+#ifdef MPAS_OPENACC
+         !$acc update host(ssh, normalVelocity, layerThickness)
+#endif
 
             call ocn_subgrid_layerThickEdgeFlux_upwind(ssh, normalVelocity, layerThickness, layerThickEdgeFlux)
 
+#ifdef MPAS_OPENACC
+         !$acc update device(layerThickEdgeFlux)
+#endif
+
          else
 #ifdef MPAS_OPENACC
             !$acc parallel loop &
@@ -2621,28 +2634,35 @@ subroutine ocn_diagnostic_solve_z_coordinates(layerThickness, zMid, zTop, ssh)!{
               + layerThickness(k  ,iCell)
          end do
 
-         if (config_use_subgrid_wetting_drying) then
-            call ocn_subgrid_ssh_lookup(layerThickness(1,iCell), &
-                                        subgridWetVolumeCellTable(:,iCell), &
-                                        subgridSshCellTableRange(:,iCell), &
-                                        bottomDepth(iCell), &
-                                        subgridCellBathymetryMin(iCell), & 
-                                        ssh(iCell))
-
-
-            zTop(1,iCell) = ssh(iCell)
-            zMid(1,iCell) = -bottomDepth(iCell) + 0.5_RKIND*layerThickness(1,iCell)         
-         else
-            ! copy zTop(1,iCell) into sea-surface height array
-            ssh(iCell) = zTop(minLevelCell(iCell),iCell)
-         end if
+         ! copy zTop(1,iCell) into sea-surface height array
+         ssh(iCell) = zTop(minLevelCell(iCell),iCell)
 
       end do
 #ifndef MPAS_OPENACC
       !$omp end do
       !$omp end parallel
 #endif
 
+      if (config_use_subgrid_wetting_drying) then
+#ifdef MPAS_OPENACC
+      !$acc update host(layerThickness)
+#endif
+         do iCell = 1, nCells
+               call ocn_subgrid_ssh_lookup(layerThickness(1,iCell), &
+                                           subgridWetVolumeCellTable(:,iCell), &
+                                           subgridSshCellTableRange(:,iCell), &
+                                           bottomDepth(iCell), &
+                                           subgridCellBathymetryMin(iCell), & 
+                                           ssh(iCell))
+
+               zTop(1,iCell) = ssh(iCell)
+               zMid(1,iCell) = -bottomDepth(iCell) + 0.5_RKIND*layerThickness(1,iCell)         
+         end do
+#ifdef MPAS_OPENACC
+      !$acc update device(zTop, zMid, ssh)
+#endif
+      end if
+
    end subroutine ocn_diagnostic_solve_z_coordinates!}}}
 
 !***********************************************************************

components/mpas-ocean/src/shared/mpas_ocn_vmix.F

@@ -1462,6 +1462,7 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool,
          !$acc    vertViscTopOfCell, vertViscTopOfEdge) &
          !$acc    private(cell1, cell2, k, Nsurf, N)
 #else
+         !$omp parallel
          !$omp do schedule(runtime) private(cell1, cell2, k, Nsurf, N)
 #endif
          do iEdge=1,nEdgesOwned
@@ -1480,6 +1481,7 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool,
          end do
 #ifndef MPAS_OPENACC
          !$omp end do
+         !$omp end parallel
 #endif
          call mpas_timer_stop('GOTM avg')
       endif
@@ -1588,7 +1590,7 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool,
 
 
 #ifdef MPAS_OPENACC
-                  !$acc parallel loop gang vector collapse(3) present(vertDiffPassiveTopOfCell, vertDiffTopOfCell)
+                  !$acc parallel loop present(vertDiffPassiveTopOfCell, vertDiffTopOfCell)
 #else
                   !$omp parallel
                   !$omp do schedule(runtime) private(k, iTracer)