CvodeKokkosSurfaceSolver.hpp

Go to the documentation of this file.

 
#ifndef OPENSWMM_ENGINE_2D_GPU_CVODE_KOKKOS_SURFACE_SOLVER_HPP
#define OPENSWMM_ENGINE_2D_GPU_CVODE_KOKKOS_SURFACE_SOLVER_HPP
 
#include "../solver/ISurfaceSolver.hpp"
#include "KokkosSurfaceKernels.hpp"
 
#include <nvector/nvector_kokkos.hpp>
#include <sundials/sundials_context.h>
#include <sundials/sundials_linearsolver.h>
#include <sundials/sundials_types.h>
 
#include <memory>
 
namespace openswmm::twoD {
 
struct MeshData;
struct SurfaceStateData;
struct SolverOptions2D;
 
namespace gpu {

using VecType = ::sundials::kokkos::Vector<ExecSpace>;
 
#if defined(OPENSWMM_HAVE_HYPRE)
class KokkosAmgPreconditioner;  // complete type only in the .cpp (guarded)
#endif

struct KokkosSolverContext {
    MeshViews*  mesh        = nullptr;
    StateViews* state       = nullptr;
    double      dry_depth   = 1.0e-3;
    double      limiter_eps = 1.0e-6;
    double      flux_dh_eps = 4.0e-3;   
    bool        prec_diag_built = false; 
#if defined(OPENSWMM_HAVE_HYPRE)
    bool                     use_amg = false;   
    KokkosAmgPreconditioner* amg     = nullptr; 
#endif
};
 
class CvodeKokkosSurfaceSolver final : public ISurfaceSolver {
public:
    // Out-of-line (see CvodeSurfaceSolver) so the unique_ptr AMG member's
    // destructor is instantiated in the .cpp where the type is complete.
    CvodeKokkosSurfaceSolver();
    ~CvodeKokkosSurfaceSolver() override;
 
    CvodeKokkosSurfaceSolver(const CvodeKokkosSurfaceSolver&)            = delete;
    CvodeKokkosSurfaceSolver& operator=(const CvodeKokkosSurfaceSolver&) = delete;
 
    void   initialize(MeshData& mesh, SurfaceStateData& state,
                      SolverOptions2D& opts) override;
    double advance(double t_current, double t_target) override;
    void   reinitialize(double t0) override;
    void   finalize() override;
 
    long   last_num_steps() const noexcept override { return last_nsteps_; }
    double last_step_size() const noexcept override { return last_h_; }
    bool   is_initialized() const noexcept override { return cvode_mem_ != nullptr; }
 
private:
    // SUNDIALS handles.
    SUNContext      sun_ctx_   = nullptr;
    void*           cvode_mem_ = nullptr;
    SUNLinearSolver ls_        = nullptr;
    std::unique_ptr<VecType> y_;        
    std::unique_ptr<VecType> abstol_;   
 
    // Device mirrors.
    MeshViews  mesh_v_;
    StateViews state_v_;
 
    // Host back-pointers (owned by SurfaceRouter2D; must outlive the solver).
    MeshData*         mesh_host_  = nullptr;
    SurfaceStateData* state_host_ = nullptr;
    SolverOptions2D*  opts_host_  = nullptr;
 
    KokkosSolverContext ctx_;
 
#if defined(OPENSWMM_HAVE_HYPRE)
    std::unique_ptr<KokkosAmgPreconditioner> amg_precond_;
#endif
 
    int  nt_ = 0;
    int  nv_ = 0;
    long last_nsteps_ = 0;
    double last_h_    = 0.0;
 
    // Host<->device transfer helpers (no-ops in Phase 1 OpenMP host space).
    void uploadMesh();
    void uploadSources();          
    void uploadBoundaryStatic();   
    void uploadBoundaryDynamic();  
    void seedVolumeFromHead();     
    void downloadState();          
 
    // CVODE C-ABI callbacks.
    static int rhs_cb(sunrealtype t, N_Vector y, N_Vector ydot, void* user_data);
    static int psetup_cb(sunrealtype t, N_Vector y, N_Vector fy,
                         sunbooleantype jok, sunbooleantype* jcurPtr,
                         sunrealtype gamma, void* user_data);
    static int psolve_cb(sunrealtype t, N_Vector y, N_Vector fy,
                         N_Vector r, N_Vector z,
                         sunrealtype gamma, sunrealtype delta, int lr,
                         void* user_data);
};
 
} // namespace gpu
} // namespace openswmm::twoD
 
#endif // OPENSWMM_ENGINE_2D_GPU_CVODE_KOKKOS_SURFACE_SOLVER_HPP