Actual source code: cupmsolverinterface.hpp
1: #ifndef PETSCCUPMSOLVERINTERFACE_HPP
2: #define PETSCCUPMSOLVERINTERFACE_HPP
4: #if defined(__cplusplus)
5: #include <petsc/private/cupmblasinterface.hpp>
6: #include <petsc/private/petscadvancedmacros.h>
8: namespace Petsc
9: {
11: namespace device
12: {
14: namespace cupm
15: {
17: namespace impl
18: {
20: #define PetscCallCUPMSOLVER(...) \
21: do { \
22: const cupmSolverError_t cupmsolver_stat_p_ = __VA_ARGS__; \
23: if (PetscUnlikely(cupmsolver_stat_p_ != CUPMSOLVER_STATUS_SUCCESS)) { \
24: if (((cupmsolver_stat_p_ == CUPMSOLVER_STATUS_NOT_INITIALIZED) || (cupmsolver_stat_p_ == CUPMSOLVER_STATUS_ALLOC_FAILED) || (cupmsolver_stat_p_ == CUPMSOLVER_STATUS_INTERNAL_ERROR)) && PetscDeviceInitialized(PETSC_DEVICE_CUPM())) { \
25: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_GPU_RESOURCE, \
26: "%s error %d (%s). " \
27: "This indicates the GPU may have run out resources", \
28: cupmSolverName(), static_cast<PetscErrorCode>(cupmsolver_stat_p_), cupmSolverGetErrorName(cupmsolver_stat_p_)); \
29: } \
30: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_GPU, "%s error %d (%s)", cupmSolverName(), static_cast<PetscErrorCode>(cupmsolver_stat_p_), cupmSolverGetErrorName(cupmsolver_stat_p_)); \
31: } \
32: } while (0)
34: #ifndef PetscConcat3
35: #define PetscConcat3(a, b, c) PetscConcat(PetscConcat(a, b), c)
36: #endif
38: #if PetscDefined(USE_COMPLEX)
39: #define PETSC_CUPMSOLVER_FP_TYPE_SPECIAL un
40: #else
41: #define PETSC_CUPMSOLVER_FP_TYPE_SPECIAL or
42: #endif // USE_COMPLEX
44: #define PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupm_name, their_prefix, fp_type, suffix) PETSC_CUPM_ALIAS_FUNCTION(cupm_name, PetscConcat3(their_prefix, fp_type, suffix))
46: template <DeviceType>
47: struct SolverInterfaceImpl;
49: #if PetscDefined(HAVE_CUDA)
50: template <>
51: struct SolverInterfaceImpl<DeviceType::CUDA> : BlasInterface<DeviceType::CUDA> {
52: // typedefs
53: using cupmSolverHandle_t = cusolverDnHandle_t;
54: using cupmSolverError_t = cusolverStatus_t;
55: using cupmSolverFillMode_t = cublasFillMode_t;
56: using cupmSolverOperation_t = cublasOperation_t;
58: // error codes
59: static const auto CUPMSOLVER_STATUS_SUCCESS = CUSOLVER_STATUS_SUCCESS;
60: static const auto CUPMSOLVER_STATUS_NOT_INITIALIZED = CUSOLVER_STATUS_NOT_INITIALIZED;
61: static const auto CUPMSOLVER_STATUS_ALLOC_FAILED = CUSOLVER_STATUS_ALLOC_FAILED;
62: static const auto CUPMSOLVER_STATUS_INTERNAL_ERROR = CUSOLVER_STATUS_INTERNAL_ERROR;
64: // enums
65: // Why do these exist just to alias the CUBLAS versions? Because AMD -- in their boundless
66: // wisdom -- decided to do so for hipSOLVER...
67: // https://github.com/ROCmSoftwarePlatform/hipSOLVER/blob/develop/library/include/internal/hipsolver-types.h
68: static const auto CUPMSOLVER_OP_T = CUBLAS_OP_T;
69: static const auto CUPMSOLVER_OP_N = CUBLAS_OP_N;
70: static const auto CUPMSOLVER_OP_C = CUBLAS_OP_C;
71: static const auto CUPMSOLVER_FILL_MODE_LOWER = CUBLAS_FILL_MODE_LOWER;
72: static const auto CUPMSOLVER_FILL_MODE_UPPER = CUBLAS_FILL_MODE_UPPER;
73: static const auto CUPMSOLVER_SIDE_LEFT = CUBLAS_SIDE_LEFT;
74: static const auto CUPMSOLVER_SIDE_RIGHT = CUBLAS_SIDE_RIGHT;
76: // utility functions
77: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverCreate, cusolverDnCreate)
78: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverDestroy, cusolverDnDestroy)
79: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverSetStream, cusolverDnSetStream)
80: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverGetStream, cusolverDnGetStream)
82: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrf_bufferSize, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, potrf_bufferSize)
83: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrf, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, potrf)
85: using cupmBlasInt_t = typename BlasInterface<DeviceType::CUDA>::cupmBlasInt_t;
86: using cupmScalar_t = typename Interface<DeviceType::CUDA>::cupmScalar_t;
88: // to match hipSOLVER version (rocm 5.4.3, CUDA 12.0.1):
89: //
90: // hipsolverStatus_t hipsolverDpotrs_bufferSize(
91: // hipsolverHandle_t handle, hipsolverFillMode_t uplo, int n, int nrhs, double *A, int lda,
92: // double *B, int ldb, int *lwork
93: // )
94: //
95: // hipsolverStatus_t hipsolverDpotrs(
96: // hipsolverHandle_t handle, hipsolverFillMode_t uplo, int n, int nrhs, double *A, int lda,
97: // double *B, int ldb, double *work, int lwork, int *devInfo
98: // )
99: PETSC_NODISCARD static cupmSolverError_t cupmSolverXpotrs_bufferSize(cupmSolverHandle_t /* handle */, cupmSolverFillMode_t /* uplo */, cupmBlasInt_t /* n */, cupmBlasInt_t /* nrhs */, cupmScalar_t * /* A */, cupmBlasInt_t /* lda */, cupmScalar_t * /* B */, cupmBlasInt_t /* ldb */, cupmBlasInt_t *lwork) noexcept
100: {
101: *lwork = 0;
102: return CUPMSOLVER_STATUS_SUCCESS;
103: }
105: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrs_p, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, potrs)
107: PETSC_NODISCARD static cupmSolverError_t cupmSolverXpotrs(cupmSolverHandle_t handle, cupmSolverFillMode_t uplo, cupmBlasInt_t n, cupmBlasInt_t nrhs, const cupmScalar_t *A, cupmBlasInt_t lda, cupmScalar_t *B, cupmBlasInt_t ldb, cupmScalar_t * /* work */, cupmBlasInt_t /* lwork */, cupmBlasInt_t *dev_info) noexcept
108: {
109: return cupmSolverXpotrs_p(handle, uplo, n, nrhs, A, lda, B, ldb, dev_info);
110: }
112: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotri_bufferSize, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, potri_bufferSize)
113: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotri, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, potri)
115: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXsytrf_bufferSize, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, sytrf_bufferSize)
116: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXsytrf, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, sytrf)
118: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrf_bufferSize, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, getrf_bufferSize)
119: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrf_p, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, getrf)
120: // to match hipSOLVER version (rocm 5.4.3, CUDA 12.0.1):
121: //
122: // hipsolverStatus_t hipsolverDgetrf(
123: // hipsolverHandle_t handle, int m, int n, double *A, int lda, double *work, int lwork,
124: // int *devIpiv, int *devInfo
125: // )
126: PETSC_NODISCARD static cupmSolverError_t cupmSolverXgetrf(cupmSolverHandle_t handle, cupmBlasInt_t m, cupmBlasInt_t n, cupmScalar_t *A, cupmBlasInt_t lda, cupmScalar_t *work, cupmBlasInt_t /* lwork */, cupmBlasInt_t *dev_ipiv, cupmBlasInt_t *dev_info) noexcept
127: {
128: return cupmSolverXgetrf_p(handle, m, n, A, lda, work, dev_ipiv, dev_info);
129: }
131: // to match hipSOLVER version (rocm 5.4.3, CUDA 12.0.1):
132: //
133: // hipsolverStatus_t hipsolverDgetrs_bufferSize(
134: // hipsolverHandle_t handle, hipsolverOperation_t trans, int n, int nrhs, double *A,
135: // int lda, int *devIpiv, double *B, int ldb, int *lwork
136: // )
137: //
138: // hipsolverStatus_t hipsolverDgetrs(
139: // hipsolverHandle_t handle, hipsolverOperation_t trans, int n, int nrhs, double *A,
140: // int lda, int *devIpiv, double *B, int ldb, double *work, int lwork, int *devInfo
141: // )
142: PETSC_NODISCARD static cupmSolverError_t cupmSolverXgetrs_bufferSize(cupmSolverHandle_t /* handle */, cupmSolverOperation_t /* op */, cupmBlasInt_t /* n */, cupmBlasInt_t /* nrhs */, cupmScalar_t * /* A */, cupmBlasInt_t /* lda */, cupmBlasInt_t * /* devIpiv */, cupmScalar_t * /* B */, cupmBlasInt_t /* ldb */, cupmBlasInt_t *lwork) noexcept
143: {
144: *lwork = 0;
145: return CUPMSOLVER_STATUS_SUCCESS;
146: }
148: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrs_p, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, getrs)
150: PETSC_NODISCARD static cupmSolverError_t cupmSolverXgetrs(cupmSolverHandle_t handle, cupmSolverOperation_t op, cupmBlasInt_t n, cupmBlasInt_t nrhs, cupmScalar_t *A, cupmBlasInt_t lda, cupmBlasInt_t *dev_ipiv, cupmScalar_t *B, cupmBlasInt_t ldb, cupmScalar_t * /* work */, cupmBlasInt_t /* lwork */, cupmBlasInt_t *dev_info) noexcept
151: {
152: return cupmSolverXgetrs_p(handle, op, n, nrhs, A, lda, dev_ipiv, B, ldb, dev_info);
153: }
155: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgeqrf_bufferSize, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, geqrf_bufferSize)
156: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgeqrf, cusolverDn, PETSC_CUPMBLAS_FP_TYPE_U, geqrf)
158: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXormqr_bufferSize, cusolverDn, PetscConcat(PETSC_CUPMBLAS_FP_TYPE_U, PETSC_CUPMSOLVER_FP_TYPE_SPECIAL), mqr_bufferSize)
159: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXormqr, cusolverDn, PetscConcat(PETSC_CUPMBLAS_FP_TYPE_U, PETSC_CUPMSOLVER_FP_TYPE_SPECIAL), mqr)
161: PETSC_NODISCARD static const char *cupmSolverGetErrorName(cupmSolverError_t status) noexcept { return PetscCUSolverGetErrorName(status); }
162: };
163: #endif
165: #if PetscDefined(HAVE_HIP)
166: template <>
167: struct SolverInterfaceImpl<DeviceType::HIP> : BlasInterface<DeviceType::HIP> {
168: // typedefs
169: using cupmSolverHandle_t = hipsolverHandle_t;
170: using cupmSolverError_t = hipsolverStatus_t;
171: using cupmSolverFillMode_t = hipsolverFillMode_t;
172: using cupmSolverOperation_t = hipsolverOperation_t;
174: // error codes
175: static const auto CUPMSOLVER_STATUS_SUCCESS = HIPSOLVER_STATUS_SUCCESS;
176: static const auto CUPMSOLVER_STATUS_NOT_INITIALIZED = HIPSOLVER_STATUS_NOT_INITIALIZED;
177: static const auto CUPMSOLVER_STATUS_ALLOC_FAILED = HIPSOLVER_STATUS_ALLOC_FAILED;
178: static const auto CUPMSOLVER_STATUS_INTERNAL_ERROR = HIPSOLVER_STATUS_INTERNAL_ERROR;
180: // enums
181: static const auto CUPMSOLVER_OP_T = HIPSOLVER_OP_T;
182: static const auto CUPMSOLVER_OP_N = HIPSOLVER_OP_N;
183: static const auto CUPMSOLVER_OP_C = HIPSOLVER_OP_C;
184: static const auto CUPMSOLVER_FILL_MODE_LOWER = HIPSOLVER_FILL_MODE_LOWER;
185: static const auto CUPMSOLVER_FILL_MODE_UPPER = HIPSOLVER_FILL_MODE_UPPER;
186: static const auto CUPMSOLVER_SIDE_LEFT = HIPSOLVER_SIDE_LEFT;
187: static const auto CUPMSOLVER_SIDE_RIGHT = HIPSOLVER_SIDE_RIGHT;
189: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverCreate, hipsolverCreate)
190: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverDestroy, hipsolverDestroy)
191: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverSetStream, hipsolverSetStream)
192: PETSC_CUPM_ALIAS_FUNCTION(cupmSolverGetStream, hipsolverGetStream)
194: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrf_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potrf_bufferSize)
195: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrf, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potrf)
197: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrs_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potrs_bufferSize)
198: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotrs, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potrs)
200: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotri_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potri_bufferSize)
201: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXpotri, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, potri)
203: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXsytrf_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, sytrf_bufferSize)
204: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXsytrf, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, sytrf)
206: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrf_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, getrf_bufferSize)
207: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrf, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, getrf)
209: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrs_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, getrs_bufferSize)
210: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgetrs, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, getrs)
212: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgeqrf_bufferSize, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, geqrf_bufferSize)
213: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXgeqrf, hipsolver, PETSC_CUPMBLAS_FP_TYPE_U, geqrf)
215: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXormqr_bufferSize, hipsolver, PetscConcat(PETSC_CUPMBLAS_FP_TYPE_U, PETSC_CUPMSOLVER_FP_TYPE_SPECIAL), mqr_bufferSize)
216: PETSC_CUPMSOLVER_ALIAS_BLAS_FUNCTION(cupmSolverXormqr, hipsolver, PetscConcat(PETSC_CUPMBLAS_FP_TYPE_U, PETSC_CUPMSOLVER_FP_TYPE_SPECIAL), mqr)
218: PETSC_NODISCARD static const char *cupmSolverGetErrorName(cupmSolverError_t status) noexcept { return PetscHIPSolverGetErrorName(status); }
219: };
220: #endif
222: #define PETSC_CUPMSOLVER_IMPL_CLASS_HEADER(T) \
223: PETSC_CUPMBLAS_INHERIT_INTERFACE_TYPEDEFS_USING(T); \
224: /* introspection */ \
225: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverGetErrorName; \
226: /* types */ \
227: using cupmSolverHandle_t = typename ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverHandle_t; \
228: using cupmSolverError_t = typename ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverError_t; \
229: using cupmSolverFillMode_t = typename ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverFillMode_t; \
230: using cupmSolverOperation_t = typename ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverOperation_t; \
231: /* error codes */ \
232: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_STATUS_SUCCESS; \
233: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_STATUS_NOT_INITIALIZED; \
234: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_STATUS_ALLOC_FAILED; \
235: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_STATUS_INTERNAL_ERROR; \
236: /* values */ \
237: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_OP_T; \
238: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_OP_N; \
239: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_OP_C; \
240: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_FILL_MODE_LOWER; \
241: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_FILL_MODE_UPPER; \
242: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_SIDE_LEFT; \
243: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::CUPMSOLVER_SIDE_RIGHT; \
244: /* utility functions */ \
245: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverCreate; \
246: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverDestroy; \
247: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverGetStream; \
248: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverSetStream; \
249: /* blas functions */ \
250: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotrf_bufferSize; \
251: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotrf; \
252: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotrs_bufferSize; \
253: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotrs; \
254: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotri_bufferSize; \
255: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXpotri; \
256: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXsytrf_bufferSize; \
257: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXsytrf; \
258: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgetrf_bufferSize; \
259: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgetrf; \
260: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgetrs_bufferSize; \
261: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgetrs; \
262: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgeqrf_bufferSize; \
263: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXgeqrf; \
264: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXormqr_bufferSize; \
265: using ::Petsc::device::cupm::impl::SolverInterfaceImpl<T>::cupmSolverXormqr
267: template <DeviceType T>
268: struct SolverInterface : SolverInterfaceImpl<T> {
269: PETSC_NODISCARD static constexpr const char *cupmSolverName() noexcept { return T == DeviceType::CUDA ? "cusolverDn" : "hipsolver"; }
270: };
272: #define PETSC_CUPMSOLVER_INHERIT_INTERFACE_TYPEDEFS_USING(T) \
273: PETSC_CUPMSOLVER_IMPL_CLASS_HEADER(T); \
274: using ::Petsc::device::cupm::impl::SolverInterface<T>::cupmSolverName
276: } // namespace impl
278: } // namespace cupm
280: } // namespace device
282: } // namespace Petsc
284: #endif // __cplusplus
286: #endif // PETSCCUPMSOLVERINTERFACE_HPP