add make support for magma

Add support for shared builds on Windows with MSVC using CMake

.appveyor.yml

@@ -31,7 +31,7 @@ install:
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
-- cmake --build build
+- cmake --build build --target metis -j 4
 - set METIS_PATH=%cd%
 - cd ..
 
@@ -40,7 +40,7 @@ install:
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
 - cmake -H. -Bbuild
-- cmake --build build
+- cmake --build build -j 4
 - cmake --build build --target install
 - cd ../..
 
@@ -49,14 +49,20 @@ install:
 
 # MFEM
 before_build:
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
+# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON
 
 build_script:
-- cmake --build build_parallel --config Release -j 4
-- cmake --build build_serial --config Release -j 4
-- cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_parallel --config Debug -j 4
+# - cmake --build build_serial --config Release -j 4
+# - cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_shared_serial --config Release -j 4
+- cmake --build build_shared_serial --target exec --config Release -j 4
 
 after_build:
-- cd build_serial
+# - cd build_serial
+# - ctest -C Release --output-on-failure
+# - cd ..
+- cd build_shared_serial
 - ctest -C Release --output-on-failure

.github/workflows/builds-and-tests.yml

@@ -74,6 +74,10 @@ jobs:
             codecov: YES
           - os: windows-latest
             codecov: NO
+          - os: windows-latest
+            target: opt
+            mpi: par
+            config-opts: '-DBUILD_SHARED_LIBS=ON'
           - os: ubuntu-latest
             target: opt
             codecov: NO
@@ -210,7 +214,7 @@ jobs:
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.3
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       with:
@@ -222,7 +226,7 @@ jobs:
         hypre-dir: ${{ env.HYPRE_TOP_DIR }}
         metis-dir: ${{ env.METIS_TOP_DIR }}
         mfem-dir: ${{ env.MFEM_TOP_DIR }}
-        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
+        config-options: ${{ matrix.config-opts }}
         library-only: ${{ matrix.target == 'dbg' }}
 
     # Run checks (and only checks) on debug targets

.gitignore

@@ -21,6 +21,9 @@ CMakeFiles/
 # Default install location
 /mfem/
 
+# Typical build directory
+/build/
+
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp

CHANGELOG

@@ -47,6 +47,10 @@ New and updated examples and miniapps
 
 Integrations, testing and documentation
 ---------------------------------------
+- Added support for shared Windows builds with MSVC through CMake.
+  Developers note: this enhancement is facilitated by the addition of the macro
+  MFEM_EXPORT, see config.hpp for more details on its usage.
+
 - Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
   functionality instead for mesh optimization. See the mesh-optimizer miniapp.
 

CMakeLists.txt

@@ -13,6 +13,7 @@
 # Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
 # Version 3.8 or newer is required for direct CUDA support.
 cmake_minimum_required(VERSION 3.8)
+message(STATUS "CMake version: ${CMAKE_VERSION}")
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
@@ -24,8 +25,11 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
-include("${USER_CONFIG}" OPTIONAL)
+include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
+if (USER_CONFIG_LOADED)
+  set(USER_CONFIG_LOADED "LOADED")
+endif()
+message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")
 
 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -183,6 +187,23 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# Shared build on Windows
+if (WIN32 AND BUILD_SHARED_LIBS)
+  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
+  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
+      "Automatically export symbols for shared Windows DLL build")
+  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
+  # the build directory, so that the .exe files can find the .dll files at
+  # runtime:
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+  # Note: CMake tests are run from the build-tree sub-directory where the test
+  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
+  # work fine.
+  # Not strictly necessary:
+  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+endif()
+
 # AMD HIP
 if (MFEM_USE_HIP)
   if (HIP_ARCH)
@@ -194,6 +215,7 @@ if (MFEM_USE_HIP)
   endif()
   find_package(HIP REQUIRED)
   find_package(HIPSPARSE REQUIRED)
+  find_package(HIPBLAS REQUIRED)
 endif()
 
 # OpenMP
@@ -461,6 +483,11 @@ if (MFEM_USE_ENZYME)
    find_package(ENZYME REQUIRED)
 endif()
 
+# MAGMA
+if (MFEM_USE_MAGMA)
+   find_package(MAGMA REQUIRED)
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -501,7 +528,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    MPI_CXX HIP HIPSPARSE HIPBLAS MOONOLITH BLITZ ALGOIM ENZYME MAGMA)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -519,6 +546,7 @@ list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 
+message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -586,6 +614,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
+set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
   "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
   "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -757,10 +786,8 @@ endif()
 
 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
-  RENAME config.hpp)
-
-install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  ${PROJECT_SOURCE_DIR}/config/config.hpp
+  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)
 
 # Package the whole thing up nicely

config/cmake/MFEMConfig.cmake.in

@@ -61,6 +61,7 @@ set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
 set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)
+set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -39,6 +39,9 @@
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// Enable shared library build of MFEM.
+#cmakedefine MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI
@@ -190,4 +193,8 @@
 // Enable Enzyme for AD
 #cmakedefine MFEM_USE_ENZYME
 
+// Enable MAGMA linear algebra package
+#cmakedefine MFEM_USE_MAGMA
+
+
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindMAGMA.cmake

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MAGMA_FOUND
+#   - MAGMA_LIBRARIES
+#   - MAGMA_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(MAGMA MAGMA MAGMA_DIR "include" "magma_v2.h" "lib" "magma;magma_sparse"
+  "Paths to headers required by MAGMA." "Libraries required by MAGMA.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -873,7 +873,7 @@ function(mfem_export_mk_files)
       MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME MFEM_USE_MAGMA)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp

@@ -30,11 +30,30 @@
 #endif
 
 // Windows specific options
-#ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
+#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+// Macro MFEM_EXPORT: this macro is used when declaring exported global
+// variables and static class variables in public header files, e.g.:
+//    extern MFEM_EXPORT Geometry Geometries;
+//    static MFEM_EXPORT Device device_singleton;
+// In cases where a class contains multiple static variables, instead of marking
+// all such variables with MFEM_EXPORT, one can mark the class with MFEM_EXPORT,
+// e.g.:
+//    class MFEM_EXPORT MemoryManager ...
+// Note: MFEM's GitHub CI includes a shared MSVC build that will fail if a
+// variable that needs MFEM_EXPORT does not have it. However, builds with
+// optional external libraries are not tested and may require separate checks to
+// determine the necessity of MFEM_EXPORT.
+#if defined(_MSC_VER) && defined(MFEM_SHARED_BUILD)
+#ifdef mfem_EXPORTS
+#define MFEM_EXPORT __declspec(dllexport)
+#else
+#define MFEM_EXPORT __declspec(dllimport)
+#endif
+#else
+#define MFEM_EXPORT
 #endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.

config/config.hpp.in

@@ -39,6 +39,9 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// Enable shared library build of MFEM.
+// #define MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -195,4 +198,8 @@
 // Enable the Enzyme LLVM plugin
 // #define MFEM_USE_ENZYME
 
+// Enable the MAGMA package
+// #define MFEM_USE_MAGMA
+
+
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -63,6 +63,7 @@ MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
 MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@
+MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -19,6 +19,7 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()
 
 # MFEM options. Set to mimic the default "defaults.mk" file.
+option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
@@ -64,6 +65,7 @@ option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack"
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
 option(MFEM_USE_ENZYME "Enable Enzyme" OFF)
+option(MFEM_USE_MAGMA "Enable MAGMA" OFF)
 
 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -255,6 +257,8 @@ set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
     "Path to CoDiPack headers.")
 set(CODIPACK_LIBRARIES "")
 
+set(MAGMA_DIR "${MFEM_DIR}/../magma" CACHE PATH "Path to MAGMA")
+
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules

config/defaults.mk

@@ -166,6 +166,7 @@ MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
 MFEM_USE_ENZYME        = NO
+MFEM_USE_MAGMA         = NO
 
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -459,11 +460,11 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB = -lcusparse -lcublas
 
 # HIP library configuration
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
+HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse -lhipblas
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
@@ -560,6 +561,10 @@ ENZYME_VERSION ?= 14
 ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
 ENZYME_LIB = ""
 
+MAGMA_DIR ?= @MFEM_DIR@/../magma
+MAGMA_OPT = -I${MAGMA_DIR}/include/
+MAGMA_LIB = -L${MAGMA_DIR}/lib/ -lmagma -lmagma_sparse
+
 # If YES, enable some informational messages
 VERBOSE = NO
 

examples/ex25.cpp

@@ -469,7 +469,7 @@ int main(int argc, char *argv[])
 
       Operator *pc_r = nullptr;
       Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother

examples/ex30.cpp

@@ -90,7 +90,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    int nc_limit = 1;
-   int max_elems = 1e5;
+   int max_elems = 100*1000;
    double double_max_elems = double(max_elems);
    bool visualization = true;
    double osc_threshold = 1e-3;

examples/ex33.cpp

@@ -123,7 +123,7 @@ int main(int argc, char *argv[])
 
    // 2. Compute the rational expansion coefficients that define the
    //    integer-order PDEs.
-   const int power_of_laplace = floor(alpha);
+   const int power_of_laplace = (int)floor(alpha);
    double exponent_to_approximate = alpha - power_of_laplace;
    bool integer_order = false;
    // Check if alpha is an integer or not.

fem/ceed/interface/util.cpp

@@ -19,11 +19,11 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
-#ifndef _WIN32
+#if !defined(_WIN32) || !defined(_MSC_VER)
 typedef struct stat struct_stat;
 #else
 #define stat(dir, buf) _stat(dir, buf)
-#define S_ISDIR(mode) _S_IFDIR(mode)
+#define S_ISDIR(mode) (((mode) & _S_IFMT) == _S_IFDIR)
 typedef struct _stat struct_stat;
 #endif
 

fem/fe/fe_base.hpp

@@ -1154,7 +1154,7 @@ public:
    ~Poly_1D();
 };
 
-extern Poly_1D poly1d;
+extern MFEM_EXPORT Poly_1D poly1d;
 
 /// An element defined as an ND tensor product of 1D elements on a segment,
 /// square, or cube

fem/geom.hpp

@@ -29,7 +29,7 @@ namespace mfem
     Geometry::PRISM - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1),(1,0,1),(0,1,1)
     Geometry::PYRAMID - w/ vert. (0,0,0),(1,0,0),(1,1,0),(0,1,0),(0,0,1)
 */
-class Geometry
+class MFEM_EXPORT Geometry
 {
 public:
    enum Type
@@ -126,7 +126,11 @@ public:
    int NumBdr(int GeomType) { return NumBdrArray[GeomType]; }
 };
 
-template <> struct Geometry::Constants<Geometry::POINT>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::POINT>
 {
    static const int Dimension = 0;
    static const int NumVert = 1;
@@ -136,7 +140,11 @@ template <> struct Geometry::Constants<Geometry::POINT>
    static const int InvOrient[NumOrient];
 };
 
-template <> struct Geometry::Constants<Geometry::SEGMENT>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::SEGMENT>
 {
    static const int Dimension = 1;
    static const int NumVert = 2;
@@ -148,7 +156,11 @@ template <> struct Geometry::Constants<Geometry::SEGMENT>
    static const int InvOrient[NumOrient];
 };
 
-template <> struct Geometry::Constants<Geometry::TRIANGLE>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::TRIANGLE>
 {
    static const int Dimension = 2;
    static const int NumVert = 3;
@@ -174,7 +186,11 @@ template <> struct Geometry::Constants<Geometry::TRIANGLE>
    static const int InvOrient[NumOrient];
 };
 
-template <> struct Geometry::Constants<Geometry::SQUARE>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::SQUARE>
 {
    static const int Dimension = 2;
    static const int NumVert = 4;
@@ -194,7 +210,11 @@ template <> struct Geometry::Constants<Geometry::SQUARE>
    static const int InvOrient[NumOrient];
 };
 
-template <> struct Geometry::Constants<Geometry::TETRAHEDRON>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::TETRAHEDRON>
 {
    static const int Dimension = 3;
    static const int NumVert = 4;
@@ -216,7 +236,11 @@ template <> struct Geometry::Constants<Geometry::TETRAHEDRON>
    static const int InvOrient[NumOrient];
 };
 
-template <> struct Geometry::Constants<Geometry::CUBE>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::CUBE>
 {
    static const int Dimension = 3;
    static const int NumVert = 8;
@@ -234,7 +258,11 @@ template <> struct Geometry::Constants<Geometry::CUBE>
    };
 };
 
-template <> struct Geometry::Constants<Geometry::PRISM>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::PRISM>
 {
    static const int Dimension = 3;
    static const int NumVert = 6;
@@ -252,7 +280,11 @@ template <> struct Geometry::Constants<Geometry::PRISM>
    };
 };
 
-template <> struct Geometry::Constants<Geometry::PYRAMID>
+template <> struct
+/// @cond Suppress_Doxygen_warnings
+   MFEM_EXPORT
+/// @endcond
+   Geometry::Constants<Geometry::PYRAMID>
 {
    static const int Dimension = 3;
    static const int NumVert = 5;
@@ -272,7 +304,7 @@ template <> struct Geometry::Constants<Geometry::PYRAMID>
 
 // Defined in fe.cpp to ensure construction after 'mfem::TriangleFE' and
 // `mfem::TetrahedronFE`.
-extern Geometry Geometries;
+extern MFEM_EXPORT Geometry Geometries;
 
 
 class RefinedGeometry
@@ -321,7 +353,7 @@ public:
    ~GeometryRefiner();
 };
 
-extern GeometryRefiner GlobGeometryRefiner;
+extern MFEM_EXPORT GeometryRefiner GlobGeometryRefiner;
 
 }
 

fem/intrules.hpp

@@ -376,10 +376,10 @@ public:
 };
 
 /// A global object with all integration rules (defined in intrules.cpp)
-extern IntegrationRules IntRules;
+extern MFEM_EXPORT IntegrationRules IntRules;
 
 /// A global object with all refined integration rules
-extern IntegrationRules RefinedIntRules;
+extern MFEM_EXPORT IntegrationRules RefinedIntRules;
 
 }
 

general/device.hpp

@@ -125,7 +125,7 @@ private:
    enum MODES {SEQUENTIAL, ACCELERATED};
 
    static bool device_env, mem_host_env, mem_device_env, mem_types_set;
-   static Device device_singleton;
+   static MFEM_EXPORT Device device_singleton;
 
    MODES mode = Device::SEQUENTIAL;
    int dev = 0;   ///< Device ID of the configured device.

general/globals.hpp

@@ -63,12 +63,12 @@ public:
 /** @brief Global stream used by the library for standard output. Initially it
     uses the same std::streambuf as std::cout, however that can be changed.
     @sa OutStream. */
-extern OutStream out;
+extern MFEM_EXPORT OutStream out;
 /** @brief Global stream used by the library for standard error output.
     Initially it uses the same std::streambuf as std::cerr, however that can be
     changed.
     @sa OutStream. */
-extern OutStream err;
+extern MFEM_EXPORT OutStream err;
 
 
 /** @brief Construct a string of the form "<prefix><myid><suffix>" where the

general/isockstream.cpp

@@ -25,11 +25,11 @@
 #include <winsock.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
+// Link with ws2_32.lib
+#pragma comment(lib, "ws2_32.lib")
 #endif
 typedef int socklen_t;
 #define close closesocket
-// Link with ws2_32.lib
-#pragma comment(lib, "ws2_32.lib")
 #endif
 
 using namespace std;

general/mem_manager.hpp

@@ -64,7 +64,7 @@ constexpr int DeviceMemoryType = static_cast<int>(MemoryType::MANAGED);
 constexpr int DeviceMemoryTypeSize = MemoryTypeSize - DeviceMemoryType;
 
 /// Memory type names, used during Device:: configuration.
-extern const char *MemoryTypeName[MemoryTypeSize];
+extern MFEM_EXPORT const char *MemoryTypeName[MemoryTypeSize];
 
 /// Memory classes identify sets of memory types.
 /** This type is used by kernels that can work with multiple MemoryType%s.
@@ -588,7 +588,7 @@ private:
 /** The MFEM memory manager class. Host-side pointers are inserted into this
     manager which keeps track of the associated device pointer, and where the
     data currently resides. */
-class MemoryManager
+class MFEM_EXPORT MemoryManager
 {
 private:
 
@@ -1281,7 +1281,7 @@ inline int Memory<T>::CompareHostAndDevice(int size) const
 
 
 /// The (single) global memory manager object
-extern MemoryManager mm;
+extern MFEM_EXPORT MemoryManager mm;
 
 } // namespace mfem
 

general/socketstream.cpp

@@ -30,10 +30,10 @@
 #include <winsock.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
-#endif
 // Link with ws2_32.lib
 #pragma comment(lib, "ws2_32.lib")
 #endif
+#endif
 
 #ifdef MFEM_USE_GNUTLS
 #include <cstdlib>  // getenv

general/tic_toc.hpp

@@ -66,7 +66,7 @@ public:
 };
 
 
-extern StopWatch tic_toc;
+extern MFEM_EXPORT StopWatch tic_toc;
 
 /// Start the tic_toc timer
 extern void tic();

general/tinyxml2.cpp

@@ -21,6 +21,8 @@ must not be misrepresented as being the original software.
 distribution.
 */
 
+// This file was modified by the MFEM team.
+
 #include "tinyxml2.h"
 
 #include <new>		// yes, this one new style header, is in the Android SDK.
@@ -106,7 +108,7 @@ distribution.
 #elif defined(__APPLE__) || (__FreeBSD__)
 	#define TIXML_FSEEK fseeko
 	#define TIXML_FTELL ftello
-#elif defined(__unix__) && defined(__x86_64__)
+#elif defined(__unix__) && defined(__x86_64__) && !defined(__CYGWIN__)
 	#define TIXML_FSEEK fseeko64
 	#define TIXML_FTELL ftello64
 #else

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   sparsemat.cpp
   sparsesmoothers.cpp
   vector.cpp
+  libBatchSolver.cpp
   )
 
 list(APPEND HDRS
@@ -57,6 +58,7 @@ list(APPEND HDRS
   tensor.hpp
   dual.hpp
   vector.hpp
+  libBatchSolver.hpp
   )
 
 if (MFEM_USE_MPI)

linalg/libBatchSolver.cpp

@@ -0,0 +1,115 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "libBatchSolver.hpp"
+
+namespace mfem
+{
+
+void LibBatchMult::Mult(const Vector &b, Vector &x)
+{
+
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+   if (Device::Allows(Backend::DEVICE_MASK))
+   {
+#if defined(MFEM_USE_MAGMA)
+      const double alpha = 1.0, beta = 0.0;
+#else
+      const double alpha_t = 1.0, beta_t = 0.0;
+      const double* alpha = &alpha_t;
+      const double* beta = &beta_t;
+#endif
+
+#if defined(MFEM_USE_HIP) || (defined(MFEM_USE_CUDA) && CUDA_VERSION >= 11070) || defined(MFEM_USE_MAGMA)
+#if defined(MFEM_USE_MAGMA)
+      magmablas_dgemv_batched_strided(MagmaNoTrans,
+#else
+      MFEM_SUB_cu_or_hip(blasStatus_t)
+      status = MFEM_SUB_cu_or_hip(blasDgemvStridedBatched)(MFEM_SUB_Cuda_or_Hip(
+                                                              BLAS::Handle)(),
+                                                           MFEM_SUB_CU_or_HIP(BLAS_OP_N),
+#endif
+                                      mat_size,
+                                      mat_size,
+                                      alpha,
+                                      MatrixBatch.Read(),
+                                      mat_size,
+                                      mat_size * mat_size,
+                                      b.Read(),
+                                      1,
+                                      mat_size,
+                                      beta,
+                                      x.Write(),
+                                      1,
+                                      mat_size,
+                                      num_mats
+#if defined(MFEM_USE_MAGMA)
+                                      , queue
+#endif
+                                     );
+#else
+      MFEM_SUB_cu_or_hip(blasStatus_t)
+      status = MFEM_SUB_cu_or_hip(blasDgemmStridedBatched)(MFEM_SUB_Cuda_or_Hip(
+                                                              BLAS::Handle)(),
+                                                           MFEM_SUB_CU_or_HIP(BLAS_OP_N),
+                                                           MFEM_SUB_CU_or_HIP(BLAS_OP_N),
+                                                           mat_size,
+                                                           1,
+                                                           mat_size,
+                                                           alpha,
+                                                           MatrixBatch.Read(),
+                                                           mat_size,
+                                                           mat_size * mat_size,
+                                                           b.Read(),
+                                                           mat_size,
+                                                           mat_size,
+                                                           beta,
+                                                           x.Write(),
+                                                           mat_size,
+                                                           mat_size,
+                                                           num_mats);
+
+
+#endif
+
+#if !defined(MFEM_USE_MAGMA)
+      MFEM_VERIFY(status == MFEM_SUB_CU_or_HIP(BLAS_STATUS_SUCCESS),
+                  "blasDgemvStridedBatched");
+#endif
+   }
+   else
+#endif
+   {
+      const int rows = MatrixBatch.SizeI();
+      const int cols = MatrixBatch.SizeJ();
+      const int N = MatrixBatch.SizeK();
+
+      for (int e=0; e<N; ++e)
+      {
+
+         //matvec
+         for (int c=0; c<cols; ++c)
+         {
+            double dot=0.0;
+            for (int r=0; r<rows; ++r)
+            {
+               int idx = r  +  rows * e;
+               dot += MatrixBatch(c, r, e) * b(idx);
+            }
+            int idx = c + cols * e;
+            x(idx) = dot;
+         }
+      }
+   }
+}
+
+
+} //mfem namespace

linalg/libBatchSolver.hpp

@@ -0,0 +1,115 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIB_BATCH_SOLVER
+#define MFEM_LIB_BATCH_SOLVER
+
+#include "../config/config.hpp"
+#include "../general/globals.hpp"
+#include "matrix.hpp"
+#include "densemat.hpp"
+
+#if defined(MFEM_USE_CUDA)
+// These macros prefixes the argument with `cuda` or `hip`, e.g. `cudaStream_t` or `hipStream_t`
+#define MFEM_SUB_cuda_or_hip(stub) cuda##stub
+#define MFEM_SUB_cu_or_hip(stub) cu##stub
+#define MFEM_SUB_Cuda_or_Hip(stub) Cuda##stub
+#define MFEM_SUB_CUDA_or_HIP(stub) CUDA##stub
+#define MFEM_SUB_CU_or_HIP(stub) CU##stub
+
+#include <cuda.h>
+#include <cublas.h>
+#include <cusolverDn.h>
+#include <cuda_runtime.h>
+#endif  // MFEM_USE_CUDA
+
+#if defined(MFEM_USE_HIP)
+// These macros prefixes the argument with `cuda` or `hip`, e.g. `cudaStream_t` or `hipStream_t`
+#define MFEM_SUB_cu_or_hip(stub) hip##stub
+#define MFEM_SUB_cuda_or_hip(stub) hip##stub
+#define MFEM_SUB_Cuda_or_Hip(stub) Hip##stub
+#define MFEM_SUB_CUDA_or_HIP(stub) HIP##stub
+#define MFEM_SUB_CU_or_HIP(stub) HIP##stub
+#include <hipblas/hipblas.h>
+#include <hip/hip_runtime.h>
+#include <hip/hip_runtime_api.h>
+#endif  // MFEM_USE_HIP
+
+#if defined(MFEM_USE_MAGMA)
+#include <magma_v2.h>
+#endif
+
+namespace mfem
+{
+
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+class MFEM_SUB_Cuda_or_Hip(BLAS)
+{
+protected:
+   MFEM_SUB_cu_or_hip(blasHandle_t) handle = nullptr;
+
+   MFEM_SUB_Cuda_or_Hip(BLAS)()
+   {
+      MFEM_SUB_cu_or_hip(blasStatus_t) status = MFEM_SUB_cu_or_hip(blasCreate)(
+                                                   &handle);
+      MFEM_VERIFY(status == MFEM_SUB_CU_or_HIP(BLAS_STATUS_SUCCESS),
+                  "Cannot initialize BLAS.");
+   }
+
+   ~MFEM_SUB_Cuda_or_Hip(BLAS)() { MFEM_SUB_cu_or_hip(blasDestroy)(handle); }
+
+   static MFEM_SUB_Cuda_or_Hip(BLAS) & Instance()
+   {
+      static MFEM_SUB_Cuda_or_Hip(BLAS) instance;
+      return instance;
+   }
+
+public:
+   static MFEM_SUB_cu_or_hip(blasHandle_t) Handle() { return Instance().handle; }
+};
+#endif
+
+/**
+   Use GPU library calls to perform action of block diagonal matrices
+**/
+class LibBatchMult
+{
+
+private:
+   DenseTensor &MatrixBatch;
+
+   int mat_size, num_mats;
+#if defined(MFEM_USE_MAGMA)
+   magma_int_t device = 0;
+   magma_queue_t queue ;
+#endif
+public:
+   LibBatchMult() = delete;
+
+   LibBatchMult(DenseTensor &MatrixBatch_) :
+      MatrixBatch(MatrixBatch_),
+      mat_size(MatrixBatch.SizeI()),
+      num_mats(MatrixBatch.SizeK())
+   {
+#if defined(MFEM_USE_MAGMA)
+      magma_setdevice(device);
+      magma_queue_create(device, &queue);
+#endif
+   };
+
+   //Action of block diagonal matrix
+   void Mult(const Vector &b, Vector &x);
+};
+
+
+} // namespace mfem
+
+#endif

linalg/linalg.hpp

@@ -18,6 +18,7 @@
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"
+#include "libBatchSolver.hpp"
 #include "complex_operator.hpp"
 #include "complex_densemat.hpp"
 #include "blockvector.hpp"

linalg/ode.hpp

@@ -204,7 +204,7 @@ public:
 class RK6Solver : public ExplicitRKSolver
 {
 private:
-   static const double a[28], b[8], c[7];
+   static MFEM_EXPORT const double a[28], b[8], c[7];
 
 public:
    RK6Solver() : ExplicitRKSolver(8, a, b, c) { }
@@ -258,7 +258,7 @@ public:
 class AB1Solver : public AdamsBashforthSolver
 {
 private:
-   static const double a[1];
+   static MFEM_EXPORT const double a[1];
 
 public:
    AB1Solver() : AdamsBashforthSolver(1, a) { }
@@ -268,7 +268,7 @@ public:
 class AB2Solver : public AdamsBashforthSolver
 {
 private:
-   static const double a[2];
+   static MFEM_EXPORT const double a[2];
 
 public:
    AB2Solver() : AdamsBashforthSolver(2, a) { }
@@ -278,7 +278,7 @@ public:
 class AB3Solver : public AdamsBashforthSolver
 {
 private:
-   static const double a[3];
+   static MFEM_EXPORT const double a[3];
 
 public:
    AB3Solver() : AdamsBashforthSolver(3, a) { }
@@ -288,7 +288,7 @@ public:
 class AB4Solver : public AdamsBashforthSolver
 {
 private:
-   static const double a[4];
+   static MFEM_EXPORT const double a[4];
 
 public:
    AB4Solver() : AdamsBashforthSolver(4, a) { }
@@ -298,7 +298,7 @@ public:
 class AB5Solver : public AdamsBashforthSolver
 {
 private:
-   static const double a[5];
+   static MFEM_EXPORT const double a[5];
 
 public:
    AB5Solver() : AdamsBashforthSolver(5, a) { }
@@ -339,7 +339,7 @@ public:
 class AM0Solver : public AdamsMoultonSolver
 {
 private:
-   static const double a[1];
+   static MFEM_EXPORT const double a[1];
 
 public:
    AM0Solver() : AdamsMoultonSolver(0, a) { }
@@ -350,7 +350,7 @@ public:
 class AM1Solver : public AdamsMoultonSolver
 {
 private:
-   static const double a[2];
+   static MFEM_EXPORT const double a[2];
 
 public:
    AM1Solver() : AdamsMoultonSolver(1, a) { }
@@ -360,7 +360,7 @@ public:
 class AM2Solver : public AdamsMoultonSolver
 {
 private:
-   static const double a[3];
+   static MFEM_EXPORT const double a[3];
 
 public:
    AM2Solver() : AdamsMoultonSolver(2, a) { }
@@ -370,7 +370,7 @@ public:
 class AM3Solver : public AdamsMoultonSolver
 {
 private:
-   static const double a[4];
+   static MFEM_EXPORT const double a[4];
 
 public:
    AM3Solver() : AdamsMoultonSolver(3, a) { }
@@ -380,7 +380,7 @@ public:
 class AM4Solver : public AdamsMoultonSolver
 {
 private:
-   static const double a[5];
+   static MFEM_EXPORT const double a[5];
 
 public:
    AM4Solver() : AdamsMoultonSolver(4, a) { }

makefile

@@ -198,6 +198,7 @@ ALL_LIBS =
 # Building static and/or shared libraries:
 MFEM_STATIC ?= $(STATIC)
 MFEM_SHARED ?= $(SHARED)
+MFEM_SHARED_BUILD = $(MFEM_SHARED)
 
 # Internal shortcuts
 override static = $(if $(MFEM_STATIC:YES=),,YES)
@@ -277,7 +278,7 @@ endif
 MFEM_REQ_LIB_DEPS = ENZYME SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
  SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP\
  GSLIB OCCA CEED RAJA UMPIRE MKL_CPARDISO AMGX CALIPER PARELAG BENCHMARK\
- MOONOLITH ALGOIM
+ MOONOLITH ALGOIM MAGMA
 
 
 PETSC_ERROR_MSG = $(if $(PETSC_FOUND),,. PETSC config not found: $(PETSC_VARS))
@@ -346,7 +347,7 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_AMGX\
  MFEM_USE_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
  MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_ALGOIM MFEM_USE_ENZYME\
- MFEM_SOURCE_DIR MFEM_INSTALL_DIR
+ MFEM_USE_MAGMA MFEM_SOURCE_DIR MFEM_INSTALL_DIR MFEM_SHARED_BUILD
 
 # List of makefile variables that will be written to config.mk:
 MFEM_CONFIG_VARS = MFEM_CXX MFEM_HOST_CXX MFEM_CPPFLAGS MFEM_CXXFLAGS\
@@ -589,7 +590,8 @@ install: $(if $(static),$(BLD)libmfem.a) $(if $(shared),$(BLD)libmfem.$(SO_EXT))
 	      > $(PREFIX_INC)/$$hdr && chmod 640 $(PREFIX_INC)/$$hdr; done
 # install config include
 	mkdir -p $(PREFIX_INC)/mfem/config
-	$(INSTALL) -m 640 $(BLD)config/_config.hpp $(PREFIX_INC)/mfem/config/config.hpp
+	$(INSTALL) -m 640 $(BLD)config/_config.hpp $(PREFIX_INC)/mfem/config/_config.hpp
+	$(INSTALL) -m 640 $(SRC)config/config.hpp $(PREFIX_INC)/mfem/config/config.hpp
 	$(INSTALL) -m 640 $(SRC)config/tconfig.hpp $(PREFIX_INC)/mfem/config
 # install remaining includes in each subdirectory
 	for dir in $(DIRS); do \
@@ -708,6 +710,7 @@ status info:
 	$(info MFEM_USE_BENCHMARK     = $(MFEM_USE_BENCHMARK))
 	$(info MFEM_USE_PARELAG       = $(MFEM_USE_PARELAG))
 	$(info MFEM_USE_ENZYME        = $(MFEM_USE_ENZYME))
+	$(info MFEM_USE_MAGMA         = $(MFEM_USE_MAGMA))
 	$(info MFEM_CXX               = $(value MFEM_CXX))
 	$(info MFEM_HOST_CXX          = $(value MFEM_HOST_CXX))
 	$(info MFEM_CPPFLAGS          = $(value MFEM_CPPFLAGS))

mesh/hexahedron.hpp

@@ -68,7 +68,7 @@ public:
    virtual ~Hexahedron() { }
 };
 
-extern class TriLinear3DFiniteElement HexahedronFE;
+extern MFEM_EXPORT class TriLinear3DFiniteElement HexahedronFE;
 
 }
 

mesh/point.hpp

@@ -63,7 +63,7 @@ public:
 };
 
 class PointFiniteElement;
-extern PointFiniteElement PointFE;
+extern MFEM_EXPORT PointFiniteElement PointFE;
 
 }
 

mesh/quadrilateral.hpp

@@ -69,7 +69,7 @@ public:
    virtual ~Quadrilateral() { }
 };
 
-extern class BiLinear2DFiniteElement QuadrilateralFE;
+extern MFEM_EXPORT class BiLinear2DFiniteElement QuadrilateralFE;
 
 }
 

mesh/segment.hpp

@@ -69,7 +69,7 @@ public:
 };
 
 class Linear1DFiniteElement;
-extern Linear1DFiniteElement SegmentFE;
+extern MFEM_EXPORT Linear1DFiniteElement SegmentFE;
 
 }
 

mesh/tetrahedron.hpp

@@ -117,7 +117,7 @@ public:
 };
 
 // Defined in fe.cpp to ensure construction before 'mfem::Geometries'.
-extern class Linear3DFiniteElement TetrahedronFE;
+extern MFEM_EXPORT class Linear3DFiniteElement TetrahedronFE;
 
 }
 

mesh/triangle.hpp

@@ -98,7 +98,7 @@ public:
 };
 
 // Defined in fe.cpp to ensure construction before 'mfem::Geometries'.
-extern Linear2DFiniteElement TriangleFE;
+extern MFEM_EXPORT Linear2DFiniteElement TriangleFE;
 
 }
 

mesh/wedge.hpp

@@ -71,7 +71,7 @@ public:
    virtual ~Wedge() { }
 };
 
-extern class LinearWedgeFiniteElement WedgeFE;
+extern MFEM_EXPORT class LinearWedgeFiniteElement WedgeFE;
 
 }