diff --git a/setup.cfg b/setup.cfg
index 2f5a36ba8..62040b3fd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -69,6 +69,8 @@ filterwarnings =
     ignore:Input WCS indicates that the spectral axis is not last:UserWarning
     ignore:No velocity defined on frame:astropy.coordinates.spectral_coordinate.NoVelocityWarning
     ignore:No observer defined on WCS:astropy.utils.exceptions.AstropyUserWarning
+    # matplotlib + python-dateutil<=2.8.2 + Python 3.12
+    ignore:datetime\.datetime\.utcfromtimestamp:DeprecationWarning
 
 [coverage:run]
 omit =
diff --git a/tox.ini b/tox.ini
index 805fbc64e..ae0aad324 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,7 +52,6 @@ deps =
 
     devdeps: numpy>=0.0.dev0
     devdeps: scipy>=0.0.dev0
-    devdeps: matplotlib>=0.0.dev0
     devdeps: pyerfa>=0.0.dev0
     devdeps: astropy>=0.0.dev0
     devdeps: git+https://github.com/spacetelescope/gwcs.git
@@ -66,8 +65,10 @@ extras =
     !oldestdeps: jwst
 
 commands =
+    # Install matplotlib after package build to make sure things are built with numpy 2.x ABI
+    devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple matplotlib>=0.0.dev0
     # Force numpy-dev after matplotlib downgrades it (https://github.com/matplotlib/matplotlib/issues/26847)
-    devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
+    devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy>=0.0.dev0
     pip freeze
     !cov: pytest --pyargs specutils '{toxinidir}/docs' {posargs}
     cov: pytest --pyargs specutils '{toxinidir}/docs' --cov specutils --cov-config='{toxinidir}/setup.cfg' {posargs}