Add regression tests for LPC3D

.github/workflows/ci.yml

@@ -55,6 +55,9 @@ jobs:
           if [ "${{ matrix.case }}" = "supercapacitor" ]; then NB=3; else NB=1; fi
           python3 tests/ci/check_outputs.py run $NB
 
+      - name: Regression test against reference results
+        run: python3 tests/ci/compare_reference.py run tests/ci/reference/${{ matrix.case }}
+
       - name: Upload outputs and environment
         if: always()
         uses: actions/upload-artifact@v4

tests/ci/compare_reference.py

@@ -0,0 +1,84 @@
+"""Regression test: compare an LPC3D run with stored reference results.
+
+Every reference file in <reference_directory> must exist in <run_directory>
+with the same shape, and its values must agree within a tolerance:
+
+    |run - ref| <= ATOL_FACTOR * max|ref| + RTOL * |ref|
+
+LPC3D fixes its random seed, so results are bit-for-bit identical from run
+to run and independent of the number of threads. Between different
+installations (Python/NumPy versions, CPU types) single-precision (float32)
+rounding differences remain: up to about 2e-5 relative on the NMR signal
+in our tests. RTOL leaves a margin above that while still catching
+physical changes (a 1 K change in temperature changes the diffusion
+coefficients by about 4e-4 relative).
+
+The phase column of the FT_signal files is not compared: where the
+spectrum magnitude is essentially zero, the phase is the angle of rounding
+noise and can take any value between -pi and pi.
+
+Usage: python3 compare_reference.py <run_directory> <reference_directory>
+"""
+import sys
+from pathlib import Path
+
+import numpy as np
+
+RTOL = 1e-4
+ATOL_FACTOR = 1e-6
+
+# Columns not compared, by file name prefix (0-based column indices)
+SKIP_COLUMNS = {
+    "FT_signal-": [2],  # frequency, magnitude, PHASE, Re(FT), Im(FT)
+}
+
+
+def main():
+    run_dir = Path(sys.argv[1])
+    ref_dir = Path(sys.argv[2])
+    ref_files = sorted(p for p in ref_dir.iterdir() if p.is_file())
+    if not ref_files:
+        print(f"No reference files found in {ref_dir}")
+        sys.exit(1)
+
+    errors = []
+    for ref_path in ref_files:
+        run_path = run_dir / ref_path.name
+        if not run_path.is_file():
+            errors.append(f"{ref_path.name}: missing from the run")
+            continue
+        ref = np.loadtxt(ref_path, ndmin=1)
+        new = np.loadtxt(run_path, ndmin=1)
+        if new.shape != ref.shape:
+            errors.append(f"{ref_path.name}: shape {new.shape}, expected {ref.shape}")
+            continue
+        for prefix, cols in SKIP_COLUMNS.items():
+            if ref_path.name.startswith(prefix) and ref.ndim == 2:
+                keep = [c for c in range(ref.shape[1]) if c not in cols]
+                ref, new = ref[:, keep], new[:, keep]
+        scale = np.max(np.abs(ref)) if ref.size else 0.0
+        atol = ATOL_FACTOR * scale
+        diff = np.abs(new - ref)
+        bad = ~(diff <= atol + RTOL * np.abs(ref))
+        if np.any(bad):
+            i = np.unravel_index(np.argmax(np.where(bad, diff, -1)), ref.shape)
+            errors.append(
+                f"{ref_path.name}: {int(bad.sum())} value(s) outside tolerance, "
+                f"largest at index {tuple(int(k) for k in i)}: "
+                f"got {new[i]:.10g}, reference {ref[i]:.10g}"
+            )
+
+    if errors:
+        print(f"Regression test FAILED against {ref_dir}:")
+        for e in errors:
+            print("  -", e)
+        print(
+            "If this change in the results is intended, regenerate the "
+            "references with tests/ci/update_reference.sh and commit them."
+        )
+        sys.exit(1)
+    print(f"Regression test passed: {len(ref_files)} file(s) match {ref_dir}")
+
+
+if __name__ == "__main__":
+    main()

tests/ci/reference/bulk/Diffusion_coeff_x_1.txt

@@ -0,0 +1,6 @@
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05
+3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05 3.555563750000000000e+05

tests/ci/reference/bulk/Diffusion_coeff_y_1.txt

@@ -0,0 +1,6 @@
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05

tests/ci/reference/bulk/Diffusion_coeff_z_1.txt

@@ -0,0 +1,6 @@
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05
+3.555564687500000000e+05 3.555564375000000000e+05 3.555564375000000000e+05 3.555564375000000000e+05

tests/ci/reference/bulk/FT_signal-1-bloc-1.dat

@@ -0,0 +1,101 @@
+# Frequency - Magnitude - Phase - Real part - Imaginary part
+-98000.0078125 5.19121474602846e-08 -3.0838900872054813 -5.182574845297083e-06 -2.993802135700461e-07
+-96000.0078125 6.940990416272192e-08 3.0526857893900106 -6.913576193050112e-06 6.16289040688807e-07
+-94000.0078125 5.671257280725463e-08 -3.0779888487703073 -5.6597897706044e-06 -3.6047038261605766e-07
+-92000.0078125 5.422886239197196e-08 -2.94874449968896 -5.322358691893758e-06 -1.0393233943760122e-06
+-90000.0078125 5.303258155724303e-08 -2.884741228189566 -5.1292827775437865e-06 -1.3472213085640494e-06
+-88000.0078125 6.04918134434375e-08 -3.061203756573152 -6.029645829798751e-06 -4.857634238464102e-07
+-86000.0078125 5.674041871158589e-08 -2.935449574004923 -5.553909079088184e-06 -1.1613979063536755e-06
+-84000.0078125 5.882589800858279e-08 -2.9449448358430814 -5.769214917274514e-06 -1.1493572131670263e-06
+-82000.0078125 5.885508584716597e-08 -2.931359877687516 -5.755923597765076e-06 -1.2282324036858433e-06
+-80000.0078125 5.7975654002553e-08 -2.8392145868492187 -5.534535781444345e-06 -1.7264640900260923e-06
+-78000.0078125 5.588379096261863e-08 -2.9548749672822483 -5.491246575513687e-06 -1.0373967276050976e-06
+-76000.0078125 5.906434302606463e-08 -2.7497132097193133 -5.458684552043824e-06 -2.2558211658472435e-06
+-74000.0078125 7.938910507973825e-08 -2.52656484689392 -6.484163931324476e-06 -4.580602380181882e-06
+-72000.0078125 5.953444123124578e-08 -2.6628300981984587 -5.284072619988273e-06 -2.7426398731617756e-06
+-70000.0078125 7.236197216162193e-08 -2.6876544890641116 -6.503366961508984e-06 -3.173132256170058e-06
+-68000.0078125 6.242669279108355e-08 -2.568101843716443 -5.243917708966723e-06 -3.387070532172076e-06
+-66000.0078125 7.274054696971521e-08 -2.777825568762437 -6.7980633929176115e-06 -2.588089225742989e-06
+-64000.00390625 6.730441989973306e-08 -2.6656928728030707 -5.982561046343415e-06 -3.0834741294796424e-06
+-62000.00390625 6.642815042734049e-08 -2.554266384399525 -5.529647084306333e-06 -3.6810317595745125e-06
+-60000.00390625 5.759203165934538e-08 -2.6609051098279055 -5.106554251228355e-06 -2.6629916984046377e-06
+-58000.00390625 8.054804799220206e-08 -2.403318998758536 -5.95758744778609e-06 -5.420980737424038e-06
+-56000.00390625 7.34968609038471e-08 -2.3703851886444394 -5.2702363736063406e-06 -5.122742838900971e-06
+-54000.00390625 7.916958883572028e-08 -2.410509284495971 -5.893791910238987e-06 -5.285967733818618e-06
+-52000.00390625 7.37428505276639e-08 -2.41531750995902 -5.51340797030094e-06 -4.897184149332743e-06
+-50000.00390625 8.26250252157562e-08 -2.3357891591215107 -5.722045974398301e-06 -5.9604645612498786e-06
+-48000.00390625 7.957575750957468e-08 -2.36058643994793 -5.651514294956424e-06 -5.602088718159458e-06
+-46000.00390625 8.340931993401902e-08 -2.288791834895305 -5.487301977051739e-06 -6.281772324049918e-06
+-44000.00390625 1.0657276133321002e-07 -2.2057958927596104 -6.321652842599359e-06 -8.579874120260092e-06
+-42000.00390625 9.787329913410306e-08 -2.1986679650835708 -5.749309061885344e-06 -7.920686343042454e-06
+-40000.00390625 9.391681379767057e-08 -2.1492478701924926 -5.134695686363391e-06 -7.863750971864791e-06
+-38000.00390625 9.312895317854807e-08 -2.0553386270589016 -4.337978257634497e-06 -8.240871545996355e-06
+-36000.00390625 1.1908662184263699e-07 -2.187274975782195 -6.885178963958579e-06 -9.716508923125349e-06
+-34000.00390625 1.1954013943219242e-07 -2.0730025585139558 -5.754190376476167e-06 -1.0477964614654371e-05
+-32000.001953125 1.1609238012479853e-07 -2.117815901773409 -6.0384757960408e-06 -9.915201323777374e-06
+-30000.001953125 8.733441125817562e-08 -2.1494070428013856 -4.7759811782047454e-06 -7.311839555375619e-06
+-28000.001953125 1.4922606041456534e-07 -2.004039557477171 -6.264757588719014e-06 -1.354389100011893e-05
+-26000.001953125 1.5190715672810645e-07 -1.9586941653643688 -5.7457859448278146e-06 -1.4062140183072919e-05
+-24000.001953125 1.636601016564675e-07 -1.9469513030474974 -6.0120047159441364e-06 -1.5221763630983048e-05
+-22000.001953125 1.7608080118307727e-07 -1.926658662504649 -6.134634017821758e-06 -1.6504870521156113e-05
+-20000.001953125 1.9227087644375565e-07 -1.8903205361534825 -6.0395144762007205e-06 -1.8253908189104642e-05
+-18000.001953125 2.1210768097399333e-07 -1.8341735603208105 -5.5220708463491795e-06 -2.047934122108311e-05
+-16000.0009765625 2.3272319174946652e-07 -1.8179871658001183 -5.6942979509067736e-06 -2.2564924343478054e-05
+-14000.0009765625 2.8521054427102784e-07 -1.7831723494994824 -6.011757205485954e-06 -2.788026759082572e-05
+-12000.0009765625 3.226050772720812e-07 -1.75241827253591 -5.827056735073093e-06 -3.1729887623871194e-05
+-10000.0009765625 3.8203738970399847e-07 -1.7254150685882237 -5.883505716147843e-06 -3.774798049971023e-05
+-8000.00048828125 4.769523199273247e-07 -1.6871323797382596 -5.536167468446187e-06 -4.737284036873753e-05
+-6000.00048828125 6.329199926871572e-07 -1.6677840564189346 -6.128928007800161e-06 -6.299455066040485e-05
+-4000.000244140625 9.439910366277647e-07 -1.638670765206932 -6.4023676176597765e-06 -9.418174165529092e-05
+-2000.0001220703125 1.8984320612759706e-06 -1.6016599006234407 -5.858309639372832e-06 -0.00018975279476459507
+0.0 1.000005702972412 0.0 100.00057029724121 0.0
+2000.0001220703125 1.8984320619728667e-06 1.60165989826707 -5.8583091943954284e-06 0.00018975279484805585
+4000.000244140625 9.439910374901022e-07 1.638670770646337 -6.4023681358009984e-06 9.418174170650104e-05
+6000.00048828125 6.329199917520836e-07 1.6677840640670252 -6.1289284805333375e-06 6.299455052046235e-05
+8000.00048828125 4.769523192678639e-07 1.6871323729956382 -5.536167141374452e-06 4.737284034056549e-05
+10000.0009765625 3.8203738959366136e-07 1.725415069018528 -5.883505730691736e-06 3.774798048627645e-05
+12000.0009765625 3.2260507884860323e-07 1.7524182720338461 -5.827056747618596e-06 3.172988778185588e-05
+14000.0009765625 2.852105444563301e-07 1.7831723549502718 -6.01175736136128e-06 2.7880267576170806e-05
+16000.0009765625 2.327231928708003e-07 1.8179871675313928 -5.694298017409775e-06 2.2564924442344587e-05
+18000.001953125 2.1210768180738761e-07 1.8341735694640329 -5.5220710552931706e-06 2.0479341251059157e-05
+20000.001953125 1.922708738608035e-07 1.8903205386763064 -6.03951444111774e-06 1.8253907928646412e-05
+22000.001953125 1.7608080024507844e-07 1.9266586575831388 -6.1346339039131044e-06 1.650487046342479e-05
+24000.001953125 1.6366010070804097e-07 1.946951302871148 -6.012004678419624e-06 1.5221763543831631e-05
+26000.001953125 1.5190715751475807e-07 1.9586941711593562 -5.745786056072306e-06 1.4062140222596992e-05
+28000.001953125 1.4922606171595404e-07 2.0040395507054374 -6.264757551637841e-06 1.354389116065741e-05
+30000.001953125 8.733441434611852e-08 2.1494070537182997 -4.775981426895125e-06 7.311839761766386e-06
+32000.001953125 1.160923818121719e-07 2.1178159020137164 -6.038475886191213e-06 9.915201466441226e-06
+34000.00390625 1.1954014067289473e-07 2.0730025638495713 -5.75419049210507e-06 1.0477964692702596e-05
+36000.00390625 1.190866229148522e-07 2.187274978023272 -6.885179047725819e-06 9.716508995179255e-06
+38000.00390625 9.312895556477734e-08 2.0553386430010847 -4.337978500163353e-06 8.240871687994131e-06
+40000.00390625 9.391681625102028e-08 2.149247847495304 -5.134695642009859e-06 7.86375129382943e-06
+42000.00390625 9.787329931289306e-08 2.198667962096705 -5.749309048729864e-06 7.92068637468398e-06
+44000.00390625 1.0657276393095935e-07 2.205795914567357 -6.321653183799641e-06 8.579874191536597e-06
+46000.00390625 8.340931975432507e-08 2.2887918534304257 -5.487302081663505e-06 6.281772208808895e-06
+48000.00390625 7.957575772432231e-08 2.3605864695165715 -5.651514475854071e-06 5.6020885661699685e-06
+50000.00390625 8.26250239449434e-08 2.3357891641713473 -5.722045916489837e-06 5.960464440679658e-06
+52000.00390625 7.374284899972566e-08 2.4153175353122527 -5.5134079802235915e-06 4.897183908081278e-06
+54000.00390625 7.916959118952235e-08 2.4105092734678015 -5.893792027173583e-06 5.2859679559742e-06
+56000.00390625 7.349686122609313e-08 2.3703851818255446 -5.270236361782179e-06 5.122742897298756e-06
+58000.00390625 8.054805038641927e-08 2.403319001374272 -5.957587639049795e-06 5.420980882974269e-06
+60000.00390625 5.759203015435542e-08 2.660905117789535 -5.106554138986076e-06 2.6629915881590797e-06
+62000.00390625 6.642815176547734e-08 2.5542663789764823 -5.529647175733835e-06 3.6810318637131806e-06
+64000.00390625 6.730441822792753e-08 2.6656928444306605 -5.982560810254225e-06 3.0834742226274774e-06
+66000.0078125 7.274054577938264e-08 2.777825590361304 -6.798063337573308e-06 2.5880890365608128e-06
+68000.0078125 6.242669295637226e-08 2.568101849745604 -5.243917743272369e-06 3.3870705095236826e-06
+70000.0078125 7.236197593516884e-08 2.687654463498756 -6.503367219525632e-06 3.1731325879041042e-06
+72000.0078125 5.953443871280598e-08 2.6628301190197914 -5.284072453565617e-06 2.7426396471205545e-06
+74000.0078125 7.938910607490337e-08 2.5265648558348253 -6.484164053560055e-06 4.580602379626748e-06
+76000.0078125 5.906433957628083e-08 2.749713220829345 -5.458684258279502e-06 2.2558209734448577e-06
+78000.0078125 5.588379118429996e-08 2.954874981173438 -5.491246611707186e-06 1.037396655440324e-06
+80000.0078125 5.7975653070193046e-08 2.839214605864438 -5.534535725267463e-06 1.7264639570208213e-06
+82000.0078125 5.88550867253905e-08 2.9313598805245284 -5.755923687138397e-06 1.228232405683668e-06
+84000.0078125 5.8825897608492776e-08 2.944944778796173 -5.7692148124693195e-06 1.149357534465825e-06
+86000.0078125 5.674041965508261e-08 2.935449558821577 -5.553909153806346e-06 1.1613980099926702e-06
+88000.0078125 6.049181397317249e-08 3.061203761007384 -6.029645884755164e-06 4.857634013634557e-07
+90000.0078125 5.303258104492116e-08 2.884741233325352 -5.1292827349113325e-06 1.347221269206302e-06
+92000.0078125 5.42288637430145e-08 2.948744525502427 -5.322358851322036e-06 1.039323282880887e-06
+94000.0078125 5.671257261441623e-08 3.077988843640005 -5.65978974951023e-06 3.6047041042679185e-07
+96000.0078125 6.940990459351065e-08 -3.05268576714412 -6.91357622224894e-06 -6.162891983124315e-07
+98000.0078125 5.1912148661038994e-08 3.0838900891549343 -5.1825749657563046e-06 2.9938021039167454e-07
+100000.0078125 5.722045898437501e-08 -3.1415926348244025 -5.7220458984375e-06 -1.073764264076227e-13

tests/ci/reference/bulk/Normalised_spectrum-1-bloc-1.dat

@@ -0,0 +1,101 @@
+# Frequency (ppm) - Magnitude
+-347.517758 0.000000
+-340.425560 0.000000
+-333.333361 0.000000
+-326.241162 0.000000
+-319.148964 0.000000
+-312.056765 0.000000
+-304.964567 0.000000
+-297.872368 0.000000
+-290.780170 0.000000
+-283.687971 0.000000
+-276.595772 0.000000
+-269.503574 0.000000
+-262.411375 0.000000
+-255.319177 0.000000
+-248.226978 0.000000
+-241.134779 0.000000
+-234.042581 0.000000
+-226.950368 0.000000
+-219.858170 0.000000
+-212.765971 0.000000
+-205.673773 0.000000
+-198.581574 0.000000
+-191.489376 0.000000
+-184.397177 0.000000
+-177.304978 0.000000
+-170.212780 0.000000
+-163.120581 0.000000
+-156.028383 0.000000
+-148.936184 0.000000
+-141.843985 0.000000
+-134.751787 0.000000
+-127.659588 0.000000
+-120.567390 0.000000
+-113.475184 0.000000
+-106.382986 0.000000
+-99.290787 0.000000
+-92.198588 0.000000
+-85.106390 0.000000
+-78.014191 0.000000
+-70.921993 0.000000
+-63.829794 0.000000
+-56.737592 0.000000
+-49.645394 0.000000
+-42.553195 0.000000
+-35.460996 0.000000
+-28.368796 0.000000
+-21.276597 0.000000
+-14.184398 0.000000
+-7.092199 0.000000
+0.000000 0.140998
+7.092199 0.000000
+14.184398 0.000000
+21.276597 0.000000
+28.368796 0.000000
+35.460996 0.000000
+42.553195 0.000000
+49.645394 0.000000
+56.737592 0.000000
+63.829794 0.000000
+70.921993 0.000000
+78.014191 0.000000
+85.106390 0.000000
+92.198588 0.000000
+99.290787 0.000000
+106.382986 0.000000
+113.475184 0.000000
+120.567390 0.000000
+127.659588 0.000000
+134.751787 0.000000
+141.843985 0.000000
+148.936184 0.000000
+156.028383 0.000000
+163.120581 0.000000
+170.212780 0.000000
+177.304978 0.000000
+184.397177 0.000000
+191.489376 0.000000
+198.581574 0.000000
+205.673773 0.000000
+212.765971 0.000000
+219.858170 0.000000
+226.950368 0.000000
+234.042581 0.000000
+241.134779 0.000000
+248.226978 0.000000
+255.319177 0.000000
+262.411375 0.000000
+269.503574 0.000000
+276.595772 0.000000
+283.687971 0.000000
+290.780170 0.000000
+297.872368 0.000000
+304.964567 0.000000
+312.056765 0.000000
+319.148964 0.000000
+326.241162 0.000000
+333.333361 0.000000
+340.425560 0.000000
+347.517758 0.000000
+354.609957 0.000000