From 7e1fb3d01312fd0e91b107db55ff1ac9f4469dbd Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 11 Nov 2024 11:05:33 -0600 Subject: [PATCH 01/11] ParameterCurveFit to support constants for min and max asymptote --- .../api/data/statistics/StatsService.java | 1 + .../core/statistics/FourParameterSimplex.java | 2 +- .../core/statistics/ParameterCurveFit.java | 33 ++++++++++++++++--- .../core/statistics/StatsServiceImpl.java | 9 ++++- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index 341137c5cce..2db38e2066c 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -120,4 +120,5 @@ public String getLabel() * @param data an array of {@code DoublePoint} instances to initialize the curve fit with. */ CurveFit getCurveFit(CurveFitType type, DoublePoint[] data); + CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Double asymptoteMin, @Nullable Double asymptoteMax); } diff --git a/core/src/org/labkey/core/statistics/FourParameterSimplex.java b/core/src/org/labkey/core/statistics/FourParameterSimplex.java index f89c137b734..f3061bbbb62 100644 --- a/core/src/org/labkey/core/statistics/FourParameterSimplex.java +++ b/core/src/org/labkey/core/statistics/FourParameterSimplex.java @@ -37,7 +37,7 @@ public class FourParameterSimplex extends ParameterCurveFit implements Multivari public FourParameterSimplex(DoublePoint[] data) { - super(data, StatsService.CurveFitType.FOUR_PARAMETER_SIMPLEX); + super(data, StatsService.CurveFitType.FOUR_PARAMETER_SIMPLEX, null, null); } @Override diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index 94b61235b9a..b54878f33c2 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -15,6 +15,7 @@ */ package org.labkey.core.statistics; +import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.data.statistics.CurveFit; import org.labkey.api.data.statistics.DoublePoint; @@ -32,7 +33,9 @@ */ public class ParameterCurveFit extends DefaultCurveFit implements CurveFit { - private final StatsService.CurveFitType _fitType; + protected final StatsService.CurveFitType _fitType; + private Double _asymptoteMin; + private Double _asymptoteMax; public static class SigmoidalParameters implements CurveFit.Parameters, Cloneable { @@ -112,10 +115,12 @@ public static SigmoidalParameters fromJSON(JSONObject json) } } - public ParameterCurveFit(DoublePoint[] data, StatsService.CurveFitType fitType) + public ParameterCurveFit(DoublePoint[] data, StatsService.CurveFitType fitType, @Nullable Double asymptoteMin, @Nullable Double asymptoteMax) { super(data); _fitType = fitType; + setAsymptoteMin(asymptoteMin); + setAsymptoteMax(asymptoteMax); } @Override @@ -124,6 +129,26 @@ public StatsService.CurveFitType getType() return _fitType; } + public Double getAsymptoteMin() + { + return _asymptoteMin; + } + + public void setAsymptoteMin(Double asymptoteMin) + { + _asymptoteMin = asymptoteMin; + } + + public Double getAsymptoteMax() + { + return _asymptoteMax; + } + + public void setAsymptoteMax(Double asymptoteMax) + { + _asymptoteMax = asymptoteMax; + } + @Override protected SigmoidalParameters computeParameters() { @@ -209,14 +234,14 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max // reaches 200 or min reaches -100, since these values don't seem biologically reasonable. for (double min = minValue; (bestFit == null || min > 0 - step) && min > (minValue - 100); min -= step ) { - parameters.min = min; + parameters.min = getAsymptoteMin() != null ? getAsymptoteMin() : min; for (double max = maxValue; (bestFit == null || max <= 100 + step) && max < (maxValue + 100); max += step ) { double absoluteCutoff = min + (0.5 * (max - min)); double relativeEC50 = getInterpolatedCutoffXValue(absoluteCutoff); if (!Double.isInfinite(relativeEC50) && !Double.isNaN(relativeEC50)) { - parameters.max = max; + parameters.max = getAsymptoteMax() != null ? getAsymptoteMax() : max; parameters.inflection = relativeEC50; for (double slopeRadians = 0; slopeRadians < Math.PI; slopeRadians += Math.PI / 30) { diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index 9610a241d23..a5d481dc6ed 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -17,6 +17,7 @@ import org.apache.commons.math3.random.RandomDataImpl; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; import org.labkey.api.data.statistics.CurveFit; @@ -56,6 +57,12 @@ public MathStat getStats(Collection data) @Override public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data) + { + return getCurveFit(type, data, null, null); + } + + @Override + public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Double asymptoteMin, @Nullable Double asymptoteMax) { switch (type) { @@ -63,7 +70,7 @@ public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data) return new FourParameterSimplex(data); case FOUR_PARAMETER: case FIVE_PARAMETER: - return new ParameterCurveFit(data, type); + return new ParameterCurveFit(data, type, asymptoteMin, asymptoteMax); case POLYNOMIAL: return new PolynomialCurveFit(data); case LINEAR: From 0ce4d448a67299814beef49d367b3f47bde41b0f Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 11 Nov 2024 11:07:10 -0600 Subject: [PATCH 02/11] Add Sigmoidal Logistic 3 Parameter curve fit --- .../api/data/statistics/StatsService.java | 1 + .../core/statistics/ParameterCurveFit.java | 15 ++++++- .../core/statistics/StatsServiceImpl.java | 2 + .../statistics/ThreeParameterCurveFit.java | 42 +++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index 2db38e2066c..cb9ebdc01c5 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -39,6 +39,7 @@ static void setInstance(StatsService impl) enum CurveFitType { + THREE_PARAMETER("Three Parameter", "3pl"), FOUR_PARAMETER("Four Parameter", "4pl"), FIVE_PARAMETER("Five Parameter", "5pl"), FOUR_PARAMETER_SIMPLEX("4 Parameter", "4param"), diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index b54878f33c2..bb0318fc3d7 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -221,12 +221,22 @@ public double solveForX(double y) } } + private boolean is3Parameter() + { + return _fitType == StatsService.CurveFitType.THREE_PARAMETER; + } + + private boolean is4Parameter() + { + return _fitType == StatsService.CurveFitType.FOUR_PARAMETER; + } + protected SigmoidalParameters calculateFitParameters(double minValue, double maxValue) { SigmoidalParameters bestFit = null; SigmoidalParameters parameters = new SigmoidalParameters(); - double step = 10; - if (_fitType == StatsService.CurveFitType.FOUR_PARAMETER) + double step = getAsymptoteMax() != null ? Math.min(getAsymptoteMax() / 100, 10) : 10; + if (is3Parameter() || is4Parameter()) parameters.asymmetry = 1; // try reasonable variants of max and min, in case there's a better fit. We'll keep going past "reasonable" if @@ -257,6 +267,7 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max bestFit = parameters.copy(); } break; + case THREE_PARAMETER: case FOUR_PARAMETER: parameters.asymmetry = 1; parameters.fitError = calculateFitError(parameters); diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index a5d481dc6ed..b1741fef939 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -66,6 +66,8 @@ public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Dou { switch (type) { + case THREE_PARAMETER: + return new ThreeParameterCurveFit(data, asymptoteMax); case FOUR_PARAMETER_SIMPLEX: return new FourParameterSimplex(data); case FOUR_PARAMETER: diff --git a/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java new file mode 100644 index 00000000000..d6df43f3a22 --- /dev/null +++ b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java @@ -0,0 +1,42 @@ +package org.labkey.core.statistics; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.statistics.DoublePoint; + +import static org.labkey.api.data.statistics.StatsService.CurveFitType.THREE_PARAMETER; + +public class ThreeParameterCurveFit extends ParameterCurveFit +{ + + public ThreeParameterCurveFit(DoublePoint[] data, @Nullable Double asymptoteMax) + { + super(data, THREE_PARAMETER, 0.0, asymptoteMax); + } + + @Override + public double fitCurve(double x, SigmoidalParameters params) + { + if (params != null) + { + // TODO hasXLogScale()? + + if (x <= 0) + { + if (params.getSlope() < 0) + return 0; + else + return params.getMax(); + } + else + { + if (params.getSlope() > 0) + return params.getMax() / (1 + Math.pow(Math.abs(x / params.getInflection()), params.getSlope())); + else + return params.getMax() * Math.pow(Math.abs(x / params.getInflection()), Math.abs(params.getSlope())) / + (1 + Math.pow(Math.abs(x / params.getInflection()), Math.abs(params.getSlope()))); + } + + } + throw new IllegalArgumentException("No curve fit parameters for " + _fitType.name()); + } +} From 64eba660f72737f54b6a55d228341801cece2667 Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 11 Nov 2024 15:52:24 -0600 Subject: [PATCH 03/11] support two variations of the 3PL curve fit for now --- api/src/org/labkey/api/data/statistics/StatsService.java | 1 + core/src/org/labkey/core/statistics/ParameterCurveFit.java | 6 ++++-- core/src/org/labkey/core/statistics/StatsServiceImpl.java | 4 +++- ...erCurveFit.java => ThreeParameterAlternateCurveFit.java} | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) rename core/src/org/labkey/core/statistics/{ThreeParameterCurveFit.java => ThreeParameterAlternateCurveFit.java} (88%) diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index cb9ebdc01c5..c7c924b7739 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -40,6 +40,7 @@ static void setInstance(StatsService impl) enum CurveFitType { THREE_PARAMETER("Three Parameter", "3pl"), + THREE_PARAMETER_ALT("Three Parameter (Alternate)", "3param"), // TODO name TBD FOUR_PARAMETER("Four Parameter", "4pl"), FIVE_PARAMETER("Five Parameter", "5pl"), FOUR_PARAMETER_SIMPLEX("4 Parameter", "4param"), diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index bb0318fc3d7..a7048670586 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -223,7 +223,7 @@ public double solveForX(double y) private boolean is3Parameter() { - return _fitType == StatsService.CurveFitType.THREE_PARAMETER; + return _fitType == StatsService.CurveFitType.THREE_PARAMETER || _fitType == StatsService.CurveFitType.THREE_PARAMETER_ALT; } private boolean is4Parameter() @@ -235,7 +235,8 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max { SigmoidalParameters bestFit = null; SigmoidalParameters parameters = new SigmoidalParameters(); - double step = getAsymptoteMax() != null ? Math.min(getAsymptoteMax() / 100, 10) : 10; + Double asymptoteDiff = getAsymptoteMax() != null && getAsymptoteMin() != null ? Math.abs(getAsymptoteMax() - getAsymptoteMin()) : null; + double step = asymptoteDiff != null ? Math.min(asymptoteDiff / 100, 10) : 10; if (is3Parameter() || is4Parameter()) parameters.asymmetry = 1; @@ -268,6 +269,7 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max } break; case THREE_PARAMETER: + case THREE_PARAMETER_ALT: case FOUR_PARAMETER: parameters.asymmetry = 1; parameters.fitError = calculateFitError(parameters); diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index b1741fef939..e923857ecf3 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -67,7 +67,9 @@ public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Dou switch (type) { case THREE_PARAMETER: - return new ThreeParameterCurveFit(data, asymptoteMax); + return new ParameterCurveFit(data, type, 0.0, asymptoteMax); + case THREE_PARAMETER_ALT: + return new ThreeParameterAlternateCurveFit(data, asymptoteMax); case FOUR_PARAMETER_SIMPLEX: return new FourParameterSimplex(data); case FOUR_PARAMETER: diff --git a/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java b/core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java similarity index 88% rename from core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java rename to core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java index d6df43f3a22..f865cf052ad 100644 --- a/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java @@ -5,10 +5,10 @@ import static org.labkey.api.data.statistics.StatsService.CurveFitType.THREE_PARAMETER; -public class ThreeParameterCurveFit extends ParameterCurveFit +public class ThreeParameterAlternateCurveFit extends ParameterCurveFit { - public ThreeParameterCurveFit(DoublePoint[] data, @Nullable Double asymptoteMax) + public ThreeParameterAlternateCurveFit(DoublePoint[] data, @Nullable Double asymptoteMax) { super(data, THREE_PARAMETER, 0.0, asymptoteMax); } From af5a3e1058d2a2909957e5ecc8407e3ac3ddda94 Mon Sep 17 00:00:00 2001 From: cnathe Date: Tue, 12 Nov 2024 12:07:26 -0600 Subject: [PATCH 04/11] StatsServiceImpl.TestCase update for 3PL curve fits --- .../labkey/core/statistics/StatsServiceImpl.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index e923857ecf3..6558a164ccc 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -128,6 +128,8 @@ public void TestCurveFits() throws Exception CurveValidation v1 = new CurveValidation(new double[]{12.54, 12.04, 9.11, 7.48, .576, -.512, 1.99, -6.60}); v1.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2, .044, .052)); + v1.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.09, .065, .065)); + v1.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.09, .065, .065)); v1.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(2.5, .031, .045)); v1.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(2.2, .046, .054)); v1.setResults(CurveFitType.LINEAR, new CurveResults(6.8, .070, .070)); @@ -135,6 +137,8 @@ public void TestCurveFits() throws Exception CurveValidation v2 = new CurveValidation(new double[]{93.28, 88.65, 74.12, 46.16, 28.34, 17.41, 6.17, -1.79}); v2.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.4, .414, .424)); + v2.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.45, .414, .414)); + v2.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.45, .414, .414)); v2.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(3.4, .403, .403)); v2.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.1, .420, .420)); v2.setResults(CurveFitType.LINEAR, new CurveResults(36.8, .553, .553)); @@ -142,6 +146,8 @@ public void TestCurveFits() throws Exception CurveValidation v3 = new CurveValidation(new double[]{10.79, 3.21, .599, 9.96, 9.5, 8.39, 1.56, -5.81}); v3.setResults(CurveFitType.POLYNOMIAL, new CurveResults(4.1, .055, .056)); + v3.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(5.0, .078, .078)); + v3.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(5.0, .078, .078)); v3.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.7, .048, .049)); v3.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(4.6, .080, .082)); v3.setResults(CurveFitType.LINEAR, new CurveResults(5.9, .070, .070)); @@ -149,6 +155,8 @@ public void TestCurveFits() throws Exception CurveValidation v4 = new CurveValidation(new double[]{75.94, 58.52, 39.42, 28.84, 19.37, 9.91, 6.04, -7.35}); v4.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2.4, .259, .273)); + v4.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(4.34, .280, .280)); + v4.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(4.34, .280, .280)); v4.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.5, .226, .247)); v4.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.7, .245, .262)); v4.setResults(CurveFitType.LINEAR, new CurveResults(27.5, .374, .374)); @@ -156,6 +164,8 @@ public void TestCurveFits() throws Exception CurveValidation v5 = new CurveValidation(new double[]{89.34, 74.24, 45.69, 18.34, .365, -1.65, -.77, -16.59}); v5.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.9, .207, .263)); + v5.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(7.86, .281, .281)); + v5.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(7.86, .281, .281)); v5.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(5, .201, .263)); v5.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(5.1, .221, .277)); v5.setResults(CurveFitType.LINEAR, new CurveResults(38.0, .363, .363)); @@ -171,9 +181,9 @@ public void TestCurveFits() throws Exception CurveResults results = validation.getResults(fitType); // validate calculated and expected fit error and auc - assertEquals(fit.getFitError(), results.getFitError(), 0.05); - assertEquals(fit.calculateAUC(AUCType.NORMAL), results.getAuc(), 0.005); - assertEquals(fit.calculateAUC(AUCType.POSITIVE), results.getPositiveAuc(), 0.005); + assertEquals(results.getFitError(), fit.getFitError(), 0.05); + assertEquals(results.getAuc(), fit.calculateAUC(AUCType.NORMAL), 0.005); + assertEquals(results.getPositiveAuc(), fit.calculateAUC(AUCType.POSITIVE), 0.005); } } } From 5b8c4c39e1046e525787095470c4ad19af582819 Mon Sep 17 00:00:00 2001 From: cnathe Date: Wed, 13 Nov 2024 10:20:47 -0600 Subject: [PATCH 05/11] Revert 3PL version based on "Four Parameter" equation --- .../labkey/api/data/statistics/StatsService.java | 3 +-- .../labkey/core/statistics/ParameterCurveFit.java | 3 +-- .../labkey/core/statistics/StatsServiceImpl.java | 9 +-------- ...teCurveFit.java => ThreeParameterCurveFit.java} | 14 +++++++++----- 4 files changed, 12 insertions(+), 17 deletions(-) rename core/src/org/labkey/core/statistics/{ThreeParameterAlternateCurveFit.java => ThreeParameterCurveFit.java} (78%) diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index c7c924b7739..81525fc5a6a 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -39,8 +39,7 @@ static void setInstance(StatsService impl) enum CurveFitType { - THREE_PARAMETER("Three Parameter", "3pl"), - THREE_PARAMETER_ALT("Three Parameter (Alternate)", "3param"), // TODO name TBD + THREE_PARAMETER("Three Parameter", "3PL"), FOUR_PARAMETER("Four Parameter", "4pl"), FIVE_PARAMETER("Five Parameter", "5pl"), FOUR_PARAMETER_SIMPLEX("4 Parameter", "4param"), diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index a7048670586..0cf07dfb3ee 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -223,7 +223,7 @@ public double solveForX(double y) private boolean is3Parameter() { - return _fitType == StatsService.CurveFitType.THREE_PARAMETER || _fitType == StatsService.CurveFitType.THREE_PARAMETER_ALT; + return _fitType == StatsService.CurveFitType.THREE_PARAMETER; } private boolean is4Parameter() @@ -269,7 +269,6 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max } break; case THREE_PARAMETER: - case THREE_PARAMETER_ALT: case FOUR_PARAMETER: parameters.asymmetry = 1; parameters.fitError = calculateFitError(parameters); diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index 6558a164ccc..b27b6efb893 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -67,9 +67,7 @@ public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Dou switch (type) { case THREE_PARAMETER: - return new ParameterCurveFit(data, type, 0.0, asymptoteMax); - case THREE_PARAMETER_ALT: - return new ThreeParameterAlternateCurveFit(data, asymptoteMax); + return new ThreeParameterCurveFit(data, asymptoteMax); case FOUR_PARAMETER_SIMPLEX: return new FourParameterSimplex(data); case FOUR_PARAMETER: @@ -129,7 +127,6 @@ public void TestCurveFits() throws Exception CurveValidation v1 = new CurveValidation(new double[]{12.54, 12.04, 9.11, 7.48, .576, -.512, 1.99, -6.60}); v1.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2, .044, .052)); v1.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.09, .065, .065)); - v1.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.09, .065, .065)); v1.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(2.5, .031, .045)); v1.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(2.2, .046, .054)); v1.setResults(CurveFitType.LINEAR, new CurveResults(6.8, .070, .070)); @@ -138,7 +135,6 @@ public void TestCurveFits() throws Exception CurveValidation v2 = new CurveValidation(new double[]{93.28, 88.65, 74.12, 46.16, 28.34, 17.41, 6.17, -1.79}); v2.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.4, .414, .424)); v2.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.45, .414, .414)); - v2.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.45, .414, .414)); v2.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(3.4, .403, .403)); v2.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.1, .420, .420)); v2.setResults(CurveFitType.LINEAR, new CurveResults(36.8, .553, .553)); @@ -147,7 +143,6 @@ public void TestCurveFits() throws Exception CurveValidation v3 = new CurveValidation(new double[]{10.79, 3.21, .599, 9.96, 9.5, 8.39, 1.56, -5.81}); v3.setResults(CurveFitType.POLYNOMIAL, new CurveResults(4.1, .055, .056)); v3.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(5.0, .078, .078)); - v3.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(5.0, .078, .078)); v3.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.7, .048, .049)); v3.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(4.6, .080, .082)); v3.setResults(CurveFitType.LINEAR, new CurveResults(5.9, .070, .070)); @@ -156,7 +151,6 @@ public void TestCurveFits() throws Exception CurveValidation v4 = new CurveValidation(new double[]{75.94, 58.52, 39.42, 28.84, 19.37, 9.91, 6.04, -7.35}); v4.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2.4, .259, .273)); v4.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(4.34, .280, .280)); - v4.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(4.34, .280, .280)); v4.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.5, .226, .247)); v4.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.7, .245, .262)); v4.setResults(CurveFitType.LINEAR, new CurveResults(27.5, .374, .374)); @@ -165,7 +159,6 @@ public void TestCurveFits() throws Exception CurveValidation v5 = new CurveValidation(new double[]{89.34, 74.24, 45.69, 18.34, .365, -1.65, -.77, -16.59}); v5.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.9, .207, .263)); v5.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(7.86, .281, .281)); - v5.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(7.86, .281, .281)); v5.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(5, .201, .263)); v5.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(5.1, .221, .277)); v5.setResults(CurveFitType.LINEAR, new CurveResults(38.0, .363, .363)); diff --git a/core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java similarity index 78% rename from core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java rename to core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java index f865cf052ad..672d3339517 100644 --- a/core/src/org/labkey/core/statistics/ThreeParameterAlternateCurveFit.java +++ b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java @@ -5,21 +5,25 @@ import static org.labkey.api.data.statistics.StatsService.CurveFitType.THREE_PARAMETER; -public class ThreeParameterAlternateCurveFit extends ParameterCurveFit +/* Based on the equation from SigmaPlot */ +public class ThreeParameterCurveFit extends ParameterCurveFit { - - public ThreeParameterAlternateCurveFit(DoublePoint[] data, @Nullable Double asymptoteMax) + public ThreeParameterCurveFit(DoublePoint[] data, @Nullable Double asymptoteMax) { super(data, THREE_PARAMETER, 0.0, asymptoteMax); } + @Override + public double solveForX(double y) + { + throw new UnsupportedOperationException("Not implemented"); + } + @Override public double fitCurve(double x, SigmoidalParameters params) { if (params != null) { - // TODO hasXLogScale()? - if (x <= 0) { if (params.getSlope() < 0) From 8eae2821e3c1609d9fc8b1f30175414ad9a54093 Mon Sep 17 00:00:00 2001 From: cnathe Date: Wed, 13 Nov 2024 10:21:14 -0600 Subject: [PATCH 06/11] StatsServiceImpl.TestCase - add 4PL simplex check --- core/src/org/labkey/core/statistics/StatsServiceImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index b27b6efb893..750b3eae3af 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -126,6 +126,7 @@ public void TestCurveFits() throws Exception CurveValidation v1 = new CurveValidation(new double[]{12.54, 12.04, 9.11, 7.48, .576, -.512, 1.99, -6.60}); v1.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2, .044, .052)); + v1.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.898, .044, .052)); v1.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.09, .065, .065)); v1.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(2.5, .031, .045)); v1.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(2.2, .046, .054)); @@ -134,6 +135,7 @@ public void TestCurveFits() throws Exception CurveValidation v2 = new CurveValidation(new double[]{93.28, 88.65, 74.12, 46.16, 28.34, 17.41, 6.17, -1.79}); v2.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.4, .414, .424)); + v2.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.994, .419, .419)); v2.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.45, .414, .414)); v2.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(3.4, .403, .403)); v2.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.1, .420, .420)); @@ -142,6 +144,7 @@ public void TestCurveFits() throws Exception CurveValidation v3 = new CurveValidation(new double[]{10.79, 3.21, .599, 9.96, 9.5, 8.39, 1.56, -5.81}); v3.setResults(CurveFitType.POLYNOMIAL, new CurveResults(4.1, .055, .056)); + v3.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.640, .052, .061)); v3.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(5.0, .078, .078)); v3.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.7, .048, .049)); v3.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(4.6, .080, .082)); @@ -150,6 +153,7 @@ public void TestCurveFits() throws Exception CurveValidation v4 = new CurveValidation(new double[]{75.94, 58.52, 39.42, 28.84, 19.37, 9.91, 6.04, -7.35}); v4.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2.4, .259, .273)); + v4.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.994, .258, .271)); v4.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(4.34, .280, .280)); v4.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.5, .226, .247)); v4.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.7, .245, .262)); @@ -158,6 +162,7 @@ public void TestCurveFits() throws Exception CurveValidation v5 = new CurveValidation(new double[]{89.34, 74.24, 45.69, 18.34, .365, -1.65, -.77, -16.59}); v5.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.9, .207, .263)); + v5.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.988, .211, .263)); v5.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(7.86, .281, .281)); v5.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(5, .201, .263)); v5.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(5.1, .221, .277)); @@ -168,7 +173,7 @@ public void TestCurveFits() throws Exception { for (CurveFitType fitType : CurveFitType.values()) { - if (fitType != CurveFitType.NONE && fitType != CurveFitType.FOUR_PARAMETER_SIMPLEX) + if (fitType != CurveFitType.NONE) { CurveFit fit = service.getCurveFit(fitType, validation.getData()); CurveResults results = validation.getResults(fitType); From ff72b4c56c19b8f756235d8f9ce0a32829063c1e Mon Sep 17 00:00:00 2001 From: cnathe Date: Wed, 13 Nov 2024 10:24:05 -0600 Subject: [PATCH 07/11] name change --- api/src/org/labkey/api/data/statistics/StatsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index 81525fc5a6a..c323cfc81b7 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -39,9 +39,9 @@ static void setInstance(StatsService impl) enum CurveFitType { - THREE_PARAMETER("Three Parameter", "3PL"), FOUR_PARAMETER("Four Parameter", "4pl"), FIVE_PARAMETER("Five Parameter", "5pl"), + THREE_PARAMETER("3 Parameter", "3param"), FOUR_PARAMETER_SIMPLEX("4 Parameter", "4param"), POLYNOMIAL("Polynomial", "poly"), LINEAR("Linear", "linear"), From b93075adc4e24542720c2d37fd39d56afc2ccd36 Mon Sep 17 00:00:00 2001 From: cnathe Date: Wed, 13 Nov 2024 11:54:03 -0600 Subject: [PATCH 08/11] Add back the original 3PL curve fit --- api/src/org/labkey/api/data/statistics/StatsService.java | 3 ++- .../org/labkey/core/statistics/ParameterCurveFit.java | 3 ++- .../src/org/labkey/core/statistics/StatsServiceImpl.java | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/data/statistics/StatsService.java b/api/src/org/labkey/api/data/statistics/StatsService.java index c323cfc81b7..42d5a69da09 100644 --- a/api/src/org/labkey/api/data/statistics/StatsService.java +++ b/api/src/org/labkey/api/data/statistics/StatsService.java @@ -39,9 +39,10 @@ static void setInstance(StatsService impl) enum CurveFitType { + THREE_PARAMETER("Three Parameter", "3pl"), FOUR_PARAMETER("Four Parameter", "4pl"), FIVE_PARAMETER("Five Parameter", "5pl"), - THREE_PARAMETER("3 Parameter", "3param"), + THREE_PARAMETER_ALT("3 Parameter", "3param"), FOUR_PARAMETER_SIMPLEX("4 Parameter", "4param"), POLYNOMIAL("Polynomial", "poly"), LINEAR("Linear", "linear"), diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index 0cf07dfb3ee..a7048670586 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -223,7 +223,7 @@ public double solveForX(double y) private boolean is3Parameter() { - return _fitType == StatsService.CurveFitType.THREE_PARAMETER; + return _fitType == StatsService.CurveFitType.THREE_PARAMETER || _fitType == StatsService.CurveFitType.THREE_PARAMETER_ALT; } private boolean is4Parameter() @@ -269,6 +269,7 @@ protected SigmoidalParameters calculateFitParameters(double minValue, double max } break; case THREE_PARAMETER: + case THREE_PARAMETER_ALT: case FOUR_PARAMETER: parameters.asymmetry = 1; parameters.fitError = calculateFitError(parameters); diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index 750b3eae3af..82b5ac196b0 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -66,10 +66,12 @@ public CurveFit getCurveFit(CurveFitType type, DoublePoint[] data, @Nullable Dou { switch (type) { - case THREE_PARAMETER: + case THREE_PARAMETER_ALT: return new ThreeParameterCurveFit(data, asymptoteMax); case FOUR_PARAMETER_SIMPLEX: return new FourParameterSimplex(data); + case THREE_PARAMETER: + return new ParameterCurveFit(data, type, 0.0, asymptoteMax); case FOUR_PARAMETER: case FIVE_PARAMETER: return new ParameterCurveFit(data, type, asymptoteMin, asymptoteMax); @@ -128,6 +130,7 @@ public void TestCurveFits() throws Exception v1.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2, .044, .052)); v1.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.898, .044, .052)); v1.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.09, .065, .065)); + v1.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.09, .065, .065)); v1.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(2.5, .031, .045)); v1.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(2.2, .046, .054)); v1.setResults(CurveFitType.LINEAR, new CurveResults(6.8, .070, .070)); @@ -137,6 +140,7 @@ public void TestCurveFits() throws Exception v2.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.4, .414, .424)); v2.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.994, .419, .419)); v2.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(3.45, .414, .414)); + v2.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(3.45, .414, .414)); v2.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(3.4, .403, .403)); v2.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.1, .420, .420)); v2.setResults(CurveFitType.LINEAR, new CurveResults(36.8, .553, .553)); @@ -146,6 +150,7 @@ public void TestCurveFits() throws Exception v3.setResults(CurveFitType.POLYNOMIAL, new CurveResults(4.1, .055, .056)); v3.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.640, .052, .061)); v3.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(5.0, .078, .078)); + v3.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(5.0, .078, .078)); v3.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.7, .048, .049)); v3.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(4.6, .080, .082)); v3.setResults(CurveFitType.LINEAR, new CurveResults(5.9, .070, .070)); @@ -155,6 +160,7 @@ public void TestCurveFits() throws Exception v4.setResults(CurveFitType.POLYNOMIAL, new CurveResults(2.4, .259, .273)); v4.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.994, .258, .271)); v4.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(4.34, .280, .280)); + v4.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(4.34, .280, .280)); v4.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(4.5, .226, .247)); v4.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(3.7, .245, .262)); v4.setResults(CurveFitType.LINEAR, new CurveResults(27.5, .374, .374)); @@ -164,6 +170,7 @@ public void TestCurveFits() throws Exception v5.setResults(CurveFitType.POLYNOMIAL, new CurveResults(5.9, .207, .263)); v5.setResults(CurveFitType.FOUR_PARAMETER_SIMPLEX, new CurveResults(.988, .211, .263)); v5.setResults(CurveFitType.THREE_PARAMETER, new CurveResults(7.86, .281, .281)); + v5.setResults(CurveFitType.THREE_PARAMETER_ALT, new CurveResults(7.86, .281, .281)); v5.setResults(CurveFitType.FOUR_PARAMETER, new CurveResults(5, .201, .263)); v5.setResults(CurveFitType.FIVE_PARAMETER, new CurveResults(5.1, .221, .277)); v5.setResults(CurveFitType.LINEAR, new CurveResults(38.0, .363, .363)); From bd1fefcfcd68331c3abfe7c58efb1507d055cfa8 Mon Sep 17 00:00:00 2001 From: cnathe Date: Wed, 13 Nov 2024 15:50:05 -0600 Subject: [PATCH 09/11] Add curve fit stats for new API: residual sum of squares, root mean square error, R2, adjusted R2 --- .../labkey/api/data/statistics/CurveFit.java | 50 +++++++++++++++++++ .../core/statistics/DefaultCurveFit.java | 9 +--- .../core/statistics/FourParameterSimplex.java | 35 +++---------- .../core/statistics/ParameterCurveFit.java | 12 +++++ .../statistics/ThreeParameterCurveFit.java | 6 +++ 5 files changed, 76 insertions(+), 36 deletions(-) diff --git a/api/src/org/labkey/api/data/statistics/CurveFit.java b/api/src/org/labkey/api/data/statistics/CurveFit.java index cabcda38610..27d85894db7 100644 --- a/api/src/org/labkey/api/data/statistics/CurveFit.java +++ b/api/src/org/labkey/api/data/statistics/CurveFit.java @@ -146,4 +146,54 @@ default JSONObject toJSON() * @return The integrated area under the curve. */ double calculateAUC(StatsService.AUCType type, double startX, double endX) throws FitFailedException; + + default double residualSumSquares(P parameters) + { + double sumSq = 0; + for (DoublePoint point : getData()) + { + double expectedValue = point.getY(); + double foundValue = fitCurve(point.getX(), parameters); + + sumSq += Math.pow(foundValue - expectedValue, 2); + } + return sumSq; + } + + default double rootMeanSquareError(P parameters) + { + return Math.sqrt(residualSumSquares(parameters) / getData().length); + } + + default double totalSumSquares() + { + double sumSq = 0; + double mean = 0; + for (DoublePoint point : getData()) + mean += point.getY(); + mean /= getData().length; + + for (DoublePoint point : getData()) + { + double expectedValue = point.getY(); + sumSq += Math.pow(expectedValue - mean, 2); + } + return sumSq; + } + + default double rSquared(P parameters) + { + return 1 - residualSumSquares(parameters) / totalSumSquares(); + } + + default double adjustedRSquared(P parameters) + { + return Double.NaN; + } + + default double adjustedRSquared(P parameters, int p) + { + int n = getData().length; + return 1 - (1 - rSquared(parameters)) * (n - 1) / (n - p - 1); + } } diff --git a/core/src/org/labkey/core/statistics/DefaultCurveFit.java b/core/src/org/labkey/core/statistics/DefaultCurveFit.java index 2737b005ab2..df6e8d45e06 100644 --- a/core/src/org/labkey/core/statistics/DefaultCurveFit.java +++ b/core/src/org/labkey/core/statistics/DefaultCurveFit.java @@ -318,14 +318,7 @@ public double getFitError() throws FitFailedException protected double calculateFitError(P parameters) { - double deviationValue = 0; - for (DoublePoint point : getData()) - { - double expectedValue = point.getY(); - double foundValue = fitCurve(point.getX(), parameters); - deviationValue += Math.pow(foundValue - expectedValue, 2); - } - return Math.sqrt(deviationValue / getData().length); + return rootMeanSquareError(parameters); } @Override diff --git a/core/src/org/labkey/core/statistics/FourParameterSimplex.java b/core/src/org/labkey/core/statistics/FourParameterSimplex.java index f3061bbbb62..f7ff80ffb72 100644 --- a/core/src/org/labkey/core/statistics/FourParameterSimplex.java +++ b/core/src/org/labkey/core/statistics/FourParameterSimplex.java @@ -106,26 +106,13 @@ private void optimize(MultivariateOptimizer optimizer, double[] start) protected double calculateFitError(SigmoidalParameters parameters) { - double deviationValue = 0; - double varianceValue = 0; - double total = 0; - - // find the mean - for (DoublePoint point : getData()) - { - total += point.getY(); - } - double mean = total / getData().length; - - for (DoublePoint point : getData()) - { - double expectedValue = point.getY(); - double foundValue = fitCurve(point.getX(), parameters); - deviationValue += Math.pow(foundValue - expectedValue, 2); - varianceValue += Math.pow(expectedValue - mean, 2); - } + return rSquared(parameters); + } - return 1 - deviationValue / varianceValue; + @Override + public double adjustedRSquared(SigmoidalParameters parameters) + { + return adjustedRSquared(parameters, 4); } @Override @@ -139,15 +126,7 @@ public double value(double[] point) private double sumSquares(double[] params) { SigmoidalParameters parameters = createParams(params); - double sumSq = 0; - for (DoublePoint point : getData()) - { - double expectedValue = point.getY(); - double foundValue = fitCurve(point.getX(), parameters); - - sumSq += Math.pow(foundValue - expectedValue, 2); - } - return sumSq; + return residualSumSquares(parameters); } private SigmoidalParameters createParams(double[] params) diff --git a/core/src/org/labkey/core/statistics/ParameterCurveFit.java b/core/src/org/labkey/core/statistics/ParameterCurveFit.java index a7048670586..bcd6c747aa0 100644 --- a/core/src/org/labkey/core/statistics/ParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ParameterCurveFit.java @@ -221,6 +221,18 @@ public double solveForX(double y) } } + @Override + public double adjustedRSquared(SigmoidalParameters parameters) + { + return switch (_fitType) + { + case THREE_PARAMETER, THREE_PARAMETER_ALT -> adjustedRSquared(parameters, 3); + case FOUR_PARAMETER -> adjustedRSquared(parameters, 4); + case FIVE_PARAMETER -> adjustedRSquared(parameters, 5); + default -> throw new IllegalStateException("Unsupported curve fit type: " + _fitType.name()); + }; + } + private boolean is3Parameter() { return _fitType == StatsService.CurveFitType.THREE_PARAMETER || _fitType == StatsService.CurveFitType.THREE_PARAMETER_ALT; diff --git a/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java index 672d3339517..dac795382e9 100644 --- a/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java +++ b/core/src/org/labkey/core/statistics/ThreeParameterCurveFit.java @@ -43,4 +43,10 @@ public double fitCurve(double x, SigmoidalParameters params) } throw new IllegalArgumentException("No curve fit parameters for " + _fitType.name()); } + + @Override + public double adjustedRSquared(SigmoidalParameters parameters) + { + return adjustedRSquared(parameters, 3); + } } From 65b8bad3e5d81700ce0de114bb4686f44d890a83 Mon Sep 17 00:00:00 2001 From: cnathe Date: Thu, 14 Nov 2024 10:24:42 -0600 Subject: [PATCH 10/11] StatsServiceImpl test cases for CurveFit parameters and stats --- .../core/statistics/StatsServiceImpl.java | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/core/src/org/labkey/core/statistics/StatsServiceImpl.java b/core/src/org/labkey/core/statistics/StatsServiceImpl.java index 82b5ac196b0..4cc63098d57 100644 --- a/core/src/org/labkey/core/statistics/StatsServiceImpl.java +++ b/core/src/org/labkey/core/statistics/StatsServiceImpl.java @@ -22,6 +22,7 @@ import org.junit.Test; import org.labkey.api.data.statistics.CurveFit; import org.labkey.api.data.statistics.DoublePoint; +import org.labkey.api.data.statistics.FitFailedException; import org.labkey.api.data.statistics.MathStat; import org.labkey.api.data.statistics.StatsService; import org.labkey.api.view.Stats; @@ -256,5 +257,136 @@ public double getPositiveAuc() return _positiveAuc; } } + + @Test + public void TestCurveFitParameters() throws Exception + { + StatsService service = StatsService.get(); + double delta = 0.005; + + DoublePoint[] data1 = new DoublePoint[]{ + new DoublePoint(0.025, 0.67), + new DoublePoint(0.05, 1), + new DoublePoint(0.1, 2), + new DoublePoint(0.2, 3.17), + }; + + CurveFit fit = service.getCurveFit(CurveFitType.LINEAR, data1); + fit.setLogXScale(false); + assertEquals(14.442, (Double) fit.getParameters().toMap().get("slope"), delta); + assertEquals(0.356, (Double) fit.getParameters().toMap().get("intercept"), delta); + assertEquals(0.986, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.POLYNOMIAL, data1); + fit.setLogXScale(false); + assertEquals(0.088, ((PolynomialCurveFit.PolynomialParameters)fit.getParameters()).getCoefficients()[0], delta); + assertEquals(21.8001, ((PolynomialCurveFit.PolynomialParameters)fit.getParameters()).getCoefficients()[1], delta); + assertEquals(-31.785, ((PolynomialCurveFit.PolynomialParameters)fit.getParameters()).getCoefficients()[2], delta); + assertEquals(0.996, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER, data1); + fit.setLogXScale(false); + verifySigmoidalParameters(fit, 0.0, 3.17, 2.246, 0.096, 1.0); + assertEquals(-1.667, fit.rSquared(fit.getParameters()), delta); + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER, data1, 0.0, 4.0); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.0, 4.0, 1.376, 0.095, 1.0); + assertEquals(0.974, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER_ALT, data1); + fit.setLogXScale(false); + verifySigmoidalParameters(fit, 0.0, 3.17, -1.732, 0.096, 1.0); + assertEquals(0.786, fit.rSquared(fit.getParameters()), delta); + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER_ALT, data1, 0.0, 4.0); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.0, 4.0, -1.376, 0.095, 1.0); + assertEquals(0.974, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER, data1); + fit.setLogXScale(false); + verifySigmoidalParameters(fit, 0.67, 3.17, null, 0.096, 1.0); + assertEquals(-1.138, fit.rSquared(fit.getParameters()), delta); + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER, data1, null, 4.0); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.67, 4.0, 2.246, 0.095, 1.0); + assertEquals(0.892, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER_SIMPLEX, data1); + fit.setLogXScale(false); + verifySigmoidalParameters(fit, 0.581, 3.787, 2.402, 0.110, 0.0); + assertEquals(1.0, fit.rSquared(fit.getParameters()), delta); + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER_SIMPLEX, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.581, 3.787, 2.402, 0.110, 0.0); + assertEquals(1.0, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.FIVE_PARAMETER, data1); + fit.setLogXScale(false); + verifySigmoidalParameters(fit, 0.67, 3.17, 1.111, 0.096, 2.932); + assertEquals(-1.624, fit.rSquared(fit.getParameters()), delta); + fit = service.getCurveFit(CurveFitType.FIVE_PARAMETER, data1, null, 4.0); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.67, 4.0, 2.246, 0.095, 1.466); + assertEquals(0.997, fit.rSquared(fit.getParameters()), delta); + } + + @Test + public void TestCurveFitParametersNAbCase() throws Exception + { + StatsService service = StatsService.get(); + double delta = 0.005; + + DoublePoint[] data1 = new DoublePoint[]{ + new DoublePoint(0.01, 93), + new DoublePoint(0.1, 89), + new DoublePoint(1, 74), + new DoublePoint(10, 46), + new DoublePoint(100, 28), + new DoublePoint(1000, 17), + new DoublePoint(10000, 6), + new DoublePoint(100000, -2), + }; + + CurveFit fit = service.getCurveFit(CurveFitType.FIVE_PARAMETER, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, -2.0, 93.0, -0.445, 10.661, 0.838); + assertEquals(3.493, fit.getFitError(), delta); + assertEquals(0.990, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, -2.0, 103.0, -0.325, 6.907, 1.0); + assertEquals(4.126, fit.getFitError(), delta); + assertEquals(0.986, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.FOUR_PARAMETER_SIMPLEX, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, -2.540, 102.643, -0.374, 9.985, 0.0); + assertEquals(0.994, fit.getFitError(), delta); + assertEquals(0.994, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.0, 103.0, -0.445, 6.907, 1.0); + assertEquals(3.631, fit.getFitError(), delta); + assertEquals(0.989, fit.rSquared(fit.getParameters()), delta); + + fit = service.getCurveFit(CurveFitType.THREE_PARAMETER_ALT, data1); + fit.setLogXScale(true); + verifySigmoidalParameters(fit, 0.0, 103.0, 0.445, 6.907, 1.0); + assertEquals(3.631, fit.getFitError(), delta); + assertEquals(0.989, fit.rSquared(fit.getParameters()), delta); + } + + private void verifySigmoidalParameters(CurveFit fit, Double min, Double max, Double slope, Double inflection, Double asymmetry) throws FitFailedException + { + double delta = 0.005; + Map params = fit.getParameters().toMap(); + assertEquals(min, (Double) params.get("min"), delta); + assertEquals(max, (Double) params.get("max"), delta); + if (slope != null) assertEquals(slope, (Double) params.get("slope"), delta); + assertEquals(inflection, (Double) params.get("inflection"), delta); + assertEquals(asymmetry, (Double) params.get("asymmetry"), delta); + } } } From 7e65c6a66bab0368bc4ce9e2e92ef0e85f03794c Mon Sep 17 00:00:00 2001 From: cnathe Date: Thu, 14 Nov 2024 17:06:37 -0600 Subject: [PATCH 11/11] add comments for CurveFit stat methods --- .../labkey/api/data/statistics/CurveFit.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/api/src/org/labkey/api/data/statistics/CurveFit.java b/api/src/org/labkey/api/data/statistics/CurveFit.java index 27d85894db7..82ad9e0f1c7 100644 --- a/api/src/org/labkey/api/data/statistics/CurveFit.java +++ b/api/src/org/labkey/api/data/statistics/CurveFit.java @@ -147,6 +147,11 @@ default JSONObject toJSON() */ double calculateAUC(StatsService.AUCType type, double startX, double endX) throws FitFailedException; + /** + * Calculates the residual sum of squares (RSS) for the curve fit (https://en.wikipedia.org/wiki/Residual_sum_of_squares) + * @param parameters the parameters to use for the give calculated curve fit + * @return the calculated residual sum of squares + */ default double residualSumSquares(P parameters) { double sumSq = 0; @@ -160,11 +165,22 @@ default double residualSumSquares(P parameters) return sumSq; } + /** + * Calculates the root mean square error (RMSE), or root mean square deviation (RMSD), + * for the curve fit (https://en.wikipedia.org/wiki/Root_mean_square_deviation) + * @param parameters the parameters to use for the give calculated curve fit + * @return the calculated root mean square error + */ default double rootMeanSquareError(P parameters) { return Math.sqrt(residualSumSquares(parameters) / getData().length); } + /** + * Calculates the total sum of squares (TSS) for the data points (https://en.wikipedia.org/wiki/Total_sum_of_squares). + * This value is used in the R^2 calculation. + * @return the calculated total sum of squares + */ default double totalSumSquares() { double sumSq = 0; @@ -181,16 +197,29 @@ default double totalSumSquares() return sumSq; } + /** + * Calculates the R^2 value for the curve fit (https://en.wikipedia.org/wiki/Coefficient_of_determination) + * using the residualSumSquares() and totalSumSquares() methods. + * @param parameters the parameters to use for the give calculated curve fit + * @return the calculated R^2 value + */ default double rSquared(P parameters) { return 1 - residualSumSquares(parameters) / totalSumSquares(); } + // see description below, this version of the method is here so that each applicable curve fit can override it + // to set the correct p value for the degrees of freedom default double adjustedRSquared(P parameters) { return Double.NaN; } + /** + * Calculates the adjusted R^2 value for the curve fit (https://en.wikipedia.org/wiki/Coefficient_of_determination) + * @param parameters the parameters to use for the give calculated curve fit + * @return the calculated adjusted R^2 value (if possible) + */ default double adjustedRSquared(P parameters, int p) { int n = getData().length;