From 545ffa8bcae81de42e1e31739c2664d75c04918b Mon Sep 17 00:00:00 2001 From: omaus Date: Wed, 26 Jan 2022 14:14:35 +0100 Subject: [PATCH 1/4] =?UTF-8?q?Add=20Dunn-=C5=A0id=C3=A1k=20correction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/FSharp.Stats.Tests/Testing.fs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/FSharp.Stats.Tests/Testing.fs b/tests/FSharp.Stats.Tests/Testing.fs index a3d475b9..a8b1c506 100644 --- a/tests/FSharp.Stats.Tests/Testing.fs +++ b/tests/FSharp.Stats.Tests/Testing.fs @@ -348,6 +348,23 @@ let benjaminiHochbergTests = ] +[] +let dunnSidakTest = + // taken from: https://www.real-statistics.com/hypothesis-testing/familywise-error/bonferroni-and-dunn-sidak-tests/ + let testPvalues = [0.01208; 0.00356; 0.11542; 0.02155; 0.03329; 0.01042] + let testAlpha1 = 0.05 + let testAlpha2 = 0.01 + let expectedValue1 = 0.008512 + let expectedValue2 = 0.001674 + let observedValue1 = MultipleTesting.dunnSidak testAlpha1 testPvalues |> round 6 + let observedValue2 = MultipleTesting.dunnSidak testAlpha2 testPvalues |> round 6 + testList "Testing.MultipleTesting.dunnSidak" [ + testCase "Alpha = 5 %" <| fun () -> + Expect.floatClose Accuracy.veryHigh observedValue1 expectedValue1 "corrected p values are about equal." + testCase "Alpha = 1 %" <| fun () -> + Expect.floatClose Accuracy.veryHigh observedValue2 expectedValue2 "corrected p values are about equal." + ] + [] let qValuesTest = From a7469a468bbd07e8567724c8d0bc28ca6838cd0a Mon Sep 17 00:00:00 2001 From: omaus Date: Wed, 26 Jan 2022 14:15:04 +0100 Subject: [PATCH 2/4] =?UTF-8?q?Add=20unit=20test=20for=20Dunn-=C5=A0id?= =?UTF-8?q?=C3=A1k=20correction=20:white=5Fcheck=5Fmark:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/FSharp.Stats/Testing/MultipleTesting.fs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/FSharp.Stats/Testing/MultipleTesting.fs b/src/FSharp.Stats/Testing/MultipleTesting.fs index 0a65bee8..d77376e2 100644 --- a/src/FSharp.Stats/Testing/MultipleTesting.fs +++ b/src/FSharp.Stats/Testing/MultipleTesting.fs @@ -63,6 +63,16 @@ module MultipleTesting = |> Seq.ofList + // adapted from: https://en.wikipedia.org/wiki/%C5%A0id%C3%A1k_correction & https://personal.utdallas.edu/~herve/Abdi-Bonferroni2007-pretty.pdf + // Old: /// Computes the Dunn-Šidák correction onto a collection of p-values with a given alpha. Returns a list of booleans which state if the p-value should be rejected or not, and the new critical value for this alpha. + /// Computes the Dunn-Šidák correction onto a collection of p-values with a given alpha. Returns a critical value. p-values above the critical value should be rejected. + let dunnSidak alpha (pValues : seq) = + let m = float (Seq.length pValues) + let criticalValue = 1. - (1. - alpha) ** (1. / m) + // List.map (fun p -> if p > criticalValue then true, p else false, p) (List.ofSeq pValues), + criticalValue + + // John D. Storey // http://dldcc-web.brc.bcm.edu/lilab/liguow/CGI/R/library/qvalue/html/qvalue.html // http://qvalue.princeton.edu/ From f08da6a1355c43b40a55de3736493b968459491a Mon Sep 17 00:00:00 2001 From: omaus Date: Tue, 1 Feb 2022 16:54:45 +0100 Subject: [PATCH 3/4] Apply code review changes :ok_hand: --- src/FSharp.Stats/Testing/MultipleTesting.fs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/FSharp.Stats/Testing/MultipleTesting.fs b/src/FSharp.Stats/Testing/MultipleTesting.fs index d77376e2..308aefd2 100644 --- a/src/FSharp.Stats/Testing/MultipleTesting.fs +++ b/src/FSharp.Stats/Testing/MultipleTesting.fs @@ -64,12 +64,15 @@ module MultipleTesting = // adapted from: https://en.wikipedia.org/wiki/%C5%A0id%C3%A1k_correction & https://personal.utdallas.edu/~herve/Abdi-Bonferroni2007-pretty.pdf - // Old: /// Computes the Dunn-Šidák correction onto a collection of p-values with a given alpha. Returns a list of booleans which state if the p-value should be rejected or not, and the new critical value for this alpha. - /// Computes the Dunn-Šidák correction onto a collection of p-values with a given alpha. Returns a critical value. p-values above the critical value should be rejected. + // checked according to: https://cran.r-project.org/web/packages/aggregation/index.html + // original derivation: https://www.tandfonline.com/doi/abs/10.1080/01621459.1967.10482935 + /// Computes the Dunn-Sidak correction onto a collection of p-values with a given alpha. + /// Returns a critical value. + /// p-values above the critical value should be rejected. Use this correction to reduce family-wise error rate when performing multiple tests on a dataset. p-values above 1, below 0, inf, and nan are excluded. let dunnSidak alpha (pValues : seq) = - let m = float (Seq.length pValues) + let checkedValues = Seq.filter (fun v -> v >= 0. && v <= 1.) pValues + let m = float (Seq.length checkedValues) let criticalValue = 1. - (1. - alpha) ** (1. / m) - // List.map (fun p -> if p > criticalValue then true, p else false, p) (List.ofSeq pValues), criticalValue From cffc7be04fb285798867e8d9a27c47a10f82cb81 Mon Sep 17 00:00:00 2001 From: omaus Date: Tue, 1 Feb 2022 17:02:42 +0100 Subject: [PATCH 4/4] Replace dead link in comment --- tests/FSharp.Stats.Tests/Testing.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FSharp.Stats.Tests/Testing.fs b/tests/FSharp.Stats.Tests/Testing.fs index a8b1c506..c088a3cc 100644 --- a/tests/FSharp.Stats.Tests/Testing.fs +++ b/tests/FSharp.Stats.Tests/Testing.fs @@ -350,7 +350,7 @@ let benjaminiHochbergTests = [] let dunnSidakTest = - // taken from: https://www.real-statistics.com/hypothesis-testing/familywise-error/bonferroni-and-dunn-sidak-tests/ + // taken from: https://web.archive.org/web/20200723122809/https://www.real-statistics.com/hypothesis-testing/familywise-error/bonferroni-and-dunn-sidak-tests/ let testPvalues = [0.01208; 0.00356; 0.11542; 0.02155; 0.03329; 0.01042] let testAlpha1 = 0.05 let testAlpha2 = 0.01