From 9aa07aebbdc62461c3ac32f958d7c3ab89ff6f73 Mon Sep 17 00:00:00 2001
From: Jeffrey Walton <noloader@gmail.com>
Date: Mon, 10 Jun 2024 10:27:30 -0400
Subject: [PATCH] Guard use of ModularSquareRoot (GH #1249)

---
 ecp.cpp      |  7 ++++++-
 nbtheory.cpp | 18 ++++++++++++++++++
 rabin.cpp    |  7 +++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/ecp.cpp b/ecp.cpp
index 643ba70a0..afff31790 100644
--- a/ecp.cpp
+++ b/ecp.cpp
@@ -119,7 +119,11 @@ bool ECP::DecodePoint(ECP::Point &P, BufferedTransformation &bt, size_t encodedP
 		if (encodedPointLen != EncodedPointSize(true))
 			return false;
 
-		Integer p = FieldSize();
+		// Check for p is prime due to GH #1249
+		const Integer p = FieldSize();
+		CRYPTOPP_ASSERT(IsPrime(p));
+		if (!IsPrime(p))
+			return false;
 
 		P.identity = false;
 		P.x.Decode(bt, GetField().MaxElementByteLength());
@@ -128,6 +132,7 @@ bool ECP::DecodePoint(ECP::Point &P, BufferedTransformation &bt, size_t encodedP
 		if (Jacobi(P.y, p) !=1)
 			return false;
 
+		// Callers must ensure p is prime, GH #1249
 		P.y = ModularSquareRoot(P.y, p);
 
 		if ((type & 1) != P.y.GetBit(0))
diff --git a/nbtheory.cpp b/nbtheory.cpp
index 7e60be4b6..17fec4cb3 100644
--- a/nbtheory.cpp
+++ b/nbtheory.cpp
@@ -11,6 +11,7 @@
 #include "smartptr.h"
 #include "misc.h"
 #include "stdcpp.h"
+#include "trap.h"
 
 #ifdef _OPENMP
 # include <omp.h>
@@ -524,6 +525,9 @@ Integer MaurerProvablePrime(RandomNumberGenerator &rng, unsigned int bits)
 
 Integer CRT(const Integer &xp, const Integer &p, const Integer &xq, const Integer &q, const Integer &u)
 {
+	// Callers must ensure p and q are prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p) && IsPrime(q));
+
 	// isn't operator overloading great?
 	return p * (u * (xq-xp) % q) + xp;
 /*
@@ -543,6 +547,9 @@ Integer CRT(const Integer &xp, const Integer &p, const Integer &xq, const Intege
 
 Integer ModularSquareRoot(const Integer &a, const Integer &p)
 {
+	// Callers must ensure p is prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p));
+
 	if (p%4 == 3)
 		return a_exp_b_mod_c(a, (p+1)/4, p);
 
@@ -592,6 +599,9 @@ Integer ModularSquareRoot(const Integer &a, const Integer &p)
 
 bool SolveModularQuadraticEquation(Integer &r1, Integer &r2, const Integer &a, const Integer &b, const Integer &c, const Integer &p)
 {
+	// Callers must ensure p is prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p));
+
 	Integer D = (b.Squared() - 4*a*c) % p;
 	switch (Jacobi(D, p))
 	{
@@ -618,6 +628,9 @@ bool SolveModularQuadraticEquation(Integer &r1, Integer &r2, const Integer &a, c
 Integer ModularRoot(const Integer &a, const Integer &dp, const Integer &dq,
 					const Integer &p, const Integer &q, const Integer &u)
 {
+	// Callers must ensure p and q are prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p) && IsPrime(q));
+
 	// GCC warning bug, https://stackoverflow.com/q/12842306/608639
 #ifdef _OPENMP
 	Integer p2, q2;
@@ -640,6 +653,9 @@ Integer ModularRoot(const Integer &a, const Integer &dp, const Integer &dq,
 Integer ModularRoot(const Integer &a, const Integer &e,
 					const Integer &p, const Integer &q)
 {
+	// Callers must ensure p and q are prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p) && IsPrime(q));
+
 	Integer dp = EuclideanMultiplicativeInverse(e, p-1);
 	Integer dq = EuclideanMultiplicativeInverse(e, q-1);
 	Integer u = EuclideanMultiplicativeInverse(p, q);
@@ -976,6 +992,8 @@ Integer Lucas(const Integer &n, const Integer &P, const Integer &modulus)
 
 Integer InverseLucas(const Integer &e, const Integer &m, const Integer &p, const Integer &q, const Integer &u)
 {
+	// Callers must ensure p and q are prime, GH #1249
+	CRYPTOPP_ASSERT(IsPrime(p) && IsPrime(q));
 
 	// GCC warning bug, https://stackoverflow.com/q/12842306/608639
 #ifdef _OPENMP
diff --git a/rabin.cpp b/rabin.cpp
index e112d9b86..5930ec470 100644
--- a/rabin.cpp
+++ b/rabin.cpp
@@ -7,6 +7,7 @@
 #include "modarith.h"
 #include "asn.h"
 #include "sha.h"
+#include "trap.h"
 
 NAMESPACE_BEGIN(CryptoPP)
 
@@ -130,6 +131,9 @@ void InvertibleRabinFunction::BERDecode(BufferedTransformation &bt)
 	m_q.BERDecode(seq);
 	m_u.BERDecode(seq);
 	seq.MessageEnd();
+
+	CRYPTOPP_ASSERT(IsPrime(m_p));
+	CRYPTOPP_ASSERT(IsPrime(m_q));
 }
 
 void InvertibleRabinFunction::DEREncode(BufferedTransformation &bt) const
@@ -146,6 +150,9 @@ void InvertibleRabinFunction::DEREncode(BufferedTransformation &bt) const
 
 Integer InvertibleRabinFunction::CalculateInverse(RandomNumberGenerator &rng, const Integer &in) const
 {
+	CRYPTOPP_ASSERT(IsPrime(m_p));
+	CRYPTOPP_ASSERT(IsPrime(m_q));
+
 	DoQuickSanityCheck();
 
 	ModularArithmetic modn(m_n);