Skip to content

Commit

Permalink
Calldata compression (#1)
Browse files Browse the repository at this point in the history 
* Added transformation of ring data to make the it easier to compress

* Fixed ring settlement without DA

* Renamed numElements to blockSize for consistency

* More flexible ring data transformation

* Small data transform change
  • Loading branch information
@Brechtpd
Brechtpd authored Jun 25, 2019
1 parent 3650674 commit 9dac6d1
Show file tree
Hide file tree
Showing 6 changed files with 243 additions and 34 deletions.
119 changes: 116 additions & 3 deletions Circuits/RingSettlementCircuit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,91 @@ using namespace ethsnarks;
namespace Loopring
{

class TransformRingSettlementDataGadget : public GadgetT
{
public:

const unsigned int ringSize = 25 * 8;

VariableArrayT data;
Bitstream transformedData;
unsigned int numRings;

std::vector<XorArrayGadget> xorGadgets;

TransformRingSettlementDataGadget(
ProtoboardT& pb,
const std::string& prefix
) :
GadgetT(pb, prefix)
{
numRings = 0;
}

VariableArrayT result()
{
return flatten(transformedData.data);
}

void generate_r1cs_witness()
{
for (unsigned int i = 0; i < xorGadgets.size(); i++)
{
xorGadgets[i].generate_r1cs_witness();
}
}

void generate_r1cs_constraints(unsigned int numRings, const VariableArrayT& data)
{
this->numRings = numRings;
this->data = data;
assert(numRings > 0);
assert(numRings * ringSize == data.size());

// XOR compress
Bitstream compressedData;
compressedData.add(subArray(data, 0, ringSize));
for (unsigned int i = 1; i < numRings; i++)
{
unsigned int previousRingStart = (i - 1) * ringSize;
unsigned int ringStart = i * ringSize;

xorGadgets.emplace_back(pb, subArray(data, previousRingStart, 5 * 8),
subArray(data, ringStart, 5 * 8),
std::string("xor_") + std::to_string(i));
xorGadgets.back().generate_r1cs_constraints();
compressedData.add(xorGadgets.back().result());
compressedData.add(subArray(data, ringStart + 5 * 8, ringSize - 5 * 8));
}

// Transform
struct Range
{
unsigned int offset;
unsigned int length;
};
std::vector<std::vector<Range>> ranges;
ranges.push_back({{0, 40}}); // ringMatcherID + fFee + tokenID
ranges.push_back({{40, 40}}); // orderA.orderID + orderB.orderID
ranges.push_back({{80, 40}}); // orderA.accountID + orderB.accountID
ranges.push_back({{120, 8}, {160, 8}}); // orderA.tokenS + orderB.tokenS
ranges.push_back({{128, 24},{168, 24}}); // orderA.fillS + orderB.fillS
ranges.push_back({{152, 8}}); // orderA.data
ranges.push_back({{192, 8}}); // orderB.data
for (const std::vector<Range>& subRanges : ranges)
{
for (unsigned int i = 0; i < numRings; i++)
{
for (const Range& subRange : subRanges)
{
unsigned int ringStart = i * ringSize;
transformedData.add(subArray(flatten(compressedData.data), ringStart + subRange.offset, subRange.length));
}
}
}
}
};

class RingSettlementGadget : public GadgetT
{
public:
Expand Down Expand Up @@ -49,6 +134,9 @@ class RingSettlementGadget : public GadgetT
OrderGadget orderA;
OrderGadget orderB;

ForceNotEqualGadget accountA_neq_ringMatcher;
ForceNotEqualGadget accountB_neq_ringMatcher;

OrderMatchingGadget orderMatching;

TernaryGadget uFillS_A;
Expand Down Expand Up @@ -147,6 +235,9 @@ class RingSettlementGadget : public GadgetT
orderA(pb, params, constants, _exchangeID, FMT(prefix, ".orderA")),
orderB(pb, params, constants, _exchangeID, FMT(prefix, ".orderB")),

accountA_neq_ringMatcher(pb, orderA.accountID.packed, minerAccountID.packed, FMT(prefix, ".accountA_neq_ringMatcher")),
accountB_neq_ringMatcher(pb, orderB.accountID.packed, minerAccountID.packed, FMT(prefix, ".accountB_neq_ringMatcher")),

// Match orders
orderMatching(pb, constants, _timestamp, orderA, orderB, FMT(prefix, ".orderMatching")),

Expand Down Expand Up @@ -350,11 +441,16 @@ class RingSettlementGadget : public GadgetT
orderA.generate_r1cs_witness(ringSettlement.ring.orderA,
ringSettlement.accountUpdate_A.before,
ringSettlement.balanceUpdateS_A.before,
ringSettlement.balanceUpdateB_A.before);
ringSettlement.balanceUpdateB_A.before,
ringSettlement.tradeHistoryUpdate_A.before);
orderB.generate_r1cs_witness(ringSettlement.ring.orderB,
ringSettlement.accountUpdate_B.before,
ringSettlement.balanceUpdateS_B.before,
ringSettlement.balanceUpdateB_B.before);
ringSettlement.balanceUpdateB_B.before,
ringSettlement.tradeHistoryUpdate_B.before);

accountA_neq_ringMatcher.generate_r1cs_witness();
accountB_neq_ringMatcher.generate_r1cs_witness();

// Match orders
orderMatching.generate_r1cs_witness();
Expand Down Expand Up @@ -457,6 +553,9 @@ class RingSettlementGadget : public GadgetT
orderA.generate_r1cs_constraints();
orderB.generate_r1cs_constraints();

accountA_neq_ringMatcher.generate_r1cs_constraints();
accountB_neq_ringMatcher.generate_r1cs_constraints();

// Match orders
orderMatching.generate_r1cs_constraints();

Expand Down Expand Up @@ -529,6 +628,8 @@ class RingSettlementCircuit : public GadgetT
std::vector<RingSettlementGadget*> ringSettlements;

libsnark::dual_variable_gadget<FieldT> publicDataHash;
Bitstream dataAvailabityData;
TransformRingSettlementDataGadget transformData;
PublicDataGadget publicData;

Constants constants;
Expand All @@ -555,6 +656,7 @@ class RingSettlementCircuit : public GadgetT
GadgetT(pb, prefix),

publicDataHash(pb, 256, FMT(prefix, ".publicDataHash")),
transformData(pb, FMT(prefix, ".transformData")),
publicData(pb, publicDataHash, FMT(prefix, ".publicData")),

constants(pb, FMT(prefix, ".constants")),
Expand Down Expand Up @@ -644,7 +746,7 @@ class RingSettlementCircuit : public GadgetT
if (onchainDataAvailability)
{
// Store data from ring settlement
publicData.add(ringSettlements.back()->getPublicData());
dataAvailabityData.add(ringSettlements.back()->getPublicData());
}
}

Expand All @@ -662,6 +764,13 @@ class RingSettlementCircuit : public GadgetT
FMT(annotation_prefix, ".updateAccount_O"));
updateAccount_O->generate_r1cs_constraints();

if (onchainDataAvailability)
{
// Transform the data
transformData.generate_r1cs_constraints(numRings, flattenReverse(dataAvailabityData.data));
publicData.add(flattenReverse({transformData.result()}));
}

// Check the input hash
publicDataHash.generate_r1cs_constraints(true);
publicData.generate_r1cs_constraints();
Expand Down Expand Up @@ -715,6 +824,10 @@ class RingSettlementCircuit : public GadgetT
updateAccount_P->generate_r1cs_witness(block.accountUpdate_P.proof);
updateAccount_O->generate_r1cs_witness(block.accountUpdate_O.proof);

if (onchainDataAvailability)
{
transformData.generate_r1cs_witness();
}
publicData.generate_r1cs_witness();

return true;
Expand Down
103 changes: 103 additions & 0 deletions Gadgets/MathGadgets.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,92 @@ class NotGadget : public GadgetT
}
};

class XorGadget : public GadgetT
{
public:
VariableT A;
VariableT B;
VariableT C;

XorGadget(
ProtoboardT& pb,
const VariableT& _A,
const VariableT& _B,
const std::string& prefix
) :
GadgetT(pb, prefix),

A(_A),
B(_B),

C(make_variable(pb, FMT(prefix, ".C")))
{

}

const VariableT& result() const
{
return C;
}

void generate_r1cs_witness()
{
pb.val(C) = pb.val(A) + pb.val(B) - ((pb.val(A) == FieldT::one() && pb.val(B) == FieldT::one()) ? 2 : 0);
}

void generate_r1cs_constraints()
{
pb.add_r1cs_constraint(ConstraintT(2 * A, B, A + B - C), FMT(annotation_prefix, ".A ^ B == C"));
}
};

class XorArrayGadget : public GadgetT
{
public:
VariableArrayT A;
VariableArrayT B;
VariableArrayT C;

XorArrayGadget(
ProtoboardT& pb,
VariableArrayT _A,
VariableArrayT _B,
const std::string& prefix
) :
GadgetT(pb, prefix),

A(_A),
B(_B),

C(make_var_array(pb, A.size(), FMT(prefix, ".C")))
{
assert(A.size() == B.size());
}

const VariableArrayT& result() const
{
return C;
}

void generate_r1cs_witness()
{
for (unsigned int i = 0; i < C.size(); i++)
{
pb.val(C[i]) = pb.val(A[i]) + pb.val(B[i]) - ((pb.val(A[i]) == FieldT::one() && pb.val(B[i]) == FieldT::one()) ? 2 : 0);
}
// printBits("A: ", A.get_bits(pb));
// printBits("B: ", B.get_bits(pb));
// printBits("C: ", C.get_bits(pb));
}

void generate_r1cs_constraints()
{
for (unsigned int i = 0; i < C.size(); i++)
{
pb.add_r1cs_constraint(ConstraintT(2 * A[i], B[i], A[i] + B[i] - C[i]), FMT(annotation_prefix, ".A ^ B == C"));
}
}
};

class EqualGadget : public GadgetT
{
Expand Down Expand Up @@ -1030,6 +1116,23 @@ class SignatureVerifier : public GadgetT
}
};

class Bitstream
{
public:

std::vector<VariableArrayT> data;

void add(const VariableArrayT& bits)
{
data.push_back(bits);
}

void add(const std::vector<VariableArrayT>& bits)
{
data.insert(data.end(), bits.begin(), bits.end());
}
};

class PublicDataGadget : public GadgetT
{
public:
Expand Down
9 changes: 5 additions & 4 deletions Gadgets/OrderGadgets.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ class OrderGadget : public GadgetT
}

void generate_r1cs_witness(const Order& order, const Account& account,
const BalanceLeaf& balanceLeafS, const BalanceLeaf& balanceLeafB)
const BalanceLeaf& balanceLeafS, const BalanceLeaf& balanceLeafB,
const TradeHistoryLeaf& tradeHistoryLeaf)
{
exchangeID.bits.fill_with_bits_of_field_element(pb, order.exchangeID);
exchangeID.generate_r1cs_witness_from_bits();
Expand Down Expand Up @@ -184,9 +185,9 @@ class OrderGadget : public GadgetT
amountS_notZero.generate_r1cs_witness();
amountB_notZero.generate_r1cs_witness();

pb.val(tradeHistoryFilled) = order.tradeHistoryFilled;
pb.val(tradeHistoryCancelled) = order.tradeHistoryCancelled;
pb.val(tradeHistoryOrderID) = order.tradeHistoryOrderID;
pb.val(tradeHistoryFilled) = tradeHistoryLeaf.filled;
pb.val(tradeHistoryCancelled) = tradeHistoryLeaf.cancelled;
pb.val(tradeHistoryOrderID) = tradeHistoryLeaf.orderID;

tradeHistory.generate_r1cs_witness();

Expand Down
8 changes: 0 additions & 8 deletions Utils/Data.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,6 @@ class Order
ethsnarks::FieldT rebateBips;

Signature signature;

ethsnarks::FieldT tradeHistoryFilled;
ethsnarks::FieldT tradeHistoryCancelled;
ethsnarks::FieldT tradeHistoryOrderID;
};

void from_json(const json& j, Order& order)
Expand All @@ -200,10 +196,6 @@ void from_json(const json& j, Order& order)
order.rebateBips = ethsnarks::FieldT(j.at("rebateBips"));

order.signature = j.at("signature").get<Signature>();

order.tradeHistoryFilled = ethsnarks::FieldT(j.at("tradeHistoryFilled").get<std::string>().c_str());
order.tradeHistoryCancelled = ethsnarks::FieldT(j.at("tradeHistoryCancelled"));
order.tradeHistoryOrderID = ethsnarks::FieldT(j.at("tradeHistoryOrderID"));
}

class Ring
Expand Down
Loading

0 comments on commit 9dac6d1

Please sign in to comment.