rpgf3_allocator.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "import numpy as np\n",
                "\n",
                "\n",
                "from core import constants\n",
                "from core.utils import *\n",
                "\n",
                "log = get_logger()\n",
                "\n",
                "pd.set_option(\"display.float_format\", \"{:.2f}\".format)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## RPGF 3 Data Check and Cleanup"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "df = pd.read_csv(\"data/dummy_data_rpgf3.csv\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "display(df.sample(10))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "log.info(\"Check - Num Ballots: \" + str(df[\"Has published\"].count()))\n",
                "log.info(\"Check - Num Submissions (Published): \" + str(df[\"Has published\"].sum()))\n",
                "\n",
                "# Check if voter_address is unique\n",
                "if df[\"Address\"].nunique() == df.shape[0]:\n",
                "    log.info(\"Check - Address is unique.\")\n",
                "else:\n",
                "    diff = df.shape[0] - df[\"Address\"].nunique()\n",
                "    log.info(f\"Check - Address is not unique. There are {diff} duplicates.\")\n",
                "\n",
                "# Check if all voters have voted\n",
                "if df[df[\"Has voted\"] == False].shape[0] > 0:\n",
                "    not_voted = df[df[\"Has voted\"] == False].shape[0]\n",
                "    total = df[\"Address\"].nunique()\n",
                "    log.info(f\"Check - {not_voted} voters out of {total} have not voted.\")\n",
                "else:\n",
                "    log.info(\"Check - All voters have voted.\")\n",
                "\n",
                "# Check if all voters have published\n",
                "if df[df[\"Has published\"] == False].shape[0] > 0:\n",
                "    not_voted = df[df[\"Has published\"] == False].shape[0]\n",
                "    total = df[\"Address\"].nunique()\n",
                "    log.info(f\"Check - {not_voted} voters out of {total} have not published.\")\n",
                "else:\n",
                "    log.info(\"Check - All voters have published.\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Apply the function and concatenate results\n",
                "expanded_list = [\n",
                "    expand_json(safe_json_loads(row), idx) for idx, row in df[\"Votes\"].items()\n",
                "]\n",
                "expanded_df = pd.concat(expanded_list, ignore_index=True)\n",
                "\n",
                "result_df = expanded_df.set_index(\"original_index\").join(df.set_index(df.index))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "testing_address = \"zgdSu8Yr87\"\n",
                "print_df = result_df[result_df[\"Address\"] == testing_address]\n",
                "print(\"Num Projects Voted : \" + str(print_df[\"projectId\"].count()))\n",
                "display(print_df.head(10))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "columns = [col for col in result_df.columns if col not in [\"amount\", \"projectId\"]]\n",
                "columns += [\"amount\", \"projectId\"]  # Add the columns to the end of the list\n",
                "result_df = result_df[columns]\n",
                "\n",
                "# Update df columns names\n",
                "result_df.columns = [\n",
                "    \"voter_address\",\n",
                "    \"has_voted\",\n",
                "    \"has_published\",\n",
                "    \"published_at\",\n",
                "    \"created_at\",\n",
                "    \"updated_at\",\n",
                "    \"projects_in_ballot\",\n",
                "    \"votes\",\n",
                "    \"amount\",\n",
                "    \"project_id\",\n",
                "]\n",
                "\n",
                "result_df.drop(columns=\"votes\", inplace=True)\n",
                "\n",
                "result_df[\"amount\"] = pd.to_numeric(result_df[\"amount\"])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# result_df.head()\n",
                "result_df[result_df[\"voter_address\"] == testing_address].head(70)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Calculate Voting Results"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "allocator = ProjectAllocator(\n",
                "    total_amount=constants.TOTAL_AMOUNT,\n",
                "    min_amount=constants.MIN_AMOUNT,\n",
                "    quorum=constants.QUORUM,\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "initial_allocation = allocator.calculate_initial_allocation(result_df)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "display(initial_allocation.sample(1))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Scaling the total to 30M OP by project and filter out those with < 1500 OP\n",
                "allocation_iter = initial_allocation[initial_allocation[\"is_eligible\"] == True].copy()\n",
                "allocation_iter[\"scaled_amount\"] = allocation_iter[\"median_amount\"]\n",
                "# display(allocation_iter)\n",
                "# Set a maximum number of iterations to prevent infinite loop\n",
                "max_iterations = 10\n",
                "current_iteration = 0\n",
                "\n",
                "while (\n",
                "    allocation_iter[\"scaled_amount\"].sum() != constants.TOTAL_AMOUNT\n",
                "    and current_iteration <= max_iterations\n",
                "):\n",
                "    allocation_iter = allocator.scale_allocations_oneby(allocation_iter)\n",
                "    current_iteration += 1\n",
                "\n",
                "    log.info(\"Check - Current iteration: \" + str(current_iteration))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Check if the loop exited due to reaching max iterations\n",
                "if (\n",
                "    current_iteration == max_iterations\n",
                "    and allocation_iter[\"scaled_amount\"].sum() != constants.TOTAL_AMOUNT\n",
                "):\n",
                "    log.info(\"Maximum iterations reached without meeting the total amount condition.\")\n",
                "else:\n",
                "    final_total = allocation_iter[\"scaled_amount\"].sum()\n",
                "    log.info(\n",
                "        f\"Condition met with {final_total} OP allocated through {current_iteration} iteration(s).\"\n",
                "    )"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# join the initial allocation with the final allocation, if scaled_amount is null then make it 0\n",
                "final_allocation = initial_allocation.merge(\n",
                "    allocation_iter[\"scaled_amount\"],\n",
                "    how=\"left\",\n",
                "    on=\"project_id\",\n",
                ").fillna({\"scaled_amount\": 0})\n",
                "\n",
                "# check if the final allocation table still contains all projects.\n",
                "if final_allocation.index.nunique() == result_df[\"project_id\"].nunique():\n",
                "    log.info(\"Check - Final allocation table has included all the projects.\")\n",
                "else:\n",
                "    log.info(\n",
                "        \"Check - Final allocation table has missing projects. Printing out the missing projects below.\"\n",
                "    )\n",
                "    log.info(\n",
                "        result_df[~result_df[\"project_id\"].isin(final_allocation.index)][\"project_id\"]\n",
                "    )\n",
                "\n",
                "# check if the final allocation table still sums to the total amount.\n",
                "if final_allocation[\"scaled_amount\"].sum() == final_total:\n",
                "    log.info(\n",
                "        \"Check - Final allocation table sums to the right amount of OP: \"\n",
                "        + str(final_total)\n",
                "    )\n",
                "else:\n",
                "    log.info(\n",
                "        \"Check - Final allocation table does not sum to the total OP. Printing out the missing amount below.\"\n",
                "    )\n",
                "    log.info(str(final_total - final_allocation[\"scaled_amount\"].sum()) + \" OP\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# export csv\n",
                "# allocation_iter.drop(columns=\"median_amount\", inplace=True)\n",
                "final_allocation.to_csv(\"data/rpgf3_allocation_final.csv\")\n",
                "\n",
                "log.info(f\"Results saved in data/rpgf3_allocation_final.csv.\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "final_allocation.head(10)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "to_cut = (\n",
                "    allocation_iter[allocation_iter[\"scaled_amount\"] < 1500]\n",
                "    .sort_values(by=\"scaled_amount\")\n",
                "    .head(1)\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# check if to_cut is empty\n",
                "to_cut.empty"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Calculate Voting Results using pytorch"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "result_tensor, num_projects = allocator.convert_df_to_tensor(result_df)\n",
                "project_tensors = allocator.get_project_tensor(result_tensor, num_projects)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "# export to onnx\n",
                "allocator.eval()\n",
                "project_tensors = project_tensors\n",
                "\n",
                "# convert projects tensors to tuple\n",
                "final_allocation_torch = allocator.forward(*project_tensors)\n",
                "final_allocation.shape\n",
                "\n",
                "\n",
                "input_names = ['input_' + str(i) for i in range(len(project_tensors))]\n",
                "    # Export the model\n",
                "torch.onnx.export(allocator,               # model being run\n",
                "                      tuple(project_tensors),          # model input (or a tuple for multiple inputs)\n",
                "                      \"network.onnx\",           # where to save the model (can be a file or file-like object)\n",
                "                      export_params=False,       # store the trained parameter weights inside the model file\n",
                "                      opset_version=17,         # the ONNX version to export the model to\n",
                "                      do_constant_folding=False, # whether to execute constant folding for optimization\n",
                "                      input_names = input_names,   # the model's input names\n",
                "                      output_names = ['output'])\n",
                "\n",
                "data_path = os.path.join(\"input.json\")\n",
                "data = dict(input_data = [tensor.detach().numpy().reshape([-1]).tolist() for tensor in project_tensors])\n",
                "# Serialize data into file:\n",
                "json.dump(data, open(data_path, 'w'))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We now include sanity checks for the data. We will check the following:\n",
                "- that the pandas and pytorch dataframes are the same\n",
                "- that the median and scaled median allocations are the same\n",
                "- that eligibility is the same"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "# a bunch of code to compare pytorch output and the pandas output as a sanity check\n",
                "final_allocation_torch_np = final_allocation_torch.detach().numpy()\n",
                "final_allocation_torch_df = pd.DataFrame(final_allocation_torch_np, columns=[\"scaled_amount\"])\n",
                "# add index project_id\n",
                "final_allocation_torch_df = final_allocation_torch_df.set_index(final_allocation.index.sort_values())\n",
                "\n",
                "# sorted vy project id\n",
                "final_allocation_torch_df = final_allocation_torch_df.sort_index()\n",
                "# sort final allocation by project id\n",
                "final_allocation_sorted = final_allocation.sort_index()\n",
                "\n",
                "display(final_allocation_torch_df.head(10))\n",
                "display(final_allocation_sorted.head(10))\n",
                "# compare the two outputs\n",
                "print(\"Are the two outputs equal?\")\n",
                "final_allocation_torch_df.compare(final_allocation_sorted[[\"scaled_amount\"]])\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "TODO: investigate the small differences in the scaled median allocations. Seems like pandas adds some small epsilons during calculations that compound."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "After which we can proceed to generate the settings file for `ezkl` and run calibrate settings to find the optimal settings for `ezkl`"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import ezkl\n",
                "import os\n",
                "\n",
                "model_path = \"network.onnx\"\n",
                "settings_path = \"settings.json\"\n",
                "compiled_model_path = os.path.join('network.compiled')\n",
                "pk_path = os.path.join('test.pk')\n",
                "vk_path = os.path.join('test.vk')\n",
                "settings_path = os.path.join('settings.json')\n",
                "\n",
                "witness_path = os.path.join('witness.json')\n",
                "data_path = os.path.join('input.json')\n",
                "\n",
                "py_run_args = ezkl.PyRunArgs()\n",
                "py_run_args.input_visibility = \"private\"\n",
                "py_run_args.output_visibility = \"public\"\n",
                "py_run_args.param_visibility = \"fixed\" # private by default\n",
                "py_run_args.num_inner_cols = 2\n",
                "# the inputs are integer values !\n",
                "py_run_args.input_scale = 0\n",
                "# the params should be very large\n",
                "py_run_args.param_scale = 14\n",
                "#  scale rebase multiplier is 10\n",
                "py_run_args.scale_rebase_multiplier = 1\n",
                "\n",
                "res = ezkl.gen_settings(model_path, settings_path, py_run_args=py_run_args)\n",
                "assert res == True"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "ezkl.calibrate_settings(data_path, model_path, settings_path, \"accuracy\", lookup_safety_margin=2, scales=[30], only_range_check_rebase=True, scale_rebase_multiplier=[1])\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Next, we will compile the model. The compilation step allow us to generate proofs faster."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "res = ezkl.compile_circuit(model_path, compiled_model_path, settings_path)\n",
                "assert res == True"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Mock prove as a sanity check to ensure that the model is working as expected.\n",
                "\n",
                "Finally, we will generate the proofs and submit the results."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# now generate the witness file \n",
                "\n",
                "res = ezkl.gen_witness(data_path, compiled_model_path, witness_path)\n",
                "assert os.path.isfile(witness_path)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "res = ezkl.mock(witness_path, compiled_model_path)\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Before we can setup the circuit params, we need a SRS (Structured Reference String). The SRS is used to generate the proofs."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# srs path\n",
                "res = ezkl.get_srs(settings_path)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now run setup, this will generate a proving key (pk) and verification key (vk). The proving key is used for proving while the verification key is used for verificaton."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "# setup\n",
                "res = ezkl.setup(\n",
                "        compiled_model_path,\n",
                "        vk_path,\n",
                "        pk_path,\n",
                "    )\n",
                "\n",
                "assert res == True\n",
                "assert os.path.isfile(vk_path)\n",
                "assert os.path.isfile(pk_path)\n",
                "assert os.path.isfile(settings_path)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# GENERATE A PROOF\n",
                "proof_path = os.path.join('test.pf')\n",
                "\n",
                "res = ezkl.prove(\n",
                "        witness_path,\n",
                "        compiled_model_path,\n",
                "        pk_path,\n",
                "        proof_path,\n",
                "        \"single\",\n",
                "    )\n",
                "\n",
                "print(res)\n",
                "assert os.path.isfile(proof_path)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can now verify the proof. "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# VERIFY IT\n",
                "res = ezkl.verify(\n",
                "        proof_path,\n",
                "        settings_path,\n",
                "        vk_path,\n",
                "    )\n",
                "\n",
                "assert res == True\n",
                "print(\"verified\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can now create an EVM / `.sol` verifier that can be deployed on chain to verify submitted proofs using a view function."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "abi_path = 'test.abi'\n",
                "sol_code_path = 'test_1.sol'\n",
                "\n",
                "res = ezkl.create_evm_verifier(\n",
                "        vk_path,\n",
                "        settings_path,\n",
                "        sol_code_path,\n",
                "        abi_path,\n",
                "    )\n",
                "assert res == True"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Sanity checks on circuit outputs"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "import onnx\n",
                "import onnxruntime\n",
                "import os\n",
                "import json\n",
                "import ezkl\n",
                "import numpy as np\n",
                "\n",
                "witness_path = os.path.join('witness.json')\n",
                "settings_path = os.path.join('settings.json')\n",
                "model_path = os.path.join('network.onnx')\n",
                "data_path = os.path.join('input.json')\n",
                "\n",
                "def get_ezkl_output(witness_file, settings_file):\n",
                "    # convert the quantized ezkl output to float value\n",
                "    witness_output = json.load(open(witness_file))\n",
                "    outputs = witness_output['outputs']\n",
                "    with open(settings_file) as f:\n",
                "        settings = json.load(f)\n",
                "    ezkl_outputs = [[ezkl.string_to_float(\n",
                "        outputs[i][j], settings['model_output_scales'][i]) for j in range(len(outputs[i]))] for i in range(len(outputs))]\n",
                "    return ezkl_outputs\n",
                "\n",
                "\n",
                "def get_onnx_output(model_file, input_file):\n",
                "    # generate the ML model output from the ONNX file\n",
                "    onnx_model = onnx.load(model_file)\n",
                "    onnx.checker.check_model(onnx_model)\n",
                "\n",
                "    with open(input_file) as f:\n",
                "        inputs = json.load(f)\n",
                "    # reshape the input to the model\n",
                "    num_inputs = len(inputs['input_data'])\n",
                "\n",
                "    onnx_input = dict()\n",
                "    for i in range(num_inputs):\n",
                "        input_node = onnx_model.graph.input[i]\n",
                "        dims = []\n",
                "        elem_type = input_node.type.tensor_type.elem_type\n",
                "        for dim in input_node.type.tensor_type.shape.dim:\n",
                "            if dim.dim_value == 0:\n",
                "                dims.append(1)\n",
                "            else:\n",
                "                dims.append(dim.dim_value)\n",
                "        if elem_type == 7:\n",
                "            inputs_onnx = np.array(inputs['input_data'][i]).astype(\n",
                "                np.int64).reshape(dims)\n",
                "        elif elem_type == 9:\n",
                "            inputs_onnx = np.array(inputs['input_data'][i]).astype(\n",
                "                bool).reshape(dims)\n",
                "        else:\n",
                "            inputs_onnx = np.array(inputs['input_data'][i]).astype(\n",
                "                np.float32).reshape(dims)\n",
                "        onnx_input[input_node.name] = inputs_onnx\n",
                "    try:\n",
                "        onnx_session = onnxruntime.InferenceSession(model_file)\n",
                "        onnx_output = onnx_session.run(None, onnx_input)\n",
                "    except Exception as e:\n",
                "        print(\"Error in ONNX runtime: \", e)\n",
                "        print(\"using inputs[output_data]\")\n",
                "        onnx_output = inputs['output_data']\n",
                "    return onnx_output[0]\n",
                "\n",
                "\n",
                "def compare_outputs(zk_output, onnx_output):\n",
                "    # calculate hamming difference between the 2 outputs (which are lists)\n",
                "\n",
                "    res = []\n",
                "\n",
                "\n",
                "    contains_sublist = any(isinstance(sub, list) for sub in zk_output)\n",
                "    if contains_sublist:\n",
                "        try:\n",
                "            if len(onnx_output) == 1:\n",
                "                zk_output = zk_output[0]\n",
                "        except Exception as e:\n",
                "            zk_output = zk_output[0]\n",
                "\n",
                "    flat_zk_output = np.array(zk_output).flatten()\n",
                "    flat_onnx_output = np.array(onnx_output).flatten()\n",
                "\n",
                "    # select every 4th element\n",
                "    scaled_amount_zk = flat_zk_output[3::4]\n",
                "    scaled_amount_onnx = flat_onnx_output[3::4]\n",
                "\n",
                "    # print the diff where they are not both 0\n",
                "    print(\"scaled amount diff\")\n",
                "    for i in range(len(scaled_amount_zk)):\n",
                "        if scaled_amount_zk[i] != 0.0 and scaled_amount_onnx[i] != 0.0:\n",
                "            print(\"scaled amount zk: \", scaled_amount_zk[i])\n",
                "            print(\"scaled amount onnx: \", scaled_amount_onnx[i])\n",
                "            print(\"diff: \", scaled_amount_zk[i] - scaled_amount_onnx[i])\n",
                "            print(\"percent diff: \", 100*(scaled_amount_zk[i] - scaled_amount_onnx[i]) / scaled_amount_onnx[i])\n",
                "\n",
                "    zip_object = zip(flat_zk_output[3::4], flat_onnx_output[3::4])\n",
                "\n",
                "    for list1_i, list2_i in zip_object:\n",
                "        if list1_i == 0.0 and list2_i == 0.0:\n",
                "            res.append(0)\n",
                "        else:\n",
                "            percent_diff = (list1_i - list2_i) / list2_i\n",
                "            res.append(percent_diff)\n",
                "\n",
                "\n",
                "    return np.abs(res)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "import plotly.express as px\n",
                "import pandas as pd\n",
                "\n",
                "import matplotlib.pyplot as plt\n",
                "    # get the ezkl output\n",
                "ezkl_output = get_ezkl_output(witness_path, settings_path)\n",
                "    # get the onnx output\n",
                "onnx_output = get_onnx_output(model_path, data_path)\n",
                "    # compare the outputs\n",
                "l1_difference = compare_outputs(ezkl_output, onnx_output)\n",
                "\n",
                "df = pd.DataFrame(l1_difference, columns=[\"percent error\"])\n",
                "\n",
                "\n",
                "# Create a histogram\n",
                "fig = px.histogram(df, x=\"percent error\",\n",
                "                  title=\"Distribution of percent error\")\n",
                "fig.show()"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "analytics-internal-nXLXwD2Z",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.9.13"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}